def prepare(self, measurand, inputs, options=dict()):
"""
Prepares a test for a regression model
Parameters
----------
measurand: dict
measurand = {'8019043': ['NO2']}
inputs: dict
inputs per device and reading
inputs = {'devicename': ['reading-1', 'reading-2']}
options: dict
Options including data processing. Defaults in config._model_def_opt
Returns
-------
df = pandas Dataframe
measurand_name = string
"""
options = dict_fmerge(options, config._model_def_opt)
# Measurand
measurand_device = list(measurand.keys())[0]
measurand_metric = measurand[measurand_device][0]
measurand_name = measurand[measurand_device][0] + '_' + measurand_device
df = DataFrame()
df[measurand_name] = self.devices[measurand_device].readings[
measurand_metric]
for input_device in inputs.keys():
combined_df = self.combine(devices=[input_device],
readings=inputs[input_device])
df = df.combine_first(combined_df)
if options['common_avg']:
common_channels = inputs[list(inputs.keys())[0]]
for input_device in inputs.keys():
common_channels = list(
set(common_channels).intersection(set(inputs[input_device])))
std_out(f'Performing avg in common columns {common_channels}')
for channel in common_channels:
columns_list = [
channel + '_' + device for device in list(inputs.keys())
]
df[channel + '_AVG'] = df[columns_list].mean(axis=1)
df = df.loc[:,
df.columns.str.contains("_AVG")
| df.columns.str.contains(measurand_name)]
if options['clean_na'] is not None:
df = clean(df, options['clean_na'], how='any')
return df, measurand_name
def apply_regressor(dataframe, **kwargs):
'''
Applies a regressor model based on a pretrained model
Parameters
----------
model: sklearn predictor
Model with .predict method
options: dict
Options for data preprocessing. Defaults in config.model_def_opt
variables: dict
variables dictionary with:
{
'measurand': {
'measurand-device-name': ['measurand']
},
'inputs': {'input-device-names': ['input-1', 'input_2', 'input-3']
}
}
Returns
----------
pandas series containing the prediction
'''
inputs = list()
for device in kwargs['variables']['inputs']:
inputs = list(
set(inputs).union(set(kwargs['variables']['inputs'][device])))
try:
inputdf = dataframe[inputs].copy()
inputdf = inputdf.reindex(sorted(inputdf.columns), axis=1)
except KeyError:
std_out('Inputs not in dataframe', 'ERROR')
pass
return None
if 'model' not in kwargs:
std_out('Model not in inputs', 'ERROR')
else:
model = kwargs['model']
if 'options' not in kwargs:
options = config.model_def_opt
else:
options = dict_fmerge(config.model_def_opt, kwargs['options'])
# Remove na
inputdf = clean(inputdf, options['clean_na'], how='any')
features = array(inputdf)
result = DataFrame(model.predict(features)).set_index(inputdf.index)
return result
def set_descriptor_attrs(self):
# Descriptor attributes
for ditem in self.description.keys():
if ditem not in vars(self):
std_out(f'Ignoring {ditem} from input')
continue
if type(self.__getattribute__(ditem)) == dict:
self.__setattr__(
ditem,
dict_fmerge(self.__getattribute__(ditem),
self.description[ditem]))
else:
self.__setattr__(ditem, self.description[ditem])
def merge_sensor_metrics(self, ignore_empty=True):
std_out('Merging sensor and metrics channels')
all_channels = dict_fmerge(self.sensors, self.metrics)
if ignore_empty:
to_ignore = []
for channel in all_channels.keys():
if channel not in self.readings: to_ignore.append(channel)
elif self.readings[channel].dropna().empty:
std_out(f'{channel} is empty')
to_ignore.append(channel)
[all_channels.pop(x) for x in to_ignore]
return all_channels
def __init__(self, blueprint, descriptor):
'''
Creates an instance of device. Devices are objects that contain sensors readings, metrics
(calculations based on sensors readings), and metadata such as units, dates, frequency and source
Parameters:
-----------
blueprint: String
Defines the type of device. For instance: sck_21, sck_20, csic_station, muv_station
parrot_soil, sc_20_station, sc_21_station. A list of all the blueprints is found in
blueprints.yaml and accessible via the scdata.utils.load_blueprints function.
descriptor: dict()
A dictionary containing information about the device itself. Depending on the blueprint, this descriptor
needs to have different data. If not all the data is present, the corresponding blueprint's default will
be used
Returns
----------
Device object
'''
self.blueprint = blueprint
# Set attributes
for bpitem in config.blueprints[blueprint]:
self.__setattr__(bpitem, config.blueprints[blueprint][bpitem])
for ditem in descriptor.keys():
if type(self.__getattribute__(ditem)) == dict:
self.__setattr__(
ditem,
dict_fmerge(self.__getattribute__(ditem),
descriptor[ditem]))
else:
self.__setattr__(ditem, descriptor[ditem])
# Add API handler if needed
if self.source == 'api':
hmod = __import__('scdata.io.read_api', fromlist=['io.read_api'])
Hclass = getattr(hmod, self.sources[self.source]['handler'])
# Create object
self.api_device = Hclass(did=self.id)
self.readings = DataFrame()
self.loaded = False
self.options = dict()
def ts_scatter(self, **kwargs):
"""
Plots timeseries and scatter comparison in matplotlib
Parameters
----------
traces: dict
Data for the plot, with the format:
traces = {
"1": {"devices": "10751",
"channel": "EXT_PM_A_1"},
"2": {"devices": "10751",
"channel": "EXT_PM_A_10"
}
}
options: dict
Options including data processing prior to plot. Defaults in config.plot_def_opt
formatting: dict
Name of auxiliary electrode found in dataframe
Returns
-------
Matplotlib figure containing timeseries and scatter plot with correlation
coefficients on it
"""
if config.framework == 'jupyterlab': plt.ioff()
plt.clf()
if 'traces' not in kwargs:
std_out('No traces defined', 'ERROR')
return None
else:
traces = kwargs['traces']
if 'options' not in kwargs:
std_out('Using default options')
options = config.plot_def_opt
else:
options = dict_fmerge(config.plot_def_opt, kwargs['options'])
if 'formatting' not in kwargs:
std_out('Using default formatting')
formatting = config.ts_scatter_def_fmt['mpl']
else:
formatting = dict_fmerge(config.ts_scatter_def_fmt['mpl'],
kwargs['formatting'])
# Style
if formatting['style'] is not None: style.use(formatting['style'])
else: style.use(config.plot_style)
# Palette
if formatting['palette'] is not None: set_palette(formatting['palette'])
# Font size
if formatting['fontsize'] is not None:
rcParams.update({'font.size': formatting['fontsize']})
# Make it standard
for trace in traces:
if 'subplot' not in trace: traces[trace]['subplot'] = 1
# Get dataframe
df, subplots = prepare_data(self, traces, options)
n_subplots = len(subplots)
fig = plt.figure(figsize=(formatting['width'], formatting['height']))
gs = gridspec.GridSpec(1, 3, figure=fig)
ax1 = fig.add_subplot(gs[0, :-1])
ax2 = fig.add_subplot(gs[0, -1])
feature_trace = subplots[0][0]
ref_trace = subplots[0][1]
# Calculate basic metrics
pearsonCorr = list(df.corr('pearson')[list(df.columns)[0]])[-1]
rmse = sqrt(
mean_squared_error(df[feature_trace].fillna(0),
df[ref_trace].fillna(0)))
std_out(f'Pearson correlation coefficient: {pearsonCorr}')
std_out(f'Coefficient of determination R²: {pearsonCorr*pearsonCorr}')
std_out(f'RMSE: {rmse}')
# Time Series plot
ax1.plot(df.index,
df[feature_trace],
color='g',
label=feature_trace,
linewidth=1,
alpha=0.9)
ax1.plot(df.index,
df[ref_trace],
color='k',
label=ref_trace,
linewidth=1,
alpha=0.7)
ax1.axis('tight')
# Correlation plot
ax2.plot(df[ref_trace],
df[feature_trace],
'go',
label=feature_trace,
linewidth=1,
alpha=0.3)
ax2.plot(df[ref_trace],
df[ref_trace],
'k',
label='1:1 Line',
linewidth=0.2,
alpha=0.6)
ax2.axis('tight')
if formatting['title'] is not None:
ax1.set_title('Time Series Plot for {}'.format(formatting['title']),
fontsize=formatting['title_fontsize'])
ax2.set_title('Scatter Plot for {}'.format(formatting['title']),
fontsize=formatting['title_fontsize'])
if formatting['grid'] is not None:
ax1.grid(formatting['grid'])
ax2.grid(formatting['grid'])
if formatting['legend']:
ax1.legend(loc="best")
ax2.legend(loc="best")
if formatting['ylabel'] is not None:
ax1.set_ylabel(formatting['ylabel'])
ax2.set_xlabel(formatting['ylabel'])
ax2.set_ylabel(formatting['ylabel'])
if formatting['xlabel'] is not None:
ax1.set_xlabel(formatting['xlabel'])
if formatting['yrange'] is not None:
ax1.set_ylim(formatting['yrange'])
ax2.set_xlim(formatting['yrange'])
ax2.set_ylim(formatting['yrange'])
if formatting['xrange'] is not None:
ax1.set_xlim(to_datetime(formatting['xrange']))
if options['show']: plt.show()
return fig
def ts_dendrogram(self, **kwargs):
"""
Plots dendrogram of devices and channels in matplotlib plot. Takes all the channels
in channels that are in the test `devices`
Parameters
----------
devices: list or string
'all'
If 'all', uses all devices in the test
channels: list
'all'
If 'all', uses all channels in the devices
metric: string
'correlation' for normal R2 or custom metric by callable
'method': string
'single'
Method for dendrogram
'options': dict
Options including data processing prior to plot. Defaults in config._plot_def_opt
formatting: dict
Name of auxiliary electrode found in dataframe. Defaults in config._ts_plot_def_fmt
Returns
-------
Dendrogram matrix, shows plot
"""
if 'metric' not in kwargs: metric = 'correlation'
else: metric = kwargs['metric']
if 'method' not in kwargs: method = 'single'
else: method = kwargs['method']
if 'devices' not in kwargs: devices = list(self.devices.keys())
else: devices = kwargs['devices']
if 'channels' not in kwargs: channels = 'all'
else: channels = kwargs['channels']
if 'options' not in kwargs:
std_out('Using default options')
options = config._plot_def_opt
else:
options = dict_fmerge(config._plot_def_opt, kwargs['options'])
if 'formatting' not in kwargs:
std_out('Using default formatting')
formatting = config._ts_plot_def_fmt['mpl']
else:
formatting = dict_fmerge(config._ts_plot_def_fmt['mpl'],
kwargs['formatting'])
# Style
if formatting['style'] is not None: style.use(formatting['style'])
else: style.use(config._plot_style)
# Palette
if formatting['palette'] is not None: set_palette(formatting['palette'])
# Size sanity check
if formatting['width'] > 50:
std_out('Reducing width to 12')
formatting['width'] = 12
if formatting['height'] > 50:
std_out('Reducing height to 10')
formatting['height'] = 10
# Font size
if formatting['fontsize'] is not None:
rcParams.update({'font.size': formatting['fontsize']})
df = DataFrame()
for device in devices:
dfd = self.devices[device].readings.copy()
dfd = dfd.resample(options['frequency']).mean()
if channels != 'all':
for channel in channels:
if channel in dfd.columns:
df = df.append(dfd[channel].rename(device + '_' + channel))
else:
df = df.append(dfd)
df = clean(df, options['clean_na'], how='any')
# if options['clean_na'] is not None:
# if options['clean_na'] == 'drop': df.dropna(axis = 1, inplace=True)
# if options['clean_na'] == 'fill': df = df.fillna(method='ffill')
# Do the clustering
Z = hac.linkage(df, method=method, metric=metric)
# Plot dendogram
plt.figure(figsize=(formatting['width'], formatting['height']))
plt.title(formatting['title'], fontsize=formatting['titlefontsize'])
plt.subplots_adjust(top=formatting['suptitle_factor'])
plt.xlabel(formatting['xlabel'])
plt.ylabel(formatting['ylabel'])
hac.dendrogram(
Z,
orientation=formatting['orientation'],
leaf_font_size=formatting[
'fontsize'], # font size for the x axis labels
labels=df.index)
plt.show()
return Z
def heatmap_plot(self, **kwargs):
"""
Plots heatmap in seaborn plot, based on period binning
Parameters
----------
traces: dict
Data for the plot, with the format:
"traces": {"1": {"devices": '8019043',
"channel" : "PM_10"}
}
options: dict
Options including data processing prior to plot. Defaults in config._plot_def_opt
formatting: dict
Name of auxiliary electrode found in dataframe. Defaults in config._heatmap_def_fmt
Returns
-------
Matplotlib figure
"""
if config._framework == 'jupyterlab': plt.ioff();
plt.clf();
if 'traces' not in kwargs:
std_out('No traces defined', 'ERROR')
return None
else:
traces = kwargs['traces']
if 'options' not in kwargs:
std_out('Using default options')
options = config._plot_def_opt
else:
options = dict_fmerge(config._plot_def_opt, kwargs['options'])
if 'formatting' not in kwargs:
std_out('Using default formatting')
formatting = config._heatmap_def_fmt['mpl']
else:
formatting = dict_fmerge(config._heatmap_def_fmt['mpl'], kwargs['formatting'])
# Style
if formatting['style'] is not None: style.use(formatting['style'])
else: style.use(config._plot_style)
# Font size
if formatting['fontsize'] is not None:
rcParams.update({'font.size': formatting['fontsize']});
# Make it standard
for trace in traces:
if 'subplot' not in trace: traces[trace]['subplot'] = 1
# Get dataframe
df, subplots = prepare_data(self, traces, options)
n_subplots = len(subplots)
gskwags = {'frequency_hours': formatting['frequency_hours']}
dfgb, labels, yaxis, _ = groupby_session(df, **gskwags)
# Sample figsize in inches
_, ax = plt.subplots(figsize=(formatting['width'], formatting['height']));
xticks = [i.strftime("%Y-%m-%d") for i in dfgb.resample(formatting['session']).mean().index]
# Pivot with 'session'
g = heatmap(dfgb.pivot(columns='session').resample(formatting['session']).mean().T, ax = ax,
cmap = formatting['cmap'], robust = formatting['robust'],
vmin = formatting['vmin'], vmax = formatting['vmax'],
xticklabels = xticks, yticklabels = labels);
# ax.set_xticks(xticks*ax.get_xlim()[1]/(2))
_ = g.set_xlabel(formatting['xlabel']);
_ = g.set_ylabel(yaxis);
# Set title
_ = g.figure.suptitle(formatting['title'], fontsize=formatting['title_fontsize']);
plt.subplots_adjust(top=formatting['suptitle_factor'])
# Show
if options['show']: plt.show()
return g.figure
def scatter_plot(self, **kwargs):
"""
Plots correlation in matplotlib plot
Parameters
----------
traces: dict
Data for the plot, with the format:
traces = {1: {'devices': ['10751', '10751'],
'channels': ['TEMP', 'GB_2A'],
'subplot': 1},
2: {'devices': ['10752', '10752'],
'channels': ['TEMP', 'GB_2A'],
'subplot': 1}
3: {'devices': ['10751', '10751'],
'channels': ['TEMP', 'GB_2W'],
'subplot': 2}
4: {'devices': ['10752', '10752'],
'channels': ['TEMP', 'GB_2W'],
'subplot': 2}
}
options: dict
Options including data processing prior to plot. Defaults in config.plot_def_opt
formatting: dict
Formatting dict. Defaults in config.scatter_plot_def_fmt
Returns
-------
Matplotlib figure and axes
"""
if config.framework == 'jupyterlab': plt.ioff()
plt.clf()
if 'traces' not in kwargs:
std_out('No traces defined', 'ERROR')
return None
else:
traces = kwargs['traces']
if 'options' not in kwargs:
std_out('Using default options')
options = config.plot_def_opt
else:
options = dict_fmerge(config.plot_def_opt, kwargs['options'])
if 'formatting' not in kwargs:
std_out('Using default formatting')
formatting = config.scatter_plot_def_fmt['mpl']
else:
formatting = dict_fmerge(config.scatter_plot_def_fmt['mpl'],
kwargs['formatting'])
# Style
if formatting['style'] is not None: style.use(formatting['style'])
else: style.use(config.plot_style)
# Palette
if formatting['palette'] is not None: set_palette(formatting['palette'])
print(formatting['palette'])
# Font size
if formatting['fontsize'] is not None:
rcParams.update({'font.size': formatting['fontsize']})
# Make it standard
ptraces = dict()
for trace in traces:
if 'subplot' not in traces[trace]: traces[trace]['subplot'] = 1
if 'channels' not in traces[trace]:
ptraces = traces
continue
ptrace_1 = trace * 10 + 1
ptrace_2 = trace * 10 + 2
ptraces[ptrace_1] = {
'devices': traces[trace]['devices'][0],
'channel': traces[trace]['channels'][0],
'subplot': traces[trace]['subplot']
}
ptraces[ptrace_2] = {
'devices': traces[trace]['devices'][1],
'channel': traces[trace]['channels'][1],
'subplot': traces[trace]['subplot']
}
# Get dataframe
df, subplots = prepare_data(self, ptraces, options)
n_subplots = len(subplots)
# Plot
nrows = min(n_subplots, formatting['nrows'])
ncols = ceil(n_subplots / nrows)
figure, axes = plt.subplots(nrows,
ncols,
figsize=(formatting['width'],
formatting['height']))
if n_subplots == 1:
axes = array(axes)
axes.shape = (1)
cind = 0
y_axes = list()
x_axes = list()
for i in subplots:
for j in range(int(len(i) / 2)):
cind += 1
if cind > len(colors) - 1: cind = 0
if nrows > 1 and ncols > 1:
row = floor(subplots.index(i) / ncols)
col = subplots.index(i) - row * ncols
ax = axes[row][col]
else:
ax = axes[subplots.index(i)]
kwargs = {
'data': df,
'ax': ax,
'label': f'{i[2*j+1]} vs. {i[2*j]}'
}
if formatting['palette'] is None: kwargs['color'] = colors[cind]
regplot(df[i[2 * j]], df[i[2 * j + 1]], **kwargs)
if formatting['legend']:
ax.legend(loc='best')
if formatting['ylabel'] is not None:
try:
ax.set_ylabel(formatting['ylabel'][subplots.index(i) + 1])
except:
std_out(f'y_label for subplot {subplots.index(i)} not set',
'WARNING')
ax.set_ylabel('')
pass
else:
ax.set_ylabel('')
if formatting['xlabel'] is not None:
try:
ax.set_xlabel(formatting['xlabel'][subplots.index(i) + 1])
except:
std_out(f'x_label for subplot {subplots.index(i)} not set',
'WARNING')
ax.set_xlabel('')
pass
else:
ax.set_xlabel('')
y_axes.append(ax.get_ylim())
x_axes.append(ax.get_xlim())
# Unify axes or set what was ordered
for i in subplots:
for j in range(int(len(i) / 2)):
if nrows > 1 and ncols > 1:
row = floor(subplots.index(i) / ncols)
col = subplots.index(i) - row * ncols
ax = axes[row][col]
else:
ax = axes[subplots.index(i)]
# Set y axis limit
if formatting['yrange'] is not None and not formatting['sharey']:
try:
ax.set_ylim(formatting['yrange'][subplots.index(i) + 1])
except:
std_out(f'yrange for subplot {subplots.index(i)} not set',
'WARNING')
pass
elif formatting['sharey']:
ax.set_ylim(min([yl[0] for yl in y_axes]),
max([yl[1] for yl in y_axes]))
# Set x axis limit
if formatting['xrange'] is not None and not formatting['sharex']:
try:
ax.set_xlim(formatting['xrange'][subplots.index(i) + 1])
except:
std_out(f'xrange for subplot {subplots.index(i)} not set',
'WARNING')
pass
elif formatting['sharex']:
ax.set_xlim(min([xl[0] for xl in x_axes]),
max([xl[1] for xl in x_axes]))
# Set title
figure.suptitle(formatting['title'], fontsize=formatting['title_fontsize'])
plt.subplots_adjust(top=formatting['suptitle_factor'])
if options['show']: plt.show()
return figure
def box_plot(self, **kwargs):
"""
Plots heatmap in seaborn plot, based on period binning
Parameters
----------
traces: dict
Data for the plot, with the format:
"traces": {"1": {"devices": ['8019043', '8019044', '8019004'],
"channel" : "PM_10",
"subplot": 1,
"extras": ['max', 'min', 'avg']},
"2": {"devices": "all",
"channel" : "TEMP",
"subplot": 2}
}
options: dict
Options including data processing prior to plot. Defaults in config._plot_def_opt
formatting: dict
Name of auxiliary electrode found in dataframe. Defaults in config._boxplot_def_fmt
Returns
-------
Matplotlib figure
"""
if config._framework == 'jupyterlab': plt.ioff()
plt.clf()
if 'traces' not in kwargs:
std_out('No traces defined', 'ERROR')
return None
else:
traces = kwargs['traces']
if 'options' not in kwargs:
std_out('Using default options')
options = config._plot_def_opt
else:
options = dict_fmerge(config._plot_def_opt, kwargs['options'])
if 'formatting' not in kwargs:
std_out('Using default formatting')
formatting = config._boxplot_def_fmt['mpl']
else:
formatting = dict_fmerge(config._boxplot_def_fmt['mpl'],
kwargs['formatting'])
# Style
if formatting['style'] is not None: style.use(formatting['style'])
else: style.use(config._plot_style)
# Make it standard
for trace in traces:
if 'subplot' not in trace: traces[trace]['subplot'] = 1
# Palette
if formatting['palette'] is not None: set_palette(formatting['palette'])
# Font size
if formatting['fontsize'] is not None:
rcParams.update({'font.size': formatting['fontsize']})
# Get dataframe
df, subplots = prepare_data(self, traces, options)
n_subplots = len(subplots)
dfgb, labels, xaxis, channel = groupby_session(
df,
frequency_hours=formatting['frequency_hours'],
periods=formatting['periods'])
# Sample figsize in inches
_, ax = plt.subplots(figsize=(formatting['width'], formatting['height']))
# Pivot with 'session'
if formatting['periods'] is not None:
g = boxplot(x=dfgb['session'], y=dfgb[channel], hue = dfgb['period'], \
ax=ax, palette = formatting['cmap'])
else:
g = boxplot(x=dfgb['session'],
y=dfgb[channel],
ax=ax,
palette=formatting['cmap'])
# TODO make this to compare to not None, so that we can send location
if formatting['ylabel'] is not None:
_ = g.set_ylabel(formatting['ylabel'])
if formatting['grid'] is not None:
_ = g.grid(formatting['grid'])
if formatting['yrange'] is not None:
ax.set_ylim(formatting['yrange'])
_ = g.set_xlabel(xaxis)
# Set title
if formatting['title'] is not None:
_ = g.figure.suptitle(formatting['title'],
fontsize=formatting['title_fontsize'])
# Suptitle factor
if formatting['suptitle_factor'] is not None:
plt.subplots_adjust(top=formatting['suptitle_factor'])
# Show
if options['show']: plt.show()
return g.figure
def ts_iplot(self, **kwargs):
"""
Plots timeseries in plotly interactive plot
Parameters
----------
traces: dict
Data for the plot, with the format:
"traces": {"1": {"devices": ['8019043', '8019044', '8019004'],
"channel" : "PM_10",
"subplot": 1,
"extras": ['max', 'min', 'avg']},
"2": {"devices": "all",
"channel" : "TEMP",
"subplot": 2}
}
options: dict
Options including data processing prior to plot. Defaults in config._plot_def_opt
formatting: dict
Name of auxiliary electrode found in dataframe. Defaults in config._ts_plot_def_fmt
Returns
-------
Plotly figure
"""
if config._framework == 'jupyterlab': renderers.default = config._framework
if 'traces' not in kwargs:
std_out('No traces defined', 'ERROR')
return None
else:
traces = kwargs['traces']
if 'options' not in kwargs:
std_out('Using default options', 'WARNING')
options = config._plot_def_opt
else:
options = dict_fmerge(config._plot_def_opt, kwargs['options'])
if 'formatting' not in kwargs:
std_out('Using default formatting', 'WARNING')
formatting = config._ts_plot_def_fmt['plotly']
else:
formatting = dict_fmerge(config._ts_plot_def_fmt['plotly'], kwargs['formatting'])
# Get dataframe
df, subplots = prepare_data(self, traces, options)
n_subplots = len(subplots)
# Size sanity check
if formatting['width'] < 100:
std_out('Setting width to 800')
formatting['width'] = 800
if formatting['height'] < 100:
std_out('Reducing height to 600')
formatting['height'] = 600
figure = make_subplots(rows = n_subplots, cols=1,
shared_xaxes = formatting['sharex'])
# Add traces
for isbplt in range(n_subplots):
for trace in subplots[isbplt]:
figure.append_trace({'x': df.index,
'y': df[trace],
'type': 'scatter',
'mode': 'lines+markers',
'name': trace},
isbplt + 1, 1)
# Name the axis
if formatting['ylabel'] is not None:
figure['layout']['yaxis' + str(isbplt+1)]['title']['text'] = formatting['ylabel'][isbplt+1]
if formatting['yrange'] is not None:
figure['layout']['yaxis' + str(isbplt+1)]['range'] = formatting['yrange'][isbplt+1]
# Add axis labels
if formatting['xlabel'] is not None:
figure['layout']['xaxis' + str(n_subplots)]['title']['text'] = formatting['xlabel']
# Add layout
figure['layout'].update(height = formatting['height'],
legend = dict(x=0.2, y=-0.3,
traceorder='normal',
font = dict(family='sans-serif',
size=10,
color='#000'),
xanchor = 'center',
orientation = 'h',
itemsizing = 'trace',
yanchor = 'bottom',
bgcolor ='rgba(0,0,0,0)',
bordercolor = 'rgba(0,0,0,0)',
borderwidth = 0),
title=dict(text=formatting['title'])
)
if options['show']: figure.show()
return figure
def ts_uplot(self, **kwargs):
"""
Plots timeseries in uplot interactive plot - Fast, fast fast
Parameters
----------
traces: dict
Data for the plot, with the format:
"traces": {"1": {"devices": ['8019043', '8019044', '8019004'],
"channel" : "PM_10",
"subplot": 1,
"extras": ['max', 'min', 'avg']},
"2": {"devices": "all",
"channel" : "TEMP",
"subplot": 2}
}
options: dict
Options including data processing prior to plot. Defaults in config._plot_def_opt
formatting: dict
Name of auxiliary electrode found in dataframe. Defaults in config._ts_plot_def_fmt
Returns
-------
uPlot figure
"""
head_template = '''
<link rel="stylesheet" href="https://leeoniya.github.io/uPlot/dist/uPlot.min.css">
<script src="https://leeoniya.github.io/uPlot/dist/uPlot.iife.js"></script>
<div style="text-align:center">
<h2 style="font-family: Roboto"> {{title}} </h2>
</div>
'''
uplot_template = '''
<div id="plot{{subplot}}"></div>
<script>
data = {{data}};
options = {{options}};
if (typeof options.scatter == 'undefined') {
options.scatter = false
}
if (options.scatter) {
for (i=1; i<data.length; i++) {
options['series'][i]["paths"] = u => null;
}
}
u = new uPlot(options, data, document.getElementById("plot{{subplot}}"))
</script>
'''
if 'traces' not in kwargs:
std_out('No traces defined', 'ERROR')
return None
else:
traces = kwargs['traces']
if 'options' not in kwargs:
std_out('Using default options', 'WARNING')
options = config._plot_def_opt
else:
options = dict_fmerge(config._plot_def_opt, kwargs['options'])
if 'formatting' not in kwargs:
std_out('Using default formatting', 'WARNING')
formatting = config._ts_plot_def_fmt['uplot']
else:
formatting = dict_fmerge(config._ts_plot_def_fmt['uplot'],
kwargs['formatting'])
# Size sanity check
if formatting['width'] < 100:
std_out('Setting width to 800')
formatting['width'] = 800
if formatting['height'] < 100:
std_out('Reducing height to 600')
formatting['height'] = 600
if 'html' not in options:
options['html'] = False
h = Template(head_template).render(title=formatting['title'])
# Get dataframe
df, subplots = prepare_data(self, traces, options)
df = df.fillna('null')
n_subplots = len(subplots)
# Get data in uplot expected format
udf = df.copy()
udf.index = udf.index.astype(int) / 10**9
for isbplt in range(n_subplots):
sdf = udf.loc[:, subplots[isbplt]]
sdf = sdf.reset_index()
data = sdf.values.T.tolist()
labels = sdf.columns
useries = [{'label': labels[0]}]
if formatting['ylabel'] is None:
ylabel = None
else:
ylabel = formatting['ylabel'][isbplt + 1]
uaxes = [{
'label': formatting['xlabel'],
'labelSize': formatting['fontsize'],
}, {
'label': ylabel,
'labelSize': formatting['fontsize']
}]
color_idx = 0
for label in labels:
if label == labels[0]: continue
if color_idx + 1 > len(colors): color_idx = 0
nser = {
'label': label,
'stroke': colors[color_idx],
'points': {
'space': 0,
'size': formatting['size']
}
}
useries.append(nser)
color_idx += 1
u_options = {
'width': formatting['width'],
'height': formatting['height'],
'cursor': {
'lock': True,
'focus': {
'prox': 16,
},
'sync': {
'key': 'moo',
'setSeries': True,
},
'drag': {
'x': True,
'y': True,
'uni': 50,
'dist': 10,
}
},
'scales': {
'x': {
'time': True
},
'y': {
'auto': True
},
},
'series': useries,
'axes': uaxes
}
h2 = Template(uplot_template).render(data=json.dumps(data),
options=json.dumps(u_options),
subplot=isbplt)
h += h2
h = h.replace('"', "'")
h = h.replace("'null'", "null")
if options['html']:
return h
else:
iframe = f'''<iframe srcdoc="{h}" src=""
frameborder="0" width={formatting['width'] + formatting['padding-right']}
height={formatting['height'] + formatting['padding-bottom']}
sandbox="allow-scripts">
</iframe>'''
return HTML(iframe)
def device_metric_map(self, channel, start_date, end_date, options=dict()):
'''
Creates a folium map showing the evolution of a metric dynamically with colors
Parameters
-------
channel: String
The channel to make the map from
start_date, end_date: String
Date convertible string
options: dict()
Possible keys are (default otherwise)
location: list
[41.400818, 2.1825157]
Center map location
tiles: (String)
'Stamen Toner'
Tiles for the folium.Map
zoom: (float)
2.5
Zoom to start with in folium.Map
period: 'String'
'1W'
Period for 'dynamic' map
radius: float
10
Circle radius for icon
fillOpacity: float
1
(<1) Fill opacity for the icon
stroke: 'String'
'false'
'true' or 'false'. For icon's stroke
icon: 'String'
'circle'
A valid folium.Map icon style
Returns
-------
Folium.Map object
'''
# Set defaults
options = dict_fmerge(config._map_def_opt, options)
# Make date range
date_r = date_range(start=start_date,
end=end_date,
normalize=True,
freq=options['period']).strftime('%Y-%m-%d')
date_l = list()
for item in date_r.values:
date_l.append(str(item))
# Get bins
for bname in config._channel_bins.keys():
if bname in channel:
bins = config._channel_bins[bname]
break
# Make features
features = []
for device in self.devices:
# Get lat, long
try:
self.devices[str(device)].api_device.get_device_lat_long()
_lat = self.devices[str(device)].api_device.lat
_long = self.devices[str(device)].api_device.long
except AttributeError:
std_out(f'Cannot retrieve [lat, long] from device {device}',
'WARNING')
pass
continue
if _lat is None or _long is None: continue
# Resample
try:
dfc = self.devices[str(device)].readings.resample(
options['period']).mean()
except:
pass
continue
if channel not in dfc.columns: continue
# Make color column
dfc['color'] = cut(dfc[channel],
bins,
labels=config._map_colors_palette)
# Add point for each date
for date in date_l:
if date not in dfc.index: continue
if date_l.index(date) > len(date_l) - 2: continue
features.append({
'type': 'Feature',
'geometry': {
'type': 'LineString',
'coordinates': [[str(_long), str(_lat)]] * 2,
'popup': str(device),
},
'properties': {
'times': [date, date_l[date_l.index(date) + 1]],
'icon': options['icon'],
'iconstyle': {
'fillColor': str(dfc.loc[date, 'color']),
'fillOpacity': options['fillOpacity'],
'stroke': options['stroke'],
'radius': options['radius']
},
'style': {
'weight': '0'
},
'id': 'man'
}
})
# Make map
m = Map(
location=options['location'],
tiles=options['tiles'],
zoom_start=options['zoom'],
)
TimestampedGeoJson(
{
'type': 'FeatureCollection',
'features': features
},
period='P' + convert_rollup(options['period']),
add_last_point=True,
auto_play=False,
loop=False,
max_speed=5,
loop_button=True,
# date_options='YYYY/MM/DD',
time_slider_drag_update=True,
duration='P' + options['period']).add_to(m)
return m
def ts_plot(self, **kwargs):
"""
Plots timeseries in matplotlib plot
Parameters
----------
traces: dict
Data for the plot, with the format:
"traces": {"1": {"devices": ['8019043', '8019044', '8019004'],
"channel" : "PM_10",
"subplot": 1,
"extras": ['max', 'min', 'avg']},
"2": {"devices": "all",
"channel" : "TEMP",
"subplot": 2}
}
options: dict
Options including data processing prior to plot. Defaults in config._plot_def_opt
formatting: dict
Formatting dict. Defaults in config._ts_plot_def_fmt
Returns
-------
Matplotlib figure
"""
if config.framework == 'jupyterlab': plt.ioff();
plt.clf();
if 'traces' not in kwargs:
std_out('No traces defined', 'ERROR')
return None
else:
traces = kwargs['traces']
if 'options' not in kwargs:
std_out('Using default options')
options = config._plot_def_opt
else:
options = dict_fmerge(config._plot_def_opt, kwargs['options'])
if 'formatting' not in kwargs:
std_out('Using default formatting')
formatting = config._ts_plot_def_fmt['mpl']
else:
formatting = dict_fmerge(config._ts_plot_def_fmt['mpl'], kwargs['formatting'])
# Style
if formatting['style'] is not None: style.use(formatting['style'])
else: style.use(config._plot_style)
# Palette
if formatting['palette'] is not None: set_palette(formatting['palette'])
# Font size
if formatting['fontsize'] is not None:
rcParams.update({'font.size': formatting['fontsize']});
# Get dataframe
df, subplots = prepare_data(self, traces, options)
n_subplots = len(subplots)
# Size sanity check
if formatting['width'] > 50:
std_out('Reducing width to 12')
formatting['width'] = 12
if formatting['height'] > 50:
std_out('Reducing height to 10')
formatting['height'] = 10
# Plot
figure, axes = plt.subplots(n_subplots, 1,
sharex = formatting['sharex'],
figsize = (formatting['width'],
formatting['height'])
);
if n_subplots == 1:
axes = array(axes)
axes.shape = (1)
for ax in axes:
isbplt = where(axes == ax)[0][0];
# Check if we are plotting any highlight for the trace
if any(['-MEAN' in trace for trace in subplots[isbplt]]): has_hl = True
elif any(['-MAX' in trace for trace in subplots[isbplt]]): has_hl = True
elif any(['-MIN' in trace for trace in subplots[isbplt]]): has_hl = True
else: has_hl = False
for trace in subplots[isbplt]:
if has_hl:
if '-MEAN' in trace: alpha = formatting['alpha_highlight']
elif '-MAX' in trace: alpha = formatting['alpha_highlight']
elif '-MIN' in trace: alpha = formatting['alpha_highlight']
else: alpha = formatting['alpha_other']
else: alpha = 1
ax.plot(df.index, df[trace], label = trace, alpha = alpha);
# TODO make this to compare to not None, so that we can send location
if formatting['legend']:
ax.legend(loc='center left', bbox_to_anchor=(1, 0.5));
if formatting['ylabel'] is not None:
ax.set_ylabel(formatting['ylabel'][isbplt+1]);
if formatting['xlabel'] is not None:
ax.set_xlabel(formatting['xlabel']);
if formatting['yrange'] is not None:
ax.set_ylim(formatting['yrange'][isbplt+1]);
if formatting['xrange'] is not None:
if formatting['sharex']: ax.set_xlim(to_datetime(formatting['xrange'][1]));
else: ax.set_xlim(to_datetime(formatting['xrange'][isbplt+1]));
if formatting['grid'] is not None:
ax.grid(formatting['grid']);
if formatting["decorators"] is not None:
if 'axvline' in formatting['decorators']:
for vline in formatting['decorators']['axvline']:
ax.axvline(to_datetime(vline), linestyle = 'dotted', color = 'gray');
if 'axhline' in formatting['decorators']:
for vline in formatting['decorators']['axhline']:
ax.axhline(vline, linestyle = 'dotted', color = 'gray');
if 'xtext' in formatting['decorators']:
for xtext in formatting['decorators']['xtext'].keys():
text = formatting['decorators']['xtext'][xtext]
position = formatting['yrange'][isbplt+1][1]-(formatting['yrange'][isbplt+1][1]-formatting['yrange'][isbplt+1][0])/10
ax.text(to_datetime(xtext), position, text, size=15, color = 'gray');
# TODO Fix
if 'ytext' in formatting['decorators']:
for ytext in formatting['decorators']['ytext'].keys():
text = formatting['decorators']['ytext'][ytext]
position = formatting['xrange'][isbplt+1][1]-(formatting['xrange'][isbplt+1][1]-formatting['yrange'][isbplt+1][0])/10
ax.text(ytext, position, text, size=15, color = 'gray');
figure.suptitle(formatting['title'], fontsize=formatting['title_fontsize']);
plt.subplots_adjust(top = formatting['suptitle_factor']);
if options['show']: plt.show();
return figure
def heatmap_iplot(self, **kwargs):
"""
Plots heatmap in plotly interactive plot
Parameters
----------
traces: dict
Data for the plot, with the format:
"traces": {"1": {"devices": '8019043',
"channel" : "PM_10"}
}
options: dict
Options including data processing prior to plot. Defaults in config.plot_def_opt
formatting: dict
Name of auxiliary electrode found in dataframe. Defaults in config.heatmap_def_fmt
Returns
-------
Plotly figure
"""
if config.framework == 'jupyterlab': renderers.default = config.framework
if 'traces' not in kwargs:
std_out('No traces defined', 'ERROR')
return None
else:
traces = kwargs['traces']
if 'options' not in kwargs:
std_out('Using default options')
options = config.plot_def_opt
else:
options = dict_fmerge(config.plot_def_opt, kwargs['options'])
if 'formatting' not in kwargs:
std_out('Using default formatting')
formatting = config.heatmap_def_fmt['plotly']
else:
formatting = dict_fmerge(config.heatmap_def_fmt['plotly'],
kwargs['formatting'])
# Make it standard
for trace in traces:
if 'subplot' not in trace: traces[trace]['subplot'] = 1
# Get dataframe
df, subplots = prepare_data(self, traces, options)
n_subplots = len(subplots)
gskwags = {'frequency_hours': formatting['frequency_hours']}
dfgb, labels, yaxis, channel = groupby_session(df, **gskwags)
xticks = [
i.strftime("%Y-%m-%d")
for i in dfgb.resample(formatting['session']).mean().index
]
# Data
data = [
Heatmap(
z=dfgb[channel],
x=dfgb.index.date,
y=dfgb['session'],
# colorscale=colorscale
)
]
layout = Layout(title=formatting['title'],
xaxis=dict(ticks=''),
yaxis=dict(ticks='',
categoryarray=labels,
autorange='reversed'))
figure = Figure(data=data, layout=layout)
if options['show']: iplot(figure)
return figure
def path_plot(self,
channel=None,
map_type='dynamic',
devices='all',
start_date=None,
end_date=None,
options=dict()):
'''
Creates a folium map showing a path
Parameters
-------
channel: String
None
If None, shows path, otherwise, colored path with channel mapping
map_type: String
'dynamic'
'dynamic' or 'static'. Whether is a dinamic map or not
devices: list or 'all'
List of devices to include, or 'all' from self.devices
channel: String
The channel to make the map from
start_date, end_date: String
Date convertible string
options: dict()
Possible keys are (default otherwise)
location: list
[41.400818, 2.1825157]
Center map location
tiles: (String)
'Stamen Toner'
Tiles for the folium.Map
zoom: (float)
2.5
Zoom to start with in folium.Map
period: 'String'
'1W'
Period for 'dynamic' map
radius: float
10
Circle radius for icon
fillOpacity: float
1
(<1) Fill opacity for the icon
stroke: 'String'
'false'
'true' or 'false'. For icon's stroke
icon: 'String'
'circle'
A valid folium.Map icon style
Returns
-------
Folium.Map object
'''
# Set defaults
options = dict_fmerge(config._map_def_opt, options)
# Make features
features = []
if devices == 'all':
mdev = self.devices
else:
mdev = list()
for device in devices:
if device in self.devices: mdev.append(device)
else: std_out(f'Device {device} not found, ignoring', 'WARNING')
if len(mdev) == 0:
std_out('Requested devices not in test', 'ERROR')
return None
for device in mdev:
chs = ['GPS_LAT', 'GPS_LONG']
if channel is not None:
if channel not in self.devices[str(device)].readings.columns:
std_out(
f'Channel {channel} not in columns: {self.devices[str(device)].readings.columns}',
'ERROR')
return None
# Get bins
minmax = False
if not options['minmax']:
if all([key not in channel for key in config._channel_bins]):
std_out(
f'Requested channel {channel} not in config mapped bins {config._channel_bins.keys()}.Using min/max mapping',
'WARNING')
minmax = True
else:
minmax = True
if minmax:
bins = linspace(
self.devices[str(device)].readings[channel].min(),
self.devices[str(device)].readings[channel].max(),
config._channel_bin_n)
else:
for bname in config._channel_bins.keys():
if bname in channel:
bins = config._channel_bins[bname]
break
chs.append(channel)
# Create copy
dfc = self.devices[str(device)].readings[chs].copy()
# Resample and cleanup
# TODO THIS CAN INPUT SOME MADE UP READINGS
dfc = clean(dfc.resample(options['period']).mean(), 'fill')
# Make color column
legend_labels = None
if channel is not None:
dfc['COLOR'] = cut(dfc[channel], bins, labels =\
config._map_colors_palette)
# Make legend labels
legend_labels = {}
for ibin in range(len(bins) - 1):
legend_labels[f'{round(bins[ibin],2)} : {round(bins[ibin+1],2)}'] =\
config._map_colors_palette[ibin]
else:
dfc['COLOR'] = config._map_colors_palette[0]
if start_date is not None:
dfc = dfc[dfc.index > start_date]
if end_date is not None:
dfc = dfc[dfc.index < end_date]
# Add point for each date
for date in dfc.index:
if date == dfc.index[-1]: break
times = []
color = str(dfc.loc[date, 'COLOR'])
if color == 'nan' or isnan(dfc.loc[date, 'GPS_LONG'])\
or isnan(dfc.loc[date, 'GPS_LAT']):
std_out(f'Skipping point {date}', 'WARNING')
continue
geometry = {
'type':
'LineString',
'coordinates':
[[dfc.loc[date, 'GPS_LONG'], dfc.loc[date, 'GPS_LAT']],
[
dfc.loc[date + dfc.index.freq, 'GPS_LONG'],
dfc.loc[date + dfc.index.freq, 'GPS_LAT']
]],
}
properties = {
'icon':
options['icon'],
'iconstyle': {
'fillColor': color,
'fillOpacity': options['fillOpacity'],
'stroke': options['stroke'],
'radius': options['radius']
},
'device':
device,
'timestamp':
date.strftime('%Y-%m-%dT%H:%M:%S'),
"coordinates": [
dfc.loc[date + dfc.index.freq, 'GPS_LAT'],
dfc.loc[date + dfc.index.freq, 'GPS_LONG']
],
'style': {
'color': color,
'stroke-width': options['stroke-width'],
'fillOpacity': options['fillOpacity']
}
}
# Add reading to tooltip
if channel is not None:
properties['channel'] = channel
properties['value'] = dfc.loc[date, channel]
if map_type == 'dynamic':
properties['times'] = [
date.strftime('%Y-%m-%dT%H:%M:%S'),
(date + dfc.index.freq).strftime('%Y-%m-%dT%H:%M:%S')
]
features.append({
'type': 'Feature',
'geometry': geometry,
'properties': properties
})
featurecol = {'type': 'FeatureCollection', 'features': features}
# Make map
if options['location'] == 'average':
avg_long = dfc['GPS_LONG'].mean()
avg_lat = dfc['GPS_LAT'].mean()
loc = [avg_lat, avg_long]
else:
loc = options['location']
m = Map(
location=loc,
tiles=options['tiles'],
zoom_start=options['zoom'],
)
if map_type == 'static':
# TODO WORKAROUND UNTIL GEOJSON ACCEPTS MARKERS
if options['markers']:
for feature in features:
Circle(location=[
feature['geometry']['coordinates'][0][1],
feature['geometry']['coordinates'][0][0]
],
fill='true',
radius=feature['properties']['iconstyle']['radius'],
color=feature['properties']['iconstyle']['fillColor'],
fill_opacity=feature['properties']['iconstyle']
['fillOpacity']).add_to(m)
if channel is not None:
fields = ["device", "channel", "timestamp", "coordinates", "value"]
aliases = [
"Device:", "Sensor:", "Timestamp:", "Coordinates:", "Reading:"
]
else:
fields = ["device", "timestamp", "coordinates"]
aliases = ["Device:", "Timestamp:", "Coordinates:"]
popup = GeoJsonPopup(
fields=fields,
aliases=aliases,
localize=True,
labels=True,
max_width=800,
)
tooltip = GeoJsonTooltip(
fields=fields,
aliases=aliases,
localize=True,
sticky=True,
labels=True,
style="""
background-color: #F0EFEF;
border: 1px solid gray;
border-radius: 1px;
box-shadow: 2px;
""",
max_width=800,
)
GeoJson(
featurecol,
tooltip=tooltip,
popup=popup,
style_function=lambda x: {
'color': x['properties']['style']['color'],
'weight': x['properties']['style']['stroke-width'],
'fillOpacity': x['properties']['style']['fillOpacity']
},
).add_to(m)
elif map_type == 'dynamic':
TimestampedGeoJson(featurecol,
period='PT' + convert_rollup(options['period']),
add_last_point=True,
auto_play=False,
loop=False,
max_speed=options['max_speed'],
loop_button=True,
time_slider_drag_update=True).add_to(m)
else:
std_out(f'Not supported map type {map_type}', 'ERROR')
return None
if options['minimap']:
minimap = MiniMap(toggle_display=True, tile_layer=options['tiles'])
minimap.add_to(m)
if options['legend'] and not legend_labels is None:
templateLoader = FileSystemLoader(searchpath=join(dirname(__file__),\
'templates'))
templateEnv = Environment(loader=templateLoader)
template = templateEnv.get_template("map_legend.html")
filled_map_legend = template.render(legend_labels=legend_labels)
map_legend_html = '{% macro html(this, kwargs) %}'+\
filled_map_legend+\
'{% endmacro %}'
legend = element.MacroElement()
legend._template = element.Template(map_legend_html)
m.get_root().add_child(legend)
return m
def scatter_iplot(self, **kwargs):
"""
Plots Correlation in plotly plot. Calls corr_plot and then converts it
Parameters
----------
traces: dict
Data for the plot, with the format:
traces = {
"1": {"devices": "10751",
"channel": "EXT_PM_A_1"},
"2": {"devices": "10751",
"channel": "EXT_PM_A_10"
}
}
options: dict
Options including data processing prior to plot. Defaults in config.plot_def_opt
formatting: dict
Name of auxiliary electrode found in dataframe. Defaults in config.corr_plot_def_fmt
Returns
-------
Plotly figure
"""
std_out('Not yet working', 'ERROR')
return None
if config.framework == 'jupyterlab': renderers.default = config.framework
if 'traces' not in kwargs:
std_out('No traces defined', 'ERROR')
return None
else:
traces = kwargs['traces']
if 'options' not in kwargs:
std_out('Using default options')
options = config.plot_def_opt
else:
options = dict_fmerge(config.plot_def_opt, kwargs['options'])
if 'formatting' not in kwargs:
std_out('Using default formatting')
formatting = config.scatter_plot_def_fmt['plotly']
else:
formatting = dict_fmerge(config.scatter_plot_def_fmt['plotly'],
kwargs['formatting'])
# Set options to not show in scatter_plot
toshow = options['show']
options['show'] = False
# Make sns plot
mfig = scatter_plot(self,
traces=traces,
options=options,
formatting=formatting)
options['show'] = toshow
pfig = tls.mpl_to_plotly(mfig)
if options['show']: pfig.show()
return pfig
def __init__(self, blueprint='sck_21', descriptor={}):
'''
Creates an instance of device. Devices are objects that contain sensors readings, metrics
(calculations based on sensors readings), and metadata such as units, dates, frequency and source
Parameters:
-----------
blueprint: String
Default: 'sck_21'
Defines the type of device. For instance: sck_21, sck_20, csic_station, muv_station
parrot_soil, sc_20_station, sc_21_station. A list of all the blueprints is found in
blueprints.yaml and accessible via the scdata.utils.load_blueprints function.
The blueprint can also be defined from the postprocessing info in SCAPI. The manual
parameter passed here is prioritary to that of the API
descriptor: dict()
Default: empty dict
A dictionary containing information about the device itself. Depending on the blueprint, this descriptor
needs to have different data. If not all the data is present, the corresponding blueprint's default will
be used
Examples:
----------
Device('sck_21', descriptor = {'source': 'api', 'id': '1919'})
device with sck_21 blueprint with 1919 ID
Device(descriptor = {'source': 'api', 'id': '1919'})
device with sck_21 blueprint with 1919 ID
Returns
----------
Device object
'''
if blueprint is not None:
self.blueprint = blueprint
# Set attributes
for bpitem in config.blueprints[blueprint]:
self.__setattr__(bpitem, config.blueprints[blueprint][bpitem])
for ditem in descriptor.keys():
if type(self.__getattribute__(ditem)) == dict:
self.__setattr__(
ditem,
dict_fmerge(self.__getattribute__(ditem),
descriptor[ditem]))
else:
self.__setattr__(ditem, descriptor[ditem])
# Add API handler if needed
if self.source == 'api':
hmod = __import__('scdata.io.device_api',
fromlist=['io.device_api'])
Hclass = getattr(hmod, self.sources[self.source]['handler'])
# Create object
self.api_device = Hclass(did=self.id)
std_out(f'Checking postprocessing info from API device')
self.load_postprocessing_info()
if self.blueprint is None:
std_out('Need a blueprint to proceed', 'ERROR')
return None
else:
std_out(f'Device {self.id} is using {blueprint}')
self.readings = DataFrame()
self.loaded = False
self.options = dict()
self.hw_id = None
self.latest_postprocessing = None
def device_history_map(map_type='dynamic', dataframe=None, options=dict()):
'''
Creates a folium map with either location of devices or their "existence period"
-------
Parameters:
map_type: String
'dynamic'
'dynamic' or 'static'. Whether is a dinamic map or not
dataframe: Pandas Dataframe
None
Contains information about when the devices started posting data, ids, location.
It follows the format of world_map in api device
options: dict
dict()
Returns:
Folium.Map object
'''
def coordinates(x):
return [x['latitude'], x['longitude']]
def color(x):
iSCAPE_IDs = [19, 20, 21, 28]
making_sense_IDs = [11, 14]
SCK_21_IDs = [26]
color = '#0019ff'
try:
if x['kit_id'] in iSCAPE_IDs: color = '#7dbd4c'
elif x['kit_id'] in making_sense_IDs: color = '#f88027'
elif x['kit_id'] in SCK_21_IDs: color = '#ffb500'
except:
print_exc()
pass
return color
def validate(x):
if x['last_reading_at'] is None: return False
if x['added_at'] is None: return False
if any(x['coordinates']) is None or any(
[isnan(item) for item in x['coordinates']]):
return False
if map_type == 'dynamic':
if x['date_list'] == []: return False
return True
def range_list(x):
date_r = date_range(start=x['added_at'],
end=x['last_reading_at'],
normalize=True,
freq=options['period']).strftime('%Y-%m-%d')
date_l = list()
for item in date_r.values:
date_l.append(str(item))
return date_l
dataframe['color'] = dataframe.apply(lambda x: color(x), axis=1)
dataframe['coordinates'] = dataframe.apply(lambda x: coordinates(x),
axis=1)
options = dict_fmerge(config._map_def_opt, options)
# Make map
m = Map(
location=options['location'],
tiles=options['tiles'],
zoom_start=options['zoom'],
)
if map_type == 'dynamic':
dataframe['date_list'] = dataframe.apply(lambda x: range_list(x),
axis=1)
dataframe['valid'] = dataframe.apply(lambda x: validate(x), axis=1)
dataframe = dataframe[(dataframe['valid'] == True)]
features = list()
for sensor in dataframe.index:
features.append({
'type': 'Feature',
'geometry': {
'type':
'LineString',
'coordinates':
[dataframe.loc[sensor, 'coordinates'][::-1]] *
len(dataframe.loc[sensor, 'date_list']),
'popup':
f'<a href="http://smartcitizen.me/kits/{sensor}">{sensor}</a>',
},
'properties': {
'times': dataframe.loc[sensor, 'date_list'],
'icon': options['icon'],
'iconstyle': {
'fillOpacity': options['fillOpacity'],
'fillColor': dataframe.loc[sensor, 'color'],
'stroke': options['stroke'],
'radius': options['radius']
},
'style': {
'weight': '0'
},
'id': 'man'
}
})
TimestampedGeoJson(
{
'type': 'FeatureCollection',
'features': features
},
period='P' + convert_rollup(options['period']),
add_last_point=True,
auto_play=False,
loop=False,
max_speed=options['max_speed'],
loop_button=True,
# date_options='YYYY/MM/DD',
time_slider_drag_update=True,
duration='P' + options['period']).add_to(m)
elif map_type == 'static':
dataframe['valid'] = dataframe.apply(lambda x: validate(x), axis=1)
dataframe = dataframe[(dataframe['valid'] == True)]
for sensor in dataframe.index:
Circle(
location=dataframe.loc[sensor, 'coordinates'],
radius=options['radius'],
color=dataframe.loc[sensor, 'color'],
fill=True,
fillOpacity=options['fillOpacity'],
fillColor=dataframe.loc[sensor, 'color'],
popup=
f'<a href="http://smartcitizen.me/kits/{sensor}">{sensor}</a>'
).add_to(m)
return m
def ts_dispersion_uplot(self, **kwargs):
'''
Plots dispersion timeseries in uplot plot
Parameters
----------
channel: string
Channel
options: dict
Options including data processing prior to plot. Defaults in config._plot_def_opt
formatting: dict
Formatting dict. Defaults in config._ts_plot_def_fmt
Returns
-------
Matplotlib figure
'''
head_template = '''
<link rel="stylesheet" href="https://leeoniya.github.io/uPlot/dist/uPlot.min.css">
<script src="https://leeoniya.github.io/uPlot/dist/uPlot.iife.js"></script>
<div style="text-align:center">
<h2 style="font-family: Roboto"> {{title}} </h2>
</div>
'''
uplot_template = '''
<div id="plot{{subplot}}"></div>
<script>
data = {{data}};
options = {{options}};
if (typeof options.scatter == 'undefined') {
options.scatter = false
}
if (options.scatter) {
for (i=1; i<data.length; i++) {
options['series'][i]["paths"] = u => null;
}
}
u = new uPlot(options, data, document.getElementById("plot{{subplot}}"))
</script>
'''
if 'channel' not in kwargs:
std_out('Needs at least one channel to plot')
return None
else:
channel = kwargs['channel']
if 'options' not in kwargs:
std_out('Using default options')
options = config._plot_def_opt
else:
options = dict_fmerge(config._plot_def_opt, kwargs['options'])
if 'formatting' not in kwargs:
std_out('Using default formatting')
formatting = config._ts_plot_def_fmt['uplot']
else:
formatting = dict_fmerge(config._ts_plot_def_fmt['uplot'],
kwargs['formatting'])
# Size sanity check
if formatting['width'] < 100:
std_out('Setting width to 800')
formatting['width'] = 800
if formatting['height'] < 100:
std_out('Reducing height to 600')
formatting['height'] = 600
if 'html' not in options:
options['html'] = False
if self.dispersion_df is None:
std_out('Perform dispersion analysis first!', 'ERROR')
return None
if self.common_channels == []: self.get_common_channels()
if channel not in self.common_channels:
std_out(f'Channel {channel} not in common_channels')
return None
if channel in config._dispersion['ignore_channels']:
std_out(f'Channel {channel} ignored per config')
return None
if len(self.devices) > config._dispersion['nt_threshold']:
distribution = 'normal'
std_out('Using normal distribution')
std_out(f"Using limit for sigma confidence:\
{config._dispersion['limit_confidence_sigma']}")
else:
distribution = 't-student'
std_out(f'Using t-student distribution.')
ch_index = self.common_channels.index(channel) + 1
total_number = len(self.common_channels)
h = Template(head_template).render(
title=f'({ch_index}/{total_number}) - {channel}')
dispersion_avg = self._dispersion_summary[channel]
if distribution == 'normal':
limit_confidence = config._dispersion['limit_confidence_sigma']
# Calculate upper and lower bounds
if (config._dispersion['instantatenous_dispersion']):
# For sensors with high variability in the measurements, it's better to use this
upper_bound = self.dispersion_df[channel + '_AVG']\
+ limit_confidence * self.dispersion_df[channel + '_STD']
lower_bound = self.dispersion_df[channel + '_AVG']\
- abs(limit_confidence * self.dispersion_df[channel + '_STD'])
else:
upper_bound = self.dispersion_df[channel + '_AVG']\
+ limit_confidence * dispersion_avg
lower_bound = self.dispersion_df[channel + '_AVG']\
- abs(limit_confidence * dispersion_avg)
else:
limit_confidence = t.interval(
config._dispersion['t_confidence_level'] / 100.0,
len(self.devices),
loc=self.dispersion_df[channel + '_AVG'],
scale=dispersion_avg)
upper_bound = limit_confidence[1]
lower_bound = limit_confidence[0]
udf = self.dispersion_df.copy()
udf['upper_bound'] = upper_bound
udf['lower_bound'] = lower_bound
udf = udf.fillna('null')
# List containing subplots. First list for TBR, second for OK
subplots = [[], []]
if formatting['join_sbplot']: n_subplots = 1
else: n_subplots = 2
udf.index = udf.index.astype(int) / 10**9
# Compose subplots lists
for device in self.devices:
ncol = channel + '-' + device
if ncol in self.dispersion_df.columns:
# Count how many times we go above the upper bound or below the lower one
count_problems_up = self.dispersion_df[ncol] > upper_bound
count_problems_down = self.dispersion_df[ncol] < lower_bound
# Count them
count_problems = [1 if (count_problems_up[i] or count_problems_down[i])\
else 0 for i in range(len(count_problems_up))]
# Add the trace in either
number_errors = np.sum(count_problems)
max_number_errors = len(count_problems)
# TBR
if number_errors / max_number_errors > config._dispersion[
'limit_errors'] / 100:
std_out(
f"Device {device} out of {config._dispersion['limit_errors']}% limit\
- {np.round(number_errors/max_number_errors*100, 1)}% out",
'WARNING')
subplots[0].append(ncol)
#OK
else:
subplots[n_subplots - 1].append(ncol)
# Add upper and low bound bound to subplot 0
subplots[0].append(channel + '_AVG')
subplots[0].append('upper_bound')
subplots[0].append('lower_bound')
if n_subplots > 1:
# Add upper and low bound bound to subplot 1
subplots[n_subplots - 1].append(channel + '_AVG')
subplots[n_subplots - 1].append('upper_bound')
subplots[n_subplots - 1].append('lower_bound')
ylabels = [channel + '_TBR', channel + '_OK']
else:
ylabels = [channel]
# Make subplots
for isbplt in range(n_subplots):
sdf = udf.loc[:, subplots[isbplt]]
sdf = sdf.reset_index()
data = sdf.values.T.tolist()
labels = sdf.columns
useries = [{'label': labels[0]}]
ylabel = ylabels[isbplt]
uaxes = [{
'label': formatting['xlabel'],
'labelSize': formatting['fontsize'],
}, {
'label': ylabel,
'labelSize': formatting['fontsize']
}]
color_idx = 0
for label in labels:
if label == labels[0]: continue
if color_idx + 1 > len(colors): color_idx = 0
# Gray bounds and averages
if '_bound' in label or '_AVG' in label:
stroke = 'gray'
point = {'space': 50, 'size': min([formatting['size'] - 2, 1])}
else:
stroke = colors[color_idx]
point = {'space': 0, 'size': formatting['size']}
nser = {'label': label, 'stroke': stroke, 'points': point}
useries.append(nser)
color_idx += 1
u_options = {
'width': formatting['width'],
'height': formatting['height'],
'legend': {
'isolate': True
},
'cursor': {
'lock': True,
'focus': {
'prox': 16,
},
'sync': {
'key': 'moo',
'setSeries': True,
},
'drag': {
'x': True,
'y': True,
'uni': 50,
'dist': 10,
}
},
'scales': {
'x': {
'time': True
},
'y': {
'auto': True
},
},
'series': useries,
'axes': uaxes
}
h2 = Template(uplot_template).render(data=json.dumps(data),
options=json.dumps(u_options),
subplot=isbplt)
h += h2
h = h.replace('"', "'")
h = h.replace("'null'", "null")
if options['html']:
return h
else:
iframe = f'''<iframe srcdoc="{h}" src=""
frameborder="0" width={formatting['width'] + formatting['padding-right']}
height={formatting['height'] + formatting['padding-bottom']}
sandbox="allow-scripts">
</iframe>'''
return HTML(iframe)
def device_metric_map(self, channel, start_date, end_date, options=dict()):
'''
Creates a folium map showing the evolution of a metric dynamically with colors
Parameters
-------
channel: String
The channel to make the map from
start_date, end_date: String
Date convertible string
options: dict()
Possible keys are (default otherwise)
location: list
[41.400818, 2.1825157]
Center map location
tiles: (String)
'Stamen Toner'
Tiles for the folium.Map
zoom: (float)
2.5
Zoom to start with in folium.Map
period: 'String'
'1W'
Period for 'dynamic' map
radius: float
10
Circle radius for icon
fillOpacity: float
1
(<1) Fill opacity for the icon
stroke: 'String'
'false'
'true' or 'false'. For icon's stroke
icon: 'String'
'circle'
A valid folium.Map icon style
Returns
-------
Folium.Map object
'''
# Map color bins
poll_colors_palette = array([
'#053061', '#2166ac', '#4393c3', '#92c5de', '#d1e5f0', '#fddbc7',
'#f4a582', '#d6604d', '#b2182b', '#67001f'
])
channel_bins = {
'NOISE': [-inf, 52, 54, 56, 58, 60, 62, 64, 66, 68, inf],
'PM': [0, 10, 20, 30, 40, 50, 75, 100, 150, 200, inf]
}
# Set defaults
options = dict_fmerge(config._map_def_opt, options)
# Make date range
date_r = date_range(start=start_date,
end=end_date,
normalize=True,
freq=options['period']).strftime('%Y-%m-%d')
date_l = list()
for item in date_r.values:
date_l.append(str(item))
# Get bins
for bname in channel_bins.keys():
if bname in channel:
bins = channel_bins[bname]
break
# Make features
features = []
for device in self.devices:
# Get lat, long
try:
self.devices[str(device)].api_device.get_device_lat_long()
lat = self.devices[str(device)].api_device.lat
long = self.devices[str(device)].api_device.long
except AttributeError:
pass
continue
if lat is None or long is None: continue
# Resample
try:
dfc = self.devices[str(device)].readings.resample(
options['period']).mean()
except:
pass
continue
if channel not in dfc.columns: continue
# Make color column
dfc['color'] = cut(dfc[channel], bins, labels=poll_colors_palette)
# Add point for each date
for date in date_l:
if date not in dfc.index: continue
if date_l.index(date) > len(date_l) - 2: continue
features.append({
'type': 'Feature',
'geometry': {
'type': 'LineString',
'coordinates': [[str(long), str(lat)]] * 2,
'popup': str(device),
},
'properties': {
'times': [date, date_l[date_l.index(date) + 1]],
'icon': options['icon'],
'iconstyle': {
'fillColor': str(dfc.loc[date, 'color']),
'fillOpacity': options['fillOpacity'],
'stroke': options['stroke'],
'radius': options['radius']
},
'style': {
'weight': '0'
},
'id': 'man'
}
})
# Make map
m = make_map(map_type='dynamic', features=features, options=options)
return m
