def base_process(cls, widget, job, data):
helper = UIWidgetHelper(widget, job)
rows = []
for rawrow in data:
row = {}
for col in helper.colmap.values():
if col.istime or col.isdate:
t = rawrow[col.dataindex]
try:
val = timeutils.datetime_to_microseconds(t) / 1000
except AttributeError:
val = t * 1000
else:
val = rawrow[col.dataindex]
row[col.key] = val
rows.append(row)
column_defs = [
c.to_json('key', 'label', 'sortable', 'formatter', 'allow_html')
for c in helper.colmap.values()
]
data = {
"chartTitle": widget.title.format(**job.actual_criteria),
"columns": column_defs,
"data": rows
}
return data
def write(self, index, doctype, data_frame, timecol, id_method='time'):
""" Write `data_frame` to elasticsearch storage.
:param index: name of index to use
:param doctype: elasticsearch `_type` for the records
:param data_frame: pandas dataframe
:param timecol: name of the column in data_frame to use for time
:param id_method: how to generate _id's for each record
'time' - microseconds of time column
'unique' - auto-generated unique value by elasticsearch
tuple - tuple of column names to be combined for each row
"""
# NOTE:
# Fix obscure pandas bug with NaT and fillna
#
# The error shows "AssertionError: Gaps in blk ref_locs" when
# executing fillna() on a dataframe that has a pandas.NaT
# reference in it.
#
# Instead of replacing values, just drop them when loading to ES,
# ES handles the missing items as null
df = data_frame
# find whether we have a write alias for the given index
if index in settings.ES_ROLLOVER:
index = settings.ES_ROLLOVER[index]['write_index']
if id_method == 'time':
_id = lambda x: datetime_to_microseconds(x[timecol])
elif id_method == 'unique':
_id = None
else:
# we are passed a tuple of columns
# try to get timestamp value otherwise just use the item itself
_id = lambda x: ':'.join(str(getattr(x[c], 'value', x[c]))
for c in id_method)
def gen_actions(data):
for i, row in data.iterrows():
action = {
'_index': index,
'_type': doctype,
'_source': row.dropna().to_dict()
}
if _id:
action['_id'] = _id(row)
yield action
logger.debug("Writing %s records from %s to %s into db. Index: %s, "
"doc_type: %s."
% (len(df), df[timecol].min(), df[timecol].max(),
index, doctype))
logger.debug('Calling Bulk load with client: %s' % self.client)
written, errors = helpers.bulk(self.client, actions=gen_actions(df),
stats_only=True)
logger.debug("Successfully wrote %s records, %s errors." % (written,
errors))
return
def parse_data(self):
"""Reformat netshark data results to be uniform tabular format."""
out = []
if self.timeseries:
# use sample times for each row
for d in self.data:
if d['t'] is not None:
t = datetime_to_microseconds(d['t']) / float(10 ** 6)
out.extend([t] + x for x in d['vals'])
else:
for d in self.data:
out.extend(x for x in d['vals'])
self.data = out
def parse_data(self):
"""Reformat netshark data results to be uniform tabular format."""
out = []
if self.timeseries:
# use sample times for each row
for d in self.data:
if d['t'] is not None:
t = datetime_to_microseconds(d['t']) / float(10**6)
out.extend([t] + x for x in d['vals'])
else:
for d in self.data:
out.extend(x for x in d['vals'])
self.data = out
def base_process(cls, widget, job, data):
helper = UIWidgetHelper(widget, job)
d_unit = SystemSettings.get_system_settings().data_units
rows = []
allcols = helper.colmap.values()
if widget.options.get('columns', None):
cols = [c for c in allcols if c.key in widget.options['columns']]
else:
cols = allcols
for rawrow in data:
row = {}
for col in cols:
if col.istime or col.isdate:
t = rawrow[col.dataindex]
try:
val = timeutils.datetime_to_microseconds(t) / 1000
except AttributeError:
val = t * 1000
else:
col.label = format_labels(col.label, d_unit,
helper.valcols)
val = format_single_value(col.col, rawrow[col.dataindex],
d_unit)
row[col.key] = val
rows.append(row)
column_defs = [
c.to_json('key', 'label', 'sortable', 'formatter', 'allow_html')
for c in cols
]
data = {
"chartTitle":
format_labels(widget.title.format(**job.actual_criteria), d_unit,
helper.valcols),
"columns":
column_defs,
"data":
rows
}
return data
def base_process(cls, widget, job, data):
helper = UIWidgetHelper(widget, job)
d_unit = SystemSettings.get_system_settings().data_units
rows = []
allcols = helper.colmap.values()
if widget.options.get('columns', None):
cols = [c for c in allcols if c.key in widget.options['columns']]
else:
cols = allcols
for rawrow in data:
row = {}
for col in cols:
if col.istime or col.isdate:
t = rawrow[col.dataindex]
try:
val = timeutils.datetime_to_microseconds(t) / 1000
except AttributeError:
val = t * 1000
else:
col.label = format_labels(col.label,
d_unit,
helper.valcols)
val = format_single_value(col.col,
rawrow[col.dataindex],
d_unit)
row[col.key] = val
rows.append(row)
column_defs = [
c.to_json('key', 'label', 'sortable', 'formatter', 'allow_html')
for c in cols
]
data = {
"chartTitle": format_labels(
widget.title.format(**job.actual_criteria),
d_unit, helper.valcols),
"columns": column_defs,
"data": rows
}
return data
def process(cls, widget, job, data):
class ColInfo:
def __init__(self, col, dataindex, istime=False, isdate=False):
self.col = col
self.key = cleankey(col.name)
self.dataindex = dataindex
self.istime = istime
self.isdate = isdate
w_keys = [] # Widget column keys in order that matches data
colinfo = {} # Map of ColInfo by key
w_columns = [] # Widget column definitions
for i, wc in enumerate(job.get_columns()):
if (widget.options.columns is not None
and wc.name not in widget.options.columns):
continue
ci = ColInfo(wc, i, wc.istime(), wc.isdate())
colinfo[ci.key] = ci
w_keys.append(ci.key)
# Widget column definitions, make sure this is in the order
# defined by the widget.options.columns, if specified
w_columns = []
for key in (widget.options.columns or w_keys):
ci = colinfo[key]
w_column = {'key': ci.key, 'label': ci.col.label, "sortable": True}
if ci.col.formatter:
w_column['formatter'] = ci.col.formatter
w_column['allowHTML'] = True
elif ci.col.isnumeric():
if ci.col.units == ci.col.UNITS_PCT:
w_column['formatter'] = 'formatPct'
else:
if ci.col.datatype == ci.col.DATATYPE_FLOAT:
w_column['formatter'] = 'formatMetric'
elif ci.col.datatype == ci.col.DATATYPE_INTEGER:
w_column['formatter'] = 'formatIntegerMetric'
elif ci.col.istime():
w_column['formatter'] = 'formatTime'
elif ci.col.isdate():
w_column['formatter'] = 'formatDate'
elif ci.col.datatype == ci.col.DATATYPE_HTML:
w_column['allowHTML'] = True
w_columns.append(w_column)
rows = []
for rawrow in data:
row = {}
for key in w_keys:
ci = colinfo[key]
if colinfo[key].istime or colinfo[key].isdate:
t = rawrow[ci.dataindex]
try:
val = timeutils.datetime_to_microseconds(t) / 1000
except AttributeError:
val = t * 1000
else:
val = rawrow[ci.dataindex]
row[key] = val
rows.append(row)
data = {
"chartTitle": widget.title.format(**job.actual_criteria),
"columns": w_columns,
"data": rows
}
return data
def process(cls, widget, job, data):
class ColInfo:
def __init__(self,
col,
dataindex,
axis,
istime=False,
isdate=False):
self.col = col
self.key = cleankey(col.name)
self.dataindex = dataindex
self.axis = axis
self.istime = istime
self.isdate = isdate
t_cols = job.get_columns()
colinfo = {} # map by widget key
# columns of None is a special case, just use all
# defined columns other than time
if widget.options.columns is None:
valuecolnames = [
col.name for col in t_cols
if not col.istime() and not col.isdate()
]
else:
valuecolnames = widget.options.columns
# Column keys are the 'cleaned' column names
w_keys = [cleankey(n) for n in valuecolnames]
# Retrieve the desired value columns
# ...and the indices for the value values
# (as the 'data' has *all* columns)
time_colinfo = None
for i, c in enumerate(t_cols):
if c.istime():
ci = ColInfo(c, i, -1, istime=True)
time_colinfo = ci
elif c.isdate():
ci = ColInfo(c, i, -1, isdate=True)
time_colinfo = ci
elif c.name in valuecolnames:
if c.isnumeric():
ci = ColInfo(c, i, -1, istime=False, isdate=False)
else:
raise KeyError(
"Cannot graph non-numeric data in timeseries widget: "
"column {0}".format(c.name))
colinfo[ci.key] = ci
if widget.options.altaxis:
altaxis = widget.options.altaxis
axes_def = {
'0': {
'position': 'left',
'columns':
[col for col in valuecolnames if col not in altaxis]
},
'1': {
'position': 'right',
'columns':
[col for col in valuecolnames if col in altaxis]
}
}
else:
axes_def = {'0': {'position': 'left', 'columns': valuecolnames}}
w_series = []
axes = Axes(axes_def)
# Setup the time axis
w_axes = {
"time": {
"keys": ["time"],
"position": "bottom",
"type": "time",
"styles": {
"label": {
"fontSize": "8pt",
"rotation": "-45"
}
}
}
}
# Create a better time format depending on t0/t1
t_dataindex = time_colinfo.dataindex
t0 = data[0][t_dataindex]
t1 = data[-1][t_dataindex]
if not hasattr(t0, 'utcfromtimestamp'):
t0 = timeutils.sec_string_to_datetime(t0)
t1 = timeutils.sec_string_to_datetime(t1)
total_seconds = timeutils.timedelta_total_seconds(t1 - t0)
if total_seconds < 2:
w_axes['time']['formatter'] = 'formatTimeMs'
elif total_seconds < 120:
w_axes['time']['labelFormat'] = '%k:%M:%S'
elif total_seconds < (24 * 60 * 60):
w_axes['time']['labelFormat'] = '%k:%M'
elif time_colinfo.isdate:
w_axes['time']['formatter'] = 'formatDate'
else:
w_axes['time']['labelFormat'] = '%D %k:%M'
# Setup the other axes, checking the axis for each column
for w_key in w_keys:
# Need to interate the valuecolnames array to preserve order
ci = colinfo[w_key]
w_series.append({
"xKey": "time",
"xDisplayName": "Time",
"yKey": ci.key,
"yDisplayName": ci.col.label,
"styles": {
"line": {
"weight": 1
},
"marker": {
"height": 3,
"width": 3
}
}
})
ci.axis = axes.getaxis(ci.col.name)
axis_name = 'axis' + str(ci.axis)
if axis_name not in w_axes:
w_axes[axis_name] = {
"type": "numeric",
"position": axes.position(ci.axis),
"keys": []
}
w_axes[axis_name]['keys'].append(ci.key)
# Output row data
rows = []
# min/max values by axis 0/1
minval = {}
maxval = {}
stacked = widget.options.stacked
# Iterate through all rows if input data
for rawrow in data:
t = rawrow[t_dataindex]
try:
t = timeutils.datetime_to_microseconds(t) / 1000
except AttributeError:
t = t * 1000
row = {'time': t}
rowmin = {}
rowmax = {}
for ci in colinfo.values():
if ci.istime or ci.isdate:
continue
a = ci.axis
val = rawrow[ci.dataindex]
row[ci.key] = val if val != '' else None
# If stacked and there is only one value, use that
# value as the rowmin. If stacked and there is more than
# one value for the axis, use a floor of 0 to give proper
# context.
if a not in rowmin:
rowmin[a] = val if val != '' else 0
rowmax[a] = val if val != '' else 0
else:
rowmin[a] = (0 if stacked else min(rowmin[a], val))
rowmax[a] = ((rowmax[a] +
val) if stacked else max(rowmax[a], val))
for a in rowmin.keys():
minval[a] = rowmin[a] if (a not in minval) else min(
minval[a], rowmin[a])
maxval[a] = rowmax[a] if (a not in maxval) else max(
maxval[a], rowmax[a])
rows.append(row)
# Setup the scale values for the axes
for ci in colinfo.values():
if ci.istime or ci.isdate:
continue
axis_name = 'axis' + str(ci.axis)
if minval and maxval:
n = NiceScale(minval[ci.axis], maxval[ci.axis])
w_axes[axis_name]['minimum'] = "%.10f" % n.nicemin
w_axes[axis_name]['maximum'] = "%.10f" % n.nicemax
w_axes[axis_name]['tickExponent'] = math.log10(n.tickspacing)
w_axes[axis_name]['styles'] = {
'majorUnit': {
'count': n.numticks
}
}
else:
# empty data which would result in keyError above
w_axes[axis_name]['minimum'] = "0"
w_axes[axis_name]['maximum'] = "1"
w_axes[axis_name]['tickExponent'] = 1
w_axes[axis_name]['styles'] = {'majorUnit': {'count': 1}}
if ci.col.units == ci.col.UNITS_PCT:
w_axes[axis_name]['formatter'] = 'formatPct'
else:
w_axes[axis_name]['formatter'] = 'formatMetric'
if stacked:
charttype = "area"
elif widget.options.bar:
charttype = "column"
else:
charttype = "combo"
data = {
"chartTitle": widget.title.format(**job.actual_criteria),
"type": charttype,
"stacked": stacked,
"dataProvider": rows,
"seriesCollection": w_series,
"axes": w_axes,
"legend": {
"position": "bottom",
"fontSize": "8pt",
"styles": {
"gap": 0
}
},
"interactionType": "planar"
}
# logger.debug("data:\n\n%s\n" % data)
return data
def process(cls, widget, job, data):
class ColInfo:
def __init__(self, col, dataindex, istime=False, isdate=False):
self.col = col
self.key = cleankey(col.name)
self.dataindex = dataindex
self.istime = istime
self.isdate = isdate
w_keys = [] # Widget column keys in order that matches data
colinfo = {} # Map of ColInfo by key
w_columns = [] # Widget column definitions
for i, wc in enumerate(job.get_columns()):
if (widget.options.columns is not None and
wc.name not in widget.options.columns):
continue
ci = ColInfo(wc, i, wc.istime(), wc.isdate())
colinfo[ci.key] = ci
w_keys.append(ci.key)
# Widget column definitions, make sure this is in the order
# defined by the widget.options.columns, if specified
w_columns = []
for key in (widget.options.columns or w_keys):
ci = colinfo[key]
w_column = {'key': ci.key, 'label': ci.col.label, "sortable": True}
if ci.col.formatter:
w_column['formatter'] = ci.col.formatter
w_column['allowHTML'] = True
elif ci.col.isnumeric():
if ci.col.units == ci.col.UNITS_PCT:
w_column['formatter'] = 'formatPct'
else:
if ci.col.datatype == ci.col.DATATYPE_FLOAT:
w_column['formatter'] = 'formatMetric'
elif ci.col.datatype == ci.col.DATATYPE_INTEGER:
w_column['formatter'] = 'formatIntegerMetric'
elif ci.col.istime():
w_column['formatter'] = 'formatTime'
elif ci.col.isdate():
w_column['formatter'] = 'formatDate'
elif ci.col.datatype == ci.col.DATATYPE_HTML:
w_column['allowHTML'] = True
w_columns.append(w_column)
rows = []
for rawrow in data:
row = {}
for key in w_keys:
ci = colinfo[key]
if colinfo[key].istime or colinfo[key].isdate:
t = rawrow[ci.dataindex]
try:
val = timeutils.datetime_to_microseconds(t) / 1000
except AttributeError:
val = t * 1000
else:
val = rawrow[ci.dataindex]
row[key] = val
rows.append(row)
data = {
"chartTitle": widget.title.format(**job.actual_criteria),
"columns": w_columns,
"data": rows
}
return data
def process(cls, widget, job, data):
class ColInfo:
def __init__(self, col, dataindex, axis,
istime=False, isdate=False):
self.col = col
self.key = cleankey(col.name)
self.dataindex = dataindex
self.axis = axis
self.istime = istime
self.isdate = isdate
t_cols = job.get_columns()
colinfo = {} # map by widget key
# columns of None is a special case, just use all
# defined columns other than time
if widget.options.columns is None:
valuecolnames = [col.name for col in t_cols
if not col.istime() and not col.isdate()]
else:
valuecolnames = widget.options.columns
# Column keys are the 'cleaned' column names
w_keys = [cleankey(n) for n in valuecolnames]
# Retrieve the desired value columns
# ...and the indices for the value values
# (as the 'data' has *all* columns)
time_colinfo = None
for i, c in enumerate(t_cols):
if c.istime():
ci = ColInfo(c, i, -1, istime=True)
time_colinfo = ci
elif c.isdate():
ci = ColInfo(c, i, -1, isdate=True)
time_colinfo = ci
elif c.name in valuecolnames:
if c.isnumeric():
ci = ColInfo(c, i, -1, istime=False, isdate=False)
else:
raise KeyError(
"Cannot graph non-numeric data in timeseries widget: "
"column {0}".format(c.name))
colinfo[ci.key] = ci
if widget.options.altaxis:
altaxis = widget.options.altaxis
axes_def = {'0': {'position': 'left',
'columns': [col for col in valuecolnames
if col not in altaxis]},
'1': {'position': 'right',
'columns': [col for col in valuecolnames
if col in altaxis]}
}
else:
axes_def = {'0': {'position': 'left',
'columns': valuecolnames}}
w_series = []
axes = Axes(axes_def)
# Setup the time axis
w_axes = {"time": {"keys": ["time"],
"position": "bottom",
"type": "time",
"styles": {"label": {"fontSize": "8pt",
"rotation": "-45"}}}}
# Create a better time format depending on t0/t1
t_dataindex = time_colinfo.dataindex
t0 = data[0][t_dataindex]
t1 = data[-1][t_dataindex]
if not hasattr(t0, 'utcfromtimestamp'):
t0 = timeutils.sec_string_to_datetime(t0)
t1 = timeutils.sec_string_to_datetime(t1)
total_seconds = timeutils.timedelta_total_seconds(t1 - t0)
if total_seconds < 2:
w_axes['time']['formatter'] = 'formatTimeMs'
elif total_seconds < 120:
w_axes['time']['labelFormat'] = '%k:%M:%S'
elif total_seconds < (24 * 60 * 60):
w_axes['time']['labelFormat'] = '%k:%M'
elif time_colinfo.isdate:
w_axes['time']['formatter'] = 'formatDate'
else:
w_axes['time']['labelFormat'] = '%D %k:%M'
# Setup the other axes, checking the axis for each column
for w_key in w_keys:
# Need to interate the valuecolnames array to preserve order
ci = colinfo[w_key]
w_series.append({"xKey": "time",
"xDisplayName": "Time",
"yKey": ci.key,
"yDisplayName": ci.col.label,
"styles": {"line": {"weight": 1},
"marker": {"height": 3,
"width": 3}}})
ci.axis = axes.getaxis(ci.col.name)
axis_name = 'axis' + str(ci.axis)
if axis_name not in w_axes:
w_axes[axis_name] = {"type": "numeric",
"position": axes.position(ci.axis),
"keys": []
}
w_axes[axis_name]['keys'].append(ci.key)
# Output row data
rows = []
# min/max values by axis 0/1
minval = {}
maxval = {}
stacked = widget.options.stacked
# Iterate through all rows if input data
for rawrow in data:
t = rawrow[t_dataindex]
try:
t = timeutils.datetime_to_microseconds(t) / 1000
except AttributeError:
t = t * 1000
row = {'time': t}
rowmin = {}
rowmax = {}
for ci in colinfo.values():
if ci.istime or ci.isdate:
continue
a = ci.axis
val = rawrow[ci.dataindex]
row[ci.key] = val if val != '' else None
# If stacked and there is only one value, use that
# value as the rowmin. If stacked and there is more than
# one value for the axis, use a floor of 0 to give proper
# context.
if a not in rowmin:
rowmin[a] = val if val != '' else 0
rowmax[a] = val if val != '' else 0
else:
rowmin[a] = (0 if stacked
else min(rowmin[a], val))
rowmax[a] = ((rowmax[a] + val) if stacked
else max(rowmax[a], val))
for a in rowmin.keys():
minval[a] = rowmin[a] if (a not in minval) else min(minval[a],
rowmin[a])
maxval[a] = rowmax[a] if (a not in maxval) else max(maxval[a],
rowmax[a])
rows.append(row)
# Setup the scale values for the axes
for ci in colinfo.values():
if ci.istime or ci.isdate:
continue
axis_name = 'axis' + str(ci.axis)
if minval and maxval:
n = NiceScale(minval[ci.axis], maxval[ci.axis])
w_axes[axis_name]['minimum'] = "%.10f" % n.nicemin
w_axes[axis_name]['maximum'] = "%.10f" % n.nicemax
w_axes[axis_name]['tickExponent'] = math.log10(n.tickspacing)
w_axes[axis_name]['styles'] = {
'majorUnit': {'count': n.numticks}
}
else:
# empty data which would result in keyError above
w_axes[axis_name]['minimum'] = "0"
w_axes[axis_name]['maximum'] = "1"
w_axes[axis_name]['tickExponent'] = 1
w_axes[axis_name]['styles'] = {'majorUnit': {'count': 1}}
if ci.col.units == ci.col.UNITS_PCT:
w_axes[axis_name]['formatter'] = 'formatPct'
else:
w_axes[axis_name]['formatter'] = 'formatMetric'
if stacked:
charttype = "area"
elif widget.options.bar:
charttype = "column"
else:
charttype = "combo"
data = {
"chartTitle": widget.title.format(**job.actual_criteria),
"type": charttype,
"stacked": stacked,
"dataProvider": rows,
"seriesCollection": w_series,
"axes": w_axes,
"legend": {"position": "bottom",
"fontSize": "8pt",
"styles": {"gap": 0}},
"interactionType": "planar"
}
# logger.debug("data:\n\n%s\n" % data)
return data
