def base_process(cls, widget, job, data): helper = UIWidgetHelper(widget, job) rows = [] for rawrow in data: row = {} for col in helper.colmap.values(): if col.istime or col.isdate: t = rawrow[col.dataindex] try: val = timeutils.datetime_to_microseconds(t) / 1000 except AttributeError: val = t * 1000 else: val = rawrow[col.dataindex] row[col.key] = val rows.append(row) column_defs = [ c.to_json('key', 'label', 'sortable', 'formatter', 'allow_html') for c in helper.colmap.values() ] data = { "chartTitle": widget.title.format(**job.actual_criteria), "columns": column_defs, "data": rows } return data
def write(self, index, doctype, data_frame, timecol, id_method='time'): """ Write `data_frame` to elasticsearch storage. :param index: name of index to use :param doctype: elasticsearch `_type` for the records :param data_frame: pandas dataframe :param timecol: name of the column in data_frame to use for time :param id_method: how to generate _id's for each record 'time' - microseconds of time column 'unique' - auto-generated unique value by elasticsearch tuple - tuple of column names to be combined for each row """ # NOTE: # Fix obscure pandas bug with NaT and fillna # # The error shows "AssertionError: Gaps in blk ref_locs" when # executing fillna() on a dataframe that has a pandas.NaT # reference in it. # # Instead of replacing values, just drop them when loading to ES, # ES handles the missing items as null df = data_frame # find whether we have a write alias for the given index if index in settings.ES_ROLLOVER: index = settings.ES_ROLLOVER[index]['write_index'] if id_method == 'time': _id = lambda x: datetime_to_microseconds(x[timecol]) elif id_method == 'unique': _id = None else: # we are passed a tuple of columns # try to get timestamp value otherwise just use the item itself _id = lambda x: ':'.join(str(getattr(x[c], 'value', x[c])) for c in id_method) def gen_actions(data): for i, row in data.iterrows(): action = { '_index': index, '_type': doctype, '_source': row.dropna().to_dict() } if _id: action['_id'] = _id(row) yield action logger.debug("Writing %s records from %s to %s into db. Index: %s, " "doc_type: %s." % (len(df), df[timecol].min(), df[timecol].max(), index, doctype)) logger.debug('Calling Bulk load with client: %s' % self.client) written, errors = helpers.bulk(self.client, actions=gen_actions(df), stats_only=True) logger.debug("Successfully wrote %s records, %s errors." % (written, errors)) return
def parse_data(self): """Reformat netshark data results to be uniform tabular format.""" out = [] if self.timeseries: # use sample times for each row for d in self.data: if d['t'] is not None: t = datetime_to_microseconds(d['t']) / float(10 ** 6) out.extend([t] + x for x in d['vals']) else: for d in self.data: out.extend(x for x in d['vals']) self.data = out
def parse_data(self): """Reformat netshark data results to be uniform tabular format.""" out = [] if self.timeseries: # use sample times for each row for d in self.data: if d['t'] is not None: t = datetime_to_microseconds(d['t']) / float(10**6) out.extend([t] + x for x in d['vals']) else: for d in self.data: out.extend(x for x in d['vals']) self.data = out
def base_process(cls, widget, job, data): helper = UIWidgetHelper(widget, job) d_unit = SystemSettings.get_system_settings().data_units rows = [] allcols = helper.colmap.values() if widget.options.get('columns', None): cols = [c for c in allcols if c.key in widget.options['columns']] else: cols = allcols for rawrow in data: row = {} for col in cols: if col.istime or col.isdate: t = rawrow[col.dataindex] try: val = timeutils.datetime_to_microseconds(t) / 1000 except AttributeError: val = t * 1000 else: col.label = format_labels(col.label, d_unit, helper.valcols) val = format_single_value(col.col, rawrow[col.dataindex], d_unit) row[col.key] = val rows.append(row) column_defs = [ c.to_json('key', 'label', 'sortable', 'formatter', 'allow_html') for c in cols ] data = { "chartTitle": format_labels(widget.title.format(**job.actual_criteria), d_unit, helper.valcols), "columns": column_defs, "data": rows } return data
def base_process(cls, widget, job, data): helper = UIWidgetHelper(widget, job) d_unit = SystemSettings.get_system_settings().data_units rows = [] allcols = helper.colmap.values() if widget.options.get('columns', None): cols = [c for c in allcols if c.key in widget.options['columns']] else: cols = allcols for rawrow in data: row = {} for col in cols: if col.istime or col.isdate: t = rawrow[col.dataindex] try: val = timeutils.datetime_to_microseconds(t) / 1000 except AttributeError: val = t * 1000 else: col.label = format_labels(col.label, d_unit, helper.valcols) val = format_single_value(col.col, rawrow[col.dataindex], d_unit) row[col.key] = val rows.append(row) column_defs = [ c.to_json('key', 'label', 'sortable', 'formatter', 'allow_html') for c in cols ] data = { "chartTitle": format_labels( widget.title.format(**job.actual_criteria), d_unit, helper.valcols), "columns": column_defs, "data": rows } return data
def process(cls, widget, job, data): class ColInfo: def __init__(self, col, dataindex, istime=False, isdate=False): self.col = col self.key = cleankey(col.name) self.dataindex = dataindex self.istime = istime self.isdate = isdate w_keys = [] # Widget column keys in order that matches data colinfo = {} # Map of ColInfo by key w_columns = [] # Widget column definitions for i, wc in enumerate(job.get_columns()): if (widget.options.columns is not None and wc.name not in widget.options.columns): continue ci = ColInfo(wc, i, wc.istime(), wc.isdate()) colinfo[ci.key] = ci w_keys.append(ci.key) # Widget column definitions, make sure this is in the order # defined by the widget.options.columns, if specified w_columns = [] for key in (widget.options.columns or w_keys): ci = colinfo[key] w_column = {'key': ci.key, 'label': ci.col.label, "sortable": True} if ci.col.formatter: w_column['formatter'] = ci.col.formatter w_column['allowHTML'] = True elif ci.col.isnumeric(): if ci.col.units == ci.col.UNITS_PCT: w_column['formatter'] = 'formatPct' else: if ci.col.datatype == ci.col.DATATYPE_FLOAT: w_column['formatter'] = 'formatMetric' elif ci.col.datatype == ci.col.DATATYPE_INTEGER: w_column['formatter'] = 'formatIntegerMetric' elif ci.col.istime(): w_column['formatter'] = 'formatTime' elif ci.col.isdate(): w_column['formatter'] = 'formatDate' elif ci.col.datatype == ci.col.DATATYPE_HTML: w_column['allowHTML'] = True w_columns.append(w_column) rows = [] for rawrow in data: row = {} for key in w_keys: ci = colinfo[key] if colinfo[key].istime or colinfo[key].isdate: t = rawrow[ci.dataindex] try: val = timeutils.datetime_to_microseconds(t) / 1000 except AttributeError: val = t * 1000 else: val = rawrow[ci.dataindex] row[key] = val rows.append(row) data = { "chartTitle": widget.title.format(**job.actual_criteria), "columns": w_columns, "data": rows } return data
def process(cls, widget, job, data): class ColInfo: def __init__(self, col, dataindex, axis, istime=False, isdate=False): self.col = col self.key = cleankey(col.name) self.dataindex = dataindex self.axis = axis self.istime = istime self.isdate = isdate t_cols = job.get_columns() colinfo = {} # map by widget key # columns of None is a special case, just use all # defined columns other than time if widget.options.columns is None: valuecolnames = [ col.name for col in t_cols if not col.istime() and not col.isdate() ] else: valuecolnames = widget.options.columns # Column keys are the 'cleaned' column names w_keys = [cleankey(n) for n in valuecolnames] # Retrieve the desired value columns # ...and the indices for the value values # (as the 'data' has *all* columns) time_colinfo = None for i, c in enumerate(t_cols): if c.istime(): ci = ColInfo(c, i, -1, istime=True) time_colinfo = ci elif c.isdate(): ci = ColInfo(c, i, -1, isdate=True) time_colinfo = ci elif c.name in valuecolnames: if c.isnumeric(): ci = ColInfo(c, i, -1, istime=False, isdate=False) else: raise KeyError( "Cannot graph non-numeric data in timeseries widget: " "column {0}".format(c.name)) colinfo[ci.key] = ci if widget.options.altaxis: altaxis = widget.options.altaxis axes_def = { '0': { 'position': 'left', 'columns': [col for col in valuecolnames if col not in altaxis] }, '1': { 'position': 'right', 'columns': [col for col in valuecolnames if col in altaxis] } } else: axes_def = {'0': {'position': 'left', 'columns': valuecolnames}} w_series = [] axes = Axes(axes_def) # Setup the time axis w_axes = { "time": { "keys": ["time"], "position": "bottom", "type": "time", "styles": { "label": { "fontSize": "8pt", "rotation": "-45" } } } } # Create a better time format depending on t0/t1 t_dataindex = time_colinfo.dataindex t0 = data[0][t_dataindex] t1 = data[-1][t_dataindex] if not hasattr(t0, 'utcfromtimestamp'): t0 = timeutils.sec_string_to_datetime(t0) t1 = timeutils.sec_string_to_datetime(t1) total_seconds = timeutils.timedelta_total_seconds(t1 - t0) if total_seconds < 2: w_axes['time']['formatter'] = 'formatTimeMs' elif total_seconds < 120: w_axes['time']['labelFormat'] = '%k:%M:%S' elif total_seconds < (24 * 60 * 60): w_axes['time']['labelFormat'] = '%k:%M' elif time_colinfo.isdate: w_axes['time']['formatter'] = 'formatDate' else: w_axes['time']['labelFormat'] = '%D %k:%M' # Setup the other axes, checking the axis for each column for w_key in w_keys: # Need to interate the valuecolnames array to preserve order ci = colinfo[w_key] w_series.append({ "xKey": "time", "xDisplayName": "Time", "yKey": ci.key, "yDisplayName": ci.col.label, "styles": { "line": { "weight": 1 }, "marker": { "height": 3, "width": 3 } } }) ci.axis = axes.getaxis(ci.col.name) axis_name = 'axis' + str(ci.axis) if axis_name not in w_axes: w_axes[axis_name] = { "type": "numeric", "position": axes.position(ci.axis), "keys": [] } w_axes[axis_name]['keys'].append(ci.key) # Output row data rows = [] # min/max values by axis 0/1 minval = {} maxval = {} stacked = widget.options.stacked # Iterate through all rows if input data for rawrow in data: t = rawrow[t_dataindex] try: t = timeutils.datetime_to_microseconds(t) / 1000 except AttributeError: t = t * 1000 row = {'time': t} rowmin = {} rowmax = {} for ci in colinfo.values(): if ci.istime or ci.isdate: continue a = ci.axis val = rawrow[ci.dataindex] row[ci.key] = val if val != '' else None # If stacked and there is only one value, use that # value as the rowmin. If stacked and there is more than # one value for the axis, use a floor of 0 to give proper # context. if a not in rowmin: rowmin[a] = val if val != '' else 0 rowmax[a] = val if val != '' else 0 else: rowmin[a] = (0 if stacked else min(rowmin[a], val)) rowmax[a] = ((rowmax[a] + val) if stacked else max(rowmax[a], val)) for a in rowmin.keys(): minval[a] = rowmin[a] if (a not in minval) else min( minval[a], rowmin[a]) maxval[a] = rowmax[a] if (a not in maxval) else max( maxval[a], rowmax[a]) rows.append(row) # Setup the scale values for the axes for ci in colinfo.values(): if ci.istime or ci.isdate: continue axis_name = 'axis' + str(ci.axis) if minval and maxval: n = NiceScale(minval[ci.axis], maxval[ci.axis]) w_axes[axis_name]['minimum'] = "%.10f" % n.nicemin w_axes[axis_name]['maximum'] = "%.10f" % n.nicemax w_axes[axis_name]['tickExponent'] = math.log10(n.tickspacing) w_axes[axis_name]['styles'] = { 'majorUnit': { 'count': n.numticks } } else: # empty data which would result in keyError above w_axes[axis_name]['minimum'] = "0" w_axes[axis_name]['maximum'] = "1" w_axes[axis_name]['tickExponent'] = 1 w_axes[axis_name]['styles'] = {'majorUnit': {'count': 1}} if ci.col.units == ci.col.UNITS_PCT: w_axes[axis_name]['formatter'] = 'formatPct' else: w_axes[axis_name]['formatter'] = 'formatMetric' if stacked: charttype = "area" elif widget.options.bar: charttype = "column" else: charttype = "combo" data = { "chartTitle": widget.title.format(**job.actual_criteria), "type": charttype, "stacked": stacked, "dataProvider": rows, "seriesCollection": w_series, "axes": w_axes, "legend": { "position": "bottom", "fontSize": "8pt", "styles": { "gap": 0 } }, "interactionType": "planar" } # logger.debug("data:\n\n%s\n" % data) return data
def process(cls, widget, job, data): class ColInfo: def __init__(self, col, dataindex, axis, istime=False, isdate=False): self.col = col self.key = cleankey(col.name) self.dataindex = dataindex self.axis = axis self.istime = istime self.isdate = isdate t_cols = job.get_columns() colinfo = {} # map by widget key # columns of None is a special case, just use all # defined columns other than time if widget.options.columns is None: valuecolnames = [col.name for col in t_cols if not col.istime() and not col.isdate()] else: valuecolnames = widget.options.columns # Column keys are the 'cleaned' column names w_keys = [cleankey(n) for n in valuecolnames] # Retrieve the desired value columns # ...and the indices for the value values # (as the 'data' has *all* columns) time_colinfo = None for i, c in enumerate(t_cols): if c.istime(): ci = ColInfo(c, i, -1, istime=True) time_colinfo = ci elif c.isdate(): ci = ColInfo(c, i, -1, isdate=True) time_colinfo = ci elif c.name in valuecolnames: if c.isnumeric(): ci = ColInfo(c, i, -1, istime=False, isdate=False) else: raise KeyError( "Cannot graph non-numeric data in timeseries widget: " "column {0}".format(c.name)) colinfo[ci.key] = ci if widget.options.altaxis: altaxis = widget.options.altaxis axes_def = {'0': {'position': 'left', 'columns': [col for col in valuecolnames if col not in altaxis]}, '1': {'position': 'right', 'columns': [col for col in valuecolnames if col in altaxis]} } else: axes_def = {'0': {'position': 'left', 'columns': valuecolnames}} w_series = [] axes = Axes(axes_def) # Setup the time axis w_axes = {"time": {"keys": ["time"], "position": "bottom", "type": "time", "styles": {"label": {"fontSize": "8pt", "rotation": "-45"}}}} # Create a better time format depending on t0/t1 t_dataindex = time_colinfo.dataindex t0 = data[0][t_dataindex] t1 = data[-1][t_dataindex] if not hasattr(t0, 'utcfromtimestamp'): t0 = timeutils.sec_string_to_datetime(t0) t1 = timeutils.sec_string_to_datetime(t1) total_seconds = timeutils.timedelta_total_seconds(t1 - t0) if total_seconds < 2: w_axes['time']['formatter'] = 'formatTimeMs' elif total_seconds < 120: w_axes['time']['labelFormat'] = '%k:%M:%S' elif total_seconds < (24 * 60 * 60): w_axes['time']['labelFormat'] = '%k:%M' elif time_colinfo.isdate: w_axes['time']['formatter'] = 'formatDate' else: w_axes['time']['labelFormat'] = '%D %k:%M' # Setup the other axes, checking the axis for each column for w_key in w_keys: # Need to interate the valuecolnames array to preserve order ci = colinfo[w_key] w_series.append({"xKey": "time", "xDisplayName": "Time", "yKey": ci.key, "yDisplayName": ci.col.label, "styles": {"line": {"weight": 1}, "marker": {"height": 3, "width": 3}}}) ci.axis = axes.getaxis(ci.col.name) axis_name = 'axis' + str(ci.axis) if axis_name not in w_axes: w_axes[axis_name] = {"type": "numeric", "position": axes.position(ci.axis), "keys": [] } w_axes[axis_name]['keys'].append(ci.key) # Output row data rows = [] # min/max values by axis 0/1 minval = {} maxval = {} stacked = widget.options.stacked # Iterate through all rows if input data for rawrow in data: t = rawrow[t_dataindex] try: t = timeutils.datetime_to_microseconds(t) / 1000 except AttributeError: t = t * 1000 row = {'time': t} rowmin = {} rowmax = {} for ci in colinfo.values(): if ci.istime or ci.isdate: continue a = ci.axis val = rawrow[ci.dataindex] row[ci.key] = val if val != '' else None # If stacked and there is only one value, use that # value as the rowmin. If stacked and there is more than # one value for the axis, use a floor of 0 to give proper # context. if a not in rowmin: rowmin[a] = val if val != '' else 0 rowmax[a] = val if val != '' else 0 else: rowmin[a] = (0 if stacked else min(rowmin[a], val)) rowmax[a] = ((rowmax[a] + val) if stacked else max(rowmax[a], val)) for a in rowmin.keys(): minval[a] = rowmin[a] if (a not in minval) else min(minval[a], rowmin[a]) maxval[a] = rowmax[a] if (a not in maxval) else max(maxval[a], rowmax[a]) rows.append(row) # Setup the scale values for the axes for ci in colinfo.values(): if ci.istime or ci.isdate: continue axis_name = 'axis' + str(ci.axis) if minval and maxval: n = NiceScale(minval[ci.axis], maxval[ci.axis]) w_axes[axis_name]['minimum'] = "%.10f" % n.nicemin w_axes[axis_name]['maximum'] = "%.10f" % n.nicemax w_axes[axis_name]['tickExponent'] = math.log10(n.tickspacing) w_axes[axis_name]['styles'] = { 'majorUnit': {'count': n.numticks} } else: # empty data which would result in keyError above w_axes[axis_name]['minimum'] = "0" w_axes[axis_name]['maximum'] = "1" w_axes[axis_name]['tickExponent'] = 1 w_axes[axis_name]['styles'] = {'majorUnit': {'count': 1}} if ci.col.units == ci.col.UNITS_PCT: w_axes[axis_name]['formatter'] = 'formatPct' else: w_axes[axis_name]['formatter'] = 'formatMetric' if stacked: charttype = "area" elif widget.options.bar: charttype = "column" else: charttype = "combo" data = { "chartTitle": widget.title.format(**job.actual_criteria), "type": charttype, "stacked": stacked, "dataProvider": rows, "seriesCollection": w_series, "axes": w_axes, "legend": {"position": "bottom", "fontSize": "8pt", "styles": {"gap": 0}}, "interactionType": "planar" } # logger.debug("data:\n\n%s\n" % data) return data