def transform_row_values(series, fields, is_transposed): # Add the values to the row index_names = series.index.names or [] row = {} for key, value in series.iteritems(): key = wrap_list(key) # Get the field for the metric metric_alias = wrap_list( series.name)[0] if is_transposed else key[0] field = fields[metric_alias] data = {RAW_VALUE: raw_value(value, field)} display = _display_value(value, field, date_as=return_none) if display is not None: data["display"] = display accessor_fields = [ fields[field_alias] for field_alias in index_names if field_alias is not None ] accessor = [ safe_value(value) for value, field in zip(key, accessor_fields) ] or key setdeepattr(row, accessor, data) return row
def schemas(self, elements, **kwargs): elements = utils.wrap_list(elements) value = utils.wrap_list(self.value) criterion = None for element, value in zip(elements, value): crit = getattr(element, self.operator)(value) if criterion: criterion = criterion & crit else: criterion = crit return criterion
def schemas(self, element): element = utils.wrap_list(element) starts = utils.wrap_list(self.start) stops = utils.wrap_list(self.stop) criterion = None for el, start, stop in zip(element, starts, stops): crit = el[start:stop] if criterion: criterion = criterion & crit else: criterion = crit return criterion
def _data_row(self, dimensions, dimension_values, dimension_display_values, references, row_data): """ WRITEME :param dimensions: :param dimension_values: :param dimension_display_values: :param row_data: :return: """ row = {} for dimension, dimension_value in zip(dimensions, utils.wrap_list(dimension_values)): df_key = format_dimension_key(dimension.key) row[dimension.key] = _render_dimension_cell(dimension_value, dimension_display_values.get(df_key)) for metric in self.items: for reference in [None] + references: key = reference_key(metric, reference) df_key = format_metric_key(key) row[key] = _render_dimensional_metric_cell(row_data, metric) \ if isinstance(row_data.index, pd.MultiIndex) \ else _format_metric_cell(row_data[df_key], metric) return row
def _display_metrics(self, metrics, operations): display = OrderedDict() axis = 0 for metrics_level in metrics: for metric_key in utils.wrap_list(metrics_level): schema = self.slicer.metrics[metric_key] display[metric_key] = { attr: getattr(schema, attr) for attr in ['label', 'precision', 'prefix', 'suffix'] if getattr(schema, attr) is not None } display[metric_key]["axis"] = axis axis += 1 for operation in operations: if not hasattr(operation, 'metric_key'): continue metric_key = operation.metric_key metric_schema = self.slicer.metrics[metric_key] key = '{}_{}'.format(metric_key, operation.key) display[key] = { attr: getattr(metric_schema, attr) for attr in ['precision', 'prefix', 'suffix'] if getattr(metric_schema, attr) is not None } display[key]['label'] = '{} {}'.format(metric_schema.label, operation.label) display[key]["axis"] = axis axis += 1 return display
def sort_data_frame(self, data_frame): if not self.sort or len(data_frame) == 1: # If there are no sort arguments or the data frame is a single row, then no need to sort return data_frame # reset the index so all columns can be sorted together index_names = data_frame.index.names unsorted = data_frame.reset_index() column_names = list(unsorted.columns) ascending = self.ascending if self.ascending is not None else True sort = wrap_list(self.sort) sort_columns = [ column_names[column] for column in sort if column < len(column_names) ] if not sort_columns: return data_frame # ignore additional values for ascending if they do not align lengthwise with sort_columns # Default to the first value or None if isinstance(ascending, list) and len(ascending) != len(sort_columns): ascending = ascending[0] if len(ascending) > 0 else None sorted = unsorted.sort_values( sort_columns, ascending=ascending).set_index(index_names) # Maintain the single metric name if hasattr(data_frame, "name"): sorted.name = data_frame.name return sorted
def transform_row_values(self, series, fields: Dict[str, Field], is_transposed: bool, is_pivoted: bool, hide_aliases: List[str]): row = {} row_colors = None for key, value in series.items(): if key in hide_aliases: continue key = wrap_list(key) # Get the field for the metric metric_alias = wrap_list( series.name)[0] if is_transposed else key[0] field = fields[metric_alias] data = { RAW_VALUE: raw_value(value, field), } if not row_colors: # No color for this field yet rule = find_rule_to_apply( self.formatting_rules_map[metric_alias], value) if rule is not None: colors = rule.determine_colors(value) data["color"], data["text_color"] = colors if not is_transposed and not is_pivoted and rule.covers_row: # No transposing or pivoting going on so set as row color if it's specified for the rule row_colors = colors display = _display_value(value, field, date_as=return_none) if display is not None: data["display"] = display accessor = self._get_row_value_accessor(series, fields, key) setdeepattr(row, accessor, data) # Assign the row colors to fields that aren't colored yet if row_colors: for key in series.keys(): accessor = self._get_row_value_accessor( series, fields, wrap_list(key)) data = getdeepattr(row, accessor) if "color" not in data: data["color"], data["text_color"] = row_colors return row, row_colors
def _render_timeseries_data(group_df, metric_key): series = [] for dimension_values, y in group_df[metric_key].iteritems(): first_dimension_value = utils.wrap_list(dimension_values)[0] if pd.isnull(first_dimension_value): # Ignore totals on the x-axis. continue series.append((formats.date_as_millis(first_dimension_value), formats.metric_value(y))) return series
def calculate_min_max(self, df, is_transposed): if not self.min_max_map: return for index, series in df.iterrows(): for key, value in series.items(): metric_alias = wrap_list( series.name)[0] if is_transposed else wrap_list(key)[0] min_max = self.min_max_map.get(metric_alias) if min_max is not None: # we need to calculate the min and max values for this metric. if value < min_max[0]: min_max[ 0] = value # value is smaller than stored smallest value, replace it. if value > min_max[1]: min_max[ 1] = value # value is bigger than stored biggest value, replace it. for rule_list in self.formatting_rules_map.values(): for rule in rule_list: if isinstance(rule, FormattingHeatMapRule): rule.set_min_max( *self.min_max_map[rule.get_field_selector()])
def _get_timeseries_positions(df: pd.DataFrame, dimension_alias: str): timeseries_positions = [] for dimensions, dimension_value in df[dimension_alias].iteritems(): datetime = utils.wrap_list(dimensions)[0] timeseries_positions.append({ "position": formats.date_as_millis(datetime), "label": dimension_value }) return timeseries_positions
def transform_data(data_frame, field_map, dimension_hyperlink_templates, is_transposed): """ Builds a list of dicts containing the data for ReactTable. This aligns with the accessors set by #transform_dimension_column_headers and #transform_metric_column_headers :param data_frame: The result set data frame :param field_map: A mapping to all the fields in the dataset used for this query. :param dimension_hyperlink_templates: :param is_transposed: """ index_names = data_frame.index.names def _get_field_label(alias): if alias not in field_map: return alias field = field_map[alias] return getattr(field, "label", field.alias) # If the metric column was dropped due to only having a single metric, add it back here so the # formatting can be applied. if hasattr(data_frame, "name"): metric_alias = data_frame.name data_frame = pd.concat( [data_frame], keys=[metric_alias], names=[F_METRICS_DIMENSION_ALIAS], axis=1, ) rows = [] for index, series in data_frame.iterrows(): index = wrap_list(index) # Get a list of values from the index. These can be metrics or dimensions so it checks in the item map if # there is a display value for the value index_values = ([_get_field_label(value) for value in index] if is_transposed else index) index_display_values = OrderedDict(zip(index_names, index_values)) rows.append({ **ReactTable.transform_row_index( index_display_values, field_map, dimension_hyperlink_templates), **ReactTable.transform_row_values(series, field_map, is_transposed), }) return rows
def _render_timeseries_data(group_df, metric_alias, metric): series = [] for dimension_values, y in group_df[metric_alias].iteritems(): first_dimension_value = utils.wrap_list(dimension_values)[0] # Ignore empty result sets where the only row is totals if first_dimension_value in TOTALS_MARKERS: continue if pd.isnull(first_dimension_value): # Ignore totals on the x-axis. continue series.append(( formats.date_as_millis(first_dimension_value), formats.raw_value(y, metric), )) return series
def _get_category_positions(df, dimension_alias, axis): dimension_positions = [] category_positions = { category: index for index, category in enumerate(axis["categories"]) } for dimensions, dimension_value in df[dimension_alias].iteritems(): category_label = utils.wrap_list(dimensions)[0] dimension_positions.append({ "position": category_positions[category_label], "label": dimension_value, }) return dimension_positions
def transform_data_row_values(cls, series, item_map): # Add the values to the row row = {} for key, value in series.iteritems(): key = wrap_list(key) value = metric_value(value) data = {RAW_VALUE: metric_value(value)} # Try to find a display value for the item item = item_map.get(key[0]) display = metric_display(value, getattr(item, 'prefix', None), getattr(item, 'suffix', None), getattr(item, 'precision', None)) if display is not None: data['display'] = display setdeepattr(row, key, data) return row
def _render_pie_series(self, metric: Field, reference: Reference, data_frame: pd.DataFrame, dimension_fields: List[Field]) -> dict: metric_alias = utils.alias_selector(metric.alias) if self.split_dimension: dimension_fields = [ dimension for dimension in dimension_fields if dimension != self.split_dimension ] data_frame = data_frame.reset_index(alias_selector( self.split_dimension.alias), drop=True) data = [] for dimension_values, y in data_frame[metric_alias].iteritems(): dimension_values = utils.wrap_list(dimension_values) name = self._format_dimension_values(dimension_fields, dimension_values) data.append({ "name": name or metric.label, "y": formats.raw_value(y, metric) }) return { "name": reference_label(metric, reference), "type": "pie", "data": data, "tooltip": { "pointFormat": '<span style="color:{point.color}">\u25CF</span> {series.name}: ' "<b>{point.y} ({point.percentage:.1f}%)</b><br/>", "valueDecimals": metric.precision, "valuePrefix": reference_prefix(metric, reference), "valueSuffix": reference_suffix(metric, reference), }, }
def render_series_label(dimension_values, metric=None, reference=None): """ Returns a string label for a metric, reference, and set of values for zero or more dimensions. :param metric: an instance of fireant.Metric :param reference: an instance of fireant.Reference :param dimension_values: a tuple of dimension values. Can be zero-length or longer. :return: """ # normalize the dimension values, as we expect them to be an iterable, in # order to calculate the number of used dimension safely dimension_values = utils.wrap_list(dimension_values) num_used_dimensions = len(dimensions) - len(dimension_values) used_dimensions = dimensions[num_used_dimensions:] dimension_labels = [utils.getdeepattr(dimension_display_values, (utils.format_dimension_key(dimension.key), dimension_value), dimension_value) if dimension_value not in TOTALS_MARKERS else 'Totals' for dimension, dimension_value in zip(used_dimensions, dimension_values)] label = ", ".join([str(label) for label in dimension_labels]) if metric is None: if reference is not None: return '{} ({})'.format(label, reference.label) return label if dimension_labels: return '{} ({})'.format(reference_label(metric, reference), label) return reference_label(metric, reference)
def _render_pie_series(self, metric, reference, data_frame, dimension_fields): metric_alias = utils.alias_selector(metric.alias) data = [] for dimension_values, y in data_frame[metric_alias].iteritems(): dimension_values = utils.wrap_list(dimension_values) name = self._format_dimension_values(dimension_fields, dimension_values) data.append( {"name": name or metric.label, "y": formats.raw_value(y, metric),} ) return { "name": reference_label(metric, reference), "type": "pie", "data": data, "tooltip": { "pointFormat": '<span style="color:{point.color}">\u25CF</span> {series.name}: ' "<b>{point.y} ({point.percentage:.1f}%)</b><br/>", "valueDecimals": metric.precision, "valuePrefix": reference_prefix(metric, reference), "valueSuffix": reference_suffix(metric, reference), }, }
def _render_series(self, axis, axis_idx, axis_color, colors, series_data_frames, render_series_label, references, is_timeseries=False): """ Renders the series configuration. https://api.highcharts.com/highcharts/series :param axis: :param axis_idx: :param axis_color: :param colors: :param series_data_frames: :param render_series_label: :param references: :param is_timeseries: :return: """ hc_series = [] for series in axis: symbols = itertools.cycle(MARKER_SYMBOLS) for (dimension_values, group_df), symbol in zip(series_data_frames, symbols): if is_timeseries: group_df = group_df.sort_index(level=0) dimension_values = utils.wrap_list(dimension_values) if isinstance(series, self.PieSeries): # pie charts suck for reference in [None] + references: hc_series.append(self._render_pie_series(series, reference, group_df, render_series_label)) continue # With a single axis, use different colors for each series # With multiple axes, use the same color for the entire axis and only change the dash style series_color = next(colors) for reference, dash_style in zip([None] + references, itertools.cycle(DASH_STYLES)): metric_key = utils.format_metric_key(reference_key(series.metric, reference)) hc_series.append({ "type": series.type, "name": render_series_label(dimension_values, series.metric, reference), "data": ( self._render_timeseries_data(group_df, metric_key) if is_timeseries else self._render_category_data(group_df, metric_key) ), "tooltip": self._render_tooltip(series.metric, reference), "yAxis": ("{}_{}".format(axis_idx, reference.key) if reference is not None and reference.delta else str(axis_idx)), "marker": ({"symbol": symbol, "fillColor": axis_color or series_color} if isinstance(series, SERIES_NEEDING_MARKER) else {}), "stacking": series.stacking, }) if isinstance(series, ContinuousAxisSeries): # Set each series in a continuous series to a specific color hc_series[-1]["color"] = series_color hc_series[-1]["dashStyle"] = dash_style return hc_series
def transform_data( self, data_frame: pd.DataFrame, field_map: Dict[str, Field], hide_aliases: List[str], dimension_hyperlink_templates: Dict[str, str], is_transposed: bool, is_pivoted: bool, ) -> List[dict]: """ Builds a list of dicts containing the data for ReactTable. This aligns with the accessors set by #transform_dimension_column_headers and #transform_metric_column_headers :param data_frame: The result set data frame. :param field_map: A mapping to all the fields in the dataset used for this query. :param hide_aliases: A set with hide dimension aliases. :param dimension_hyperlink_templates: A mapping to fields and its hyperlink dimension, if any. :param is_transposed: Whether the table is transposed or not. :param is_pivoted: Whether the table is pivoted or not. """ index_names = data_frame.index.names def _get_field_label(alias): if alias not in field_map: return alias field = field_map[alias] return getattr(field, "label", field.alias) # If the metric column was dropped due to only having a single metric, add it back here so the # formatting can be applied. if hasattr(data_frame, "name"): metric_alias = data_frame.name data_frame = pd.concat( [data_frame], keys=[metric_alias], names=[F_METRICS_DIMENSION_ALIAS], axis=1, ) self.calculate_min_max(data_frame, is_transposed) rows = [] for index, series in data_frame.iterrows(): row_values, row_colors = self.transform_row_values( series, field_map, is_transposed, is_pivoted, hide_aliases) index = wrap_list(index) # Get a list of values from the index. These can be metrics or dimensions so it checks in the item map if # there is a display value for the value index_values = [_get_field_label(value) for value in index] if is_transposed else index index_display_values = OrderedDict(zip(index_names, index_values)) row_index = self.transform_row_index( index_display_values, field_map, dimension_hyperlink_templates, hide_aliases, row_colors) rows.append({ **row_index, **row_values, }) return rows
def _render_series( self, axis, axis_idx: int, axis_color: str, colors: List[str], data_frame: pd.DataFrame, series_data_frames, dimensions: List[Field], references: List[Reference], is_timeseries: bool = False, ) -> List[dict]: """ Renders the series configuration. https://api.highcharts.com/highcharts/series :param axis: :param axis_idx: :param axis_color: :param colors: :param series_data_frames: :param dimensions: :param references: :param is_timeseries: :return: """ hc_series = [] for series in axis: # Pie Charts, do not break data out into groups, render all data in one pie chart series if isinstance(series, self.PieSeries): # Pie charts do not group the data up by series so render them separately and then go to the next series for reference in [None] + references: hc_series.append( self._render_pie_series(series.metric, reference, data_frame, dimensions)) continue # For other series types, create a highcharts series for each group (combination of dimension values) symbols = itertools.cycle(MARKER_SYMBOLS) for (dimension_values, group_df), symbol in zip(series_data_frames, symbols): dimension_values = utils.wrap_list(dimension_values) dimension_label = self._format_dimension_values( dimensions[1:], dimension_values) hc_series += self._render_highcharts_series( series, group_df, references, dimension_label, is_timeseries, symbol, axis_idx, axis_color, next(colors), ) return hc_series