def transform_row_values(series, fields, is_transposed):
# Add the values to the row
index_names = series.index.names or []
row = {}
for key, value in series.iteritems():
key = wrap_list(key)
# Get the field for the metric
metric_alias = wrap_list(
series.name)[0] if is_transposed else key[0]
field = fields[metric_alias]
data = {RAW_VALUE: raw_value(value, field)}
display = _display_value(value, field, date_as=return_none)
if display is not None:
data["display"] = display
accessor_fields = [
fields[field_alias] for field_alias in index_names
if field_alias is not None
]
accessor = [
safe_value(value)
for value, field in zip(key, accessor_fields)
] or key
setdeepattr(row, accessor, data)
return row
def schemas(self, elements, **kwargs):
elements = utils.wrap_list(elements)
value = utils.wrap_list(self.value)
criterion = None
for element, value in zip(elements, value):
crit = getattr(element, self.operator)(value)
if criterion:
criterion = criterion & crit
else:
criterion = crit
return criterion
def schemas(self, element):
element = utils.wrap_list(element)
starts = utils.wrap_list(self.start)
stops = utils.wrap_list(self.stop)
criterion = None
for el, start, stop in zip(element, starts, stops):
crit = el[start:stop]
if criterion:
criterion = criterion & crit
else:
criterion = crit
return criterion
def _data_row(self, dimensions, dimension_values, dimension_display_values, references, row_data):
"""
WRITEME
:param dimensions:
:param dimension_values:
:param dimension_display_values:
:param row_data:
:return:
"""
row = {}
for dimension, dimension_value in zip(dimensions, utils.wrap_list(dimension_values)):
df_key = format_dimension_key(dimension.key)
row[dimension.key] = _render_dimension_cell(dimension_value, dimension_display_values.get(df_key))
for metric in self.items:
for reference in [None] + references:
key = reference_key(metric, reference)
df_key = format_metric_key(key)
row[key] = _render_dimensional_metric_cell(row_data, metric) \
if isinstance(row_data.index, pd.MultiIndex) \
else _format_metric_cell(row_data[df_key], metric)
return row
def _display_metrics(self, metrics, operations):
display = OrderedDict()
axis = 0
for metrics_level in metrics:
for metric_key in utils.wrap_list(metrics_level):
schema = self.slicer.metrics[metric_key]
display[metric_key] = {
attr: getattr(schema, attr)
for attr in ['label', 'precision', 'prefix', 'suffix']
if getattr(schema, attr) is not None
}
display[metric_key]["axis"] = axis
axis += 1
for operation in operations:
if not hasattr(operation, 'metric_key'):
continue
metric_key = operation.metric_key
metric_schema = self.slicer.metrics[metric_key]
key = '{}_{}'.format(metric_key, operation.key)
display[key] = {
attr: getattr(metric_schema, attr)
for attr in ['precision', 'prefix', 'suffix']
if getattr(metric_schema, attr) is not None
}
display[key]['label'] = '{} {}'.format(metric_schema.label,
operation.label)
display[key]["axis"] = axis
axis += 1
return display
def sort_data_frame(self, data_frame):
if not self.sort or len(data_frame) == 1:
# If there are no sort arguments or the data frame is a single row, then no need to sort
return data_frame
# reset the index so all columns can be sorted together
index_names = data_frame.index.names
unsorted = data_frame.reset_index()
column_names = list(unsorted.columns)
ascending = self.ascending if self.ascending is not None else True
sort = wrap_list(self.sort)
sort_columns = [
column_names[column] for column in sort
if column < len(column_names)
]
if not sort_columns:
return data_frame
# ignore additional values for ascending if they do not align lengthwise with sort_columns
# Default to the first value or None
if isinstance(ascending, list) and len(ascending) != len(sort_columns):
ascending = ascending[0] if len(ascending) > 0 else None
sorted = unsorted.sort_values(
sort_columns, ascending=ascending).set_index(index_names)
# Maintain the single metric name
if hasattr(data_frame, "name"):
sorted.name = data_frame.name
return sorted
def transform_row_values(self, series, fields: Dict[str, Field],
is_transposed: bool, is_pivoted: bool,
hide_aliases: List[str]):
row = {}
row_colors = None
for key, value in series.items():
if key in hide_aliases:
continue
key = wrap_list(key)
# Get the field for the metric
metric_alias = wrap_list(
series.name)[0] if is_transposed else key[0]
field = fields[metric_alias]
data = {
RAW_VALUE: raw_value(value, field),
}
if not row_colors:
# No color for this field yet
rule = find_rule_to_apply(
self.formatting_rules_map[metric_alias], value)
if rule is not None:
colors = rule.determine_colors(value)
data["color"], data["text_color"] = colors
if not is_transposed and not is_pivoted and rule.covers_row:
# No transposing or pivoting going on so set as row color if it's specified for the rule
row_colors = colors
display = _display_value(value, field, date_as=return_none)
if display is not None:
data["display"] = display
accessor = self._get_row_value_accessor(series, fields, key)
setdeepattr(row, accessor, data)
# Assign the row colors to fields that aren't colored yet
if row_colors:
for key in series.keys():
accessor = self._get_row_value_accessor(
series, fields, wrap_list(key))
data = getdeepattr(row, accessor)
if "color" not in data:
data["color"], data["text_color"] = row_colors
return row, row_colors
def _render_timeseries_data(group_df, metric_key):
series = []
for dimension_values, y in group_df[metric_key].iteritems():
first_dimension_value = utils.wrap_list(dimension_values)[0]
if pd.isnull(first_dimension_value):
# Ignore totals on the x-axis.
continue
series.append((formats.date_as_millis(first_dimension_value),
formats.metric_value(y)))
return series
def calculate_min_max(self, df, is_transposed):
if not self.min_max_map:
return
for index, series in df.iterrows():
for key, value in series.items():
metric_alias = wrap_list(
series.name)[0] if is_transposed else wrap_list(key)[0]
min_max = self.min_max_map.get(metric_alias)
if min_max is not None: # we need to calculate the min and max values for this metric.
if value < min_max[0]:
min_max[
0] = value # value is smaller than stored smallest value, replace it.
if value > min_max[1]:
min_max[
1] = value # value is bigger than stored biggest value, replace it.
for rule_list in self.formatting_rules_map.values():
for rule in rule_list:
if isinstance(rule, FormattingHeatMapRule):
rule.set_min_max(
*self.min_max_map[rule.get_field_selector()])
def _get_timeseries_positions(df: pd.DataFrame, dimension_alias: str):
timeseries_positions = []
for dimensions, dimension_value in df[dimension_alias].iteritems():
datetime = utils.wrap_list(dimensions)[0]
timeseries_positions.append({
"position":
formats.date_as_millis(datetime),
"label":
dimension_value
})
return timeseries_positions
def transform_data(data_frame, field_map, dimension_hyperlink_templates,
is_transposed):
"""
Builds a list of dicts containing the data for ReactTable. This aligns with the accessors set by
#transform_dimension_column_headers and #transform_metric_column_headers
:param data_frame:
The result set data frame
:param field_map:
A mapping to all the fields in the dataset used for this query.
:param dimension_hyperlink_templates:
:param is_transposed:
"""
index_names = data_frame.index.names
def _get_field_label(alias):
if alias not in field_map:
return alias
field = field_map[alias]
return getattr(field, "label", field.alias)
# If the metric column was dropped due to only having a single metric, add it back here so the
# formatting can be applied.
if hasattr(data_frame, "name"):
metric_alias = data_frame.name
data_frame = pd.concat(
[data_frame],
keys=[metric_alias],
names=[F_METRICS_DIMENSION_ALIAS],
axis=1,
)
rows = []
for index, series in data_frame.iterrows():
index = wrap_list(index)
# Get a list of values from the index. These can be metrics or dimensions so it checks in the item map if
# there is a display value for the value
index_values = ([_get_field_label(value)
for value in index] if is_transposed else index)
index_display_values = OrderedDict(zip(index_names, index_values))
rows.append({
**ReactTable.transform_row_index(
index_display_values, field_map, dimension_hyperlink_templates),
**ReactTable.transform_row_values(series, field_map, is_transposed),
})
return rows
def _render_timeseries_data(group_df, metric_alias, metric):
series = []
for dimension_values, y in group_df[metric_alias].iteritems():
first_dimension_value = utils.wrap_list(dimension_values)[0]
# Ignore empty result sets where the only row is totals
if first_dimension_value in TOTALS_MARKERS:
continue
if pd.isnull(first_dimension_value):
# Ignore totals on the x-axis.
continue
series.append((
formats.date_as_millis(first_dimension_value),
formats.raw_value(y, metric),
))
return series
def _get_category_positions(df, dimension_alias, axis):
dimension_positions = []
category_positions = {
category: index
for index, category in enumerate(axis["categories"])
}
for dimensions, dimension_value in df[dimension_alias].iteritems():
category_label = utils.wrap_list(dimensions)[0]
dimension_positions.append({
"position":
category_positions[category_label],
"label":
dimension_value,
})
return dimension_positions
def transform_data_row_values(cls, series, item_map):
# Add the values to the row
row = {}
for key, value in series.iteritems():
key = wrap_list(key)
value = metric_value(value)
data = {RAW_VALUE: metric_value(value)}
# Try to find a display value for the item
item = item_map.get(key[0])
display = metric_display(value, getattr(item, 'prefix', None),
getattr(item, 'suffix', None),
getattr(item, 'precision', None))
if display is not None:
data['display'] = display
setdeepattr(row, key, data)
return row
def _render_pie_series(self, metric: Field, reference: Reference,
data_frame: pd.DataFrame,
dimension_fields: List[Field]) -> dict:
metric_alias = utils.alias_selector(metric.alias)
if self.split_dimension:
dimension_fields = [
dimension for dimension in dimension_fields
if dimension != self.split_dimension
]
data_frame = data_frame.reset_index(alias_selector(
self.split_dimension.alias),
drop=True)
data = []
for dimension_values, y in data_frame[metric_alias].iteritems():
dimension_values = utils.wrap_list(dimension_values)
name = self._format_dimension_values(dimension_fields,
dimension_values)
data.append({
"name": name or metric.label,
"y": formats.raw_value(y, metric)
})
return {
"name": reference_label(metric, reference),
"type": "pie",
"data": data,
"tooltip": {
"pointFormat":
'<span style="color:{point.color}">\u25CF</span> {series.name}: '
"<b>{point.y} ({point.percentage:.1f}%)</b><br/>",
"valueDecimals":
metric.precision,
"valuePrefix":
reference_prefix(metric, reference),
"valueSuffix":
reference_suffix(metric, reference),
},
}
def render_series_label(dimension_values, metric=None, reference=None):
"""
Returns a string label for a metric, reference, and set of values for zero or more dimensions.
:param metric:
an instance of fireant.Metric
:param reference:
an instance of fireant.Reference
:param dimension_values:
a tuple of dimension values. Can be zero-length or longer.
:return:
"""
# normalize the dimension values, as we expect them to be an iterable, in
# order to calculate the number of used dimension safely
dimension_values = utils.wrap_list(dimension_values)
num_used_dimensions = len(dimensions) - len(dimension_values)
used_dimensions = dimensions[num_used_dimensions:]
dimension_labels = [utils.getdeepattr(dimension_display_values,
(utils.format_dimension_key(dimension.key), dimension_value),
dimension_value)
if dimension_value not in TOTALS_MARKERS
else 'Totals'
for dimension, dimension_value in zip(used_dimensions, dimension_values)]
label = ", ".join([str(label) for label in dimension_labels])
if metric is None:
if reference is not None:
return '{} ({})'.format(label, reference.label)
return label
if dimension_labels:
return '{} ({})'.format(reference_label(metric, reference),
label)
return reference_label(metric, reference)
def _render_pie_series(self, metric, reference, data_frame, dimension_fields):
metric_alias = utils.alias_selector(metric.alias)
data = []
for dimension_values, y in data_frame[metric_alias].iteritems():
dimension_values = utils.wrap_list(dimension_values)
name = self._format_dimension_values(dimension_fields, dimension_values)
data.append(
{"name": name or metric.label, "y": formats.raw_value(y, metric),}
)
return {
"name": reference_label(metric, reference),
"type": "pie",
"data": data,
"tooltip": {
"pointFormat": '<span style="color:{point.color}">\u25CF</span> {series.name}: '
"<b>{point.y} ({point.percentage:.1f}%)</b><br/>",
"valueDecimals": metric.precision,
"valuePrefix": reference_prefix(metric, reference),
"valueSuffix": reference_suffix(metric, reference),
},
}
def _render_series(self, axis, axis_idx, axis_color, colors, series_data_frames, render_series_label,
references, is_timeseries=False):
"""
Renders the series configuration.
https://api.highcharts.com/highcharts/series
:param axis:
:param axis_idx:
:param axis_color:
:param colors:
:param series_data_frames:
:param render_series_label:
:param references:
:param is_timeseries:
:return:
"""
hc_series = []
for series in axis:
symbols = itertools.cycle(MARKER_SYMBOLS)
for (dimension_values, group_df), symbol in zip(series_data_frames, symbols):
if is_timeseries:
group_df = group_df.sort_index(level=0)
dimension_values = utils.wrap_list(dimension_values)
if isinstance(series, self.PieSeries):
# pie charts suck
for reference in [None] + references:
hc_series.append(self._render_pie_series(series,
reference,
group_df,
render_series_label))
continue
# With a single axis, use different colors for each series
# With multiple axes, use the same color for the entire axis and only change the dash style
series_color = next(colors)
for reference, dash_style in zip([None] + references, itertools.cycle(DASH_STYLES)):
metric_key = utils.format_metric_key(reference_key(series.metric, reference))
hc_series.append({
"type": series.type,
"name": render_series_label(dimension_values, series.metric, reference),
"data": (
self._render_timeseries_data(group_df, metric_key)
if is_timeseries
else self._render_category_data(group_df, metric_key)
),
"tooltip": self._render_tooltip(series.metric, reference),
"yAxis": ("{}_{}".format(axis_idx, reference.key)
if reference is not None and reference.delta
else str(axis_idx)),
"marker": ({"symbol": symbol, "fillColor": axis_color or series_color}
if isinstance(series, SERIES_NEEDING_MARKER)
else {}),
"stacking": series.stacking,
})
if isinstance(series, ContinuousAxisSeries):
# Set each series in a continuous series to a specific color
hc_series[-1]["color"] = series_color
hc_series[-1]["dashStyle"] = dash_style
return hc_series
def transform_data(
self,
data_frame: pd.DataFrame,
field_map: Dict[str, Field],
hide_aliases: List[str],
dimension_hyperlink_templates: Dict[str, str],
is_transposed: bool,
is_pivoted: bool,
) -> List[dict]:
"""
Builds a list of dicts containing the data for ReactTable. This aligns with the accessors set by
#transform_dimension_column_headers and #transform_metric_column_headers
:param data_frame:
The result set data frame.
:param field_map:
A mapping to all the fields in the dataset used for this query.
:param hide_aliases:
A set with hide dimension aliases.
:param dimension_hyperlink_templates:
A mapping to fields and its hyperlink dimension, if any.
:param is_transposed:
Whether the table is transposed or not.
:param is_pivoted:
Whether the table is pivoted or not.
"""
index_names = data_frame.index.names
def _get_field_label(alias):
if alias not in field_map:
return alias
field = field_map[alias]
return getattr(field, "label", field.alias)
# If the metric column was dropped due to only having a single metric, add it back here so the
# formatting can be applied.
if hasattr(data_frame, "name"):
metric_alias = data_frame.name
data_frame = pd.concat(
[data_frame],
keys=[metric_alias],
names=[F_METRICS_DIMENSION_ALIAS],
axis=1,
)
self.calculate_min_max(data_frame, is_transposed)
rows = []
for index, series in data_frame.iterrows():
row_values, row_colors = self.transform_row_values(
series, field_map, is_transposed, is_pivoted, hide_aliases)
index = wrap_list(index)
# Get a list of values from the index. These can be metrics or dimensions so it checks in the item map if
# there is a display value for the value
index_values = [_get_field_label(value)
for value in index] if is_transposed else index
index_display_values = OrderedDict(zip(index_names, index_values))
row_index = self.transform_row_index(
index_display_values, field_map, dimension_hyperlink_templates,
hide_aliases, row_colors)
rows.append({
**row_index,
**row_values,
})
return rows
def _render_series(
self,
axis,
axis_idx: int,
axis_color: str,
colors: List[str],
data_frame: pd.DataFrame,
series_data_frames,
dimensions: List[Field],
references: List[Reference],
is_timeseries: bool = False,
) -> List[dict]:
"""
Renders the series configuration.
https://api.highcharts.com/highcharts/series
:param axis:
:param axis_idx:
:param axis_color:
:param colors:
:param series_data_frames:
:param dimensions:
:param references:
:param is_timeseries:
:return:
"""
hc_series = []
for series in axis:
# Pie Charts, do not break data out into groups, render all data in one pie chart series
if isinstance(series, self.PieSeries):
# Pie charts do not group the data up by series so render them separately and then go to the next series
for reference in [None] + references:
hc_series.append(
self._render_pie_series(series.metric, reference,
data_frame, dimensions))
continue
# For other series types, create a highcharts series for each group (combination of dimension values)
symbols = itertools.cycle(MARKER_SYMBOLS)
for (dimension_values,
group_df), symbol in zip(series_data_frames, symbols):
dimension_values = utils.wrap_list(dimension_values)
dimension_label = self._format_dimension_values(
dimensions[1:], dimension_values)
hc_series += self._render_highcharts_series(
series,
group_df,
references,
dimension_label,
is_timeseries,
symbol,
axis_idx,
axis_color,
next(colors),
)
return hc_series
