def RowEntityToRowDict(entity):
entities_read.inc()
try:
d = {
'revision': entity.key.id,
'value': FloatHack(entity['value']),
'std_error': FloatHack(entity.get('error')),
'timestamp': entity['timestamp'].isoformat(),
'test': entity.key.parent.name,
}
# Add the expando properties as a JSON-encoded dict.
properties = {}
for key, value in entity.items():
if key in d or key in ['parent_test', 'error']:
# skip properties with dedicated columns.
continue
if isinstance(value, float):
value = FloatHack(value)
properties[key] = value
d['properties'] = json.dumps(properties) if properties else None
# Add columns derived from test: master, bot.
test_path_parts = d['test'].split('/', 2)
if len(test_path_parts) >= 3:
d['master'] = test_path_parts[0]
d['bot'] = test_path_parts[1]
return [d]
except KeyError:
logging.getLogger().exception('Failed to convert Row')
failed_entity_transforms.inc()
return []
def FlattenElement(elem):
d = dict(fixed_cols_provider.get().as_dict())
d.update(elem[0].as_dict())
d.update(elem[1].as_dict())
# TODO: try temp_file_format=AVRO on the BQ writer instead of needing
# FloatHack?
d['variance'] = FloatHack(d['variance'])
d['cv'] = FloatHack(d['cv'])
d['std_dev'] = FloatHack(d['std_dev'])
d['std_err'] = FloatHack(d['std_err'])
return d
def AnomalyEntityToRowDict(entity):
entities_read.inc()
try:
# We do the iso conversion of the nullable timestamps in isolation.
earliest_input_timestamp = entity.get('earliest_input_timestamp')
if earliest_input_timestamp:
earliest_input_timestamp = earliest_input_timestamp.isoformat()
latest_input_timestamp = entity.get('latest_input_timestamp')
if latest_input_timestamp:
latest_input_timestamp = latest_input_timestamp.isoformat()
d = {
'id': entity.key.id,
# TODO: 'sheriff'
# 'subscriptions' omitted; subscription_names is sufficient
'subscription_names': entity.get('subscription_names', []),
'test': TestPath(entity['test']),
'start_revision': entity['start_revision'],
'end_revision': entity['end_revision'],
'display_start': entity.get('display_start'),
'display_end': entity.get('display_end'),
# TODO: 'ownership'
'statistic': entity['statistic'],
'bug_id': entity['bug_id'],
'internal_only': entity['internal_only'],
'timestamp': entity['timestamp'].isoformat(),
'segment_size_before': entity.get('segment_size_before'),
'segment_size_after': entity.get('segment_size_after'),
'median_before_anomaly': entity.get('median_before_anomaly'),
'median_after_anomaly': entity.get('median_after_anomaly'),
'std_dev_before_anomaly': entity.get('std_dev_before_anomaly'),
'window_end_revision': entity.get('window_end_revision'),
't_statistic': FloatHack(entity.get('t_statistic')),
'degrees_of_freedom': entity.get('degrees_of_freedom'),
'p_value': entity.get('p_value'),
'is_improvement': entity.get('is_improvement', False),
'recovered': entity.get('recovered', False),
# TODO: 'ref_test'
'units': entity.get('units'),
# TODO: 'recipe_bisects'
'pinpoint_bisects': entity.get('pinpoint_bisects', []),
# These are critical to "time-to-culprit" calculations.
'earliest_input_timestamp': earliest_input_timestamp,
'latest_input_timestamp': latest_input_timestamp,
}
if d['statistic'] is None:
# Some years-old anomalies lack this.
raise UnconvertibleAnomalyError()
return [d]
except (KeyError, UnconvertibleAnomalyError):
failed_entity_transforms.inc()
return []