def testNormalize_ResultMeanIsZeroAndStdDevIsOne(self): # When a data series is normalized, it is guaranteed that the result # should have a mean of 0.0 and a standard deviation and variance of 1.0. _, y_values = zip(*_QUITE_STEPPISH) normalized = find_step._Normalize(y_values) self.assertAlmostEqual(0.0, math_utils.Mean(normalized)) self.assertAlmostEqual(1.0, math_utils.StandardDeviation(normalized))
def _ExtractValueAndError(trace):
"""Returns the value and measure of error from a chartjson trace dict.
Args:
trace: A dict that has one "result" from a performance test, e.g. one
"value" in a Telemetry test, with the keys "trace_type", "value", etc.
Returns:
A pair (value, error) where |value| is a float and |error| is some measure
of variance used to show error bars; |error| could be None.
Raises:
BadRequestError: Data format was invalid.
"""
trace_type = trace.get('type')
if trace_type == 'scalar':
value = trace.get('value')
if value is None and trace.get('none_value_reason'):
return float('nan'), 0
try:
return float(value), 0
except:
raise BadRequestError('Expected scalar value, got: %r' % value)
if trace_type == 'list_of_scalar_values':
values = trace.get('values')
if not isinstance(values, list) and values is not None:
# Something else (such as a single scalar, or string) was given.
raise BadRequestError('Expected list of scalar values, got: %r' %
values)
if not values or None in values:
# None was included or values is None; this is not an error if there
# is a reason.
if trace.get('none_value_reason'):
return float('nan'), float('nan')
raise BadRequestError('Expected list of scalar values, got: %r' %
values)
if not all(_IsNumber(v) for v in values):
raise BadRequestError('Non-number found in values list: %r' %
values)
value = math_utils.Mean(values)
std = trace.get('std')
if std is not None:
error = std
else:
error = math_utils.StandardDeviation(values)
return value, error
if trace_type == 'histogram':
return _GeomMeanAndStdDevFromHistogram(trace)
raise BadRequestError('Invalid value type in chart object: %r' %
trace_type)
def _FlattenTrace(test_suite_name, chart_name, trace_name, trace,
is_ref=False, tracing_links=None, benchmark_description=''):
"""Takes a trace dict from dashboard JSON and readies it for display.
Traces can be either scalars or lists; if scalar we take the value directly;
if list we average the values and compute their standard deviation. We also
extract fields that are normally part of v0 row dicts that are uploaded
using add_point but are actually part of traces in the v1.0 format.
Args:
test_suite_name: The name of the test suite (benchmark).
chart_name: The name of the chart to which this trace belongs.
trace_name: The name of the passed trace.
trace: A trace dict extracted from a dashboard JSON chart.
is_ref: A boolean which indicates whether this trace comes from a
reference build.
tracing_links: A dictionary mapping trace names to about:tracing trace
urls in cloud storage
benchmark_description: A string documenting the benchmark suite to which
this trace belongs.
Returns:
A dict containing units, value, and error for this trace.
Raises:
BadRequestError: The data wasn't valid.
"""
if '@@' in chart_name:
tir_label, chart_name = chart_name.split('@@')
chart_name = chart_name + '/' + tir_label
trace_type = trace.get('type')
if trace_type == 'scalar':
value = trace.get('value')
if value is None:
if trace.get('none_value_reason'):
value = float('nan')
else:
# TODO(qyearsley): Add test coverage. See http://crbug.com/447432
raise BadRequestError('Expected scalar value, got: ' + value)
error = 0
elif trace_type == 'list_of_scalar_values':
values = trace.get('values')
if not values or None in values:
if trace.get('none_value_reason'):
value = float('nan')
error = float('nan')
else:
raise BadRequestError('Expected list of scalar values, got: ' + values)
else:
value = math_utils.Mean(values)
std = trace.get('std')
if std is not None:
error = std
else:
error = math_utils.StandardDeviation(values)
elif trace_type == 'histogram':
value, error = _GeomMeanAndStdDevFromHistogram(trace)
elif trace_type is not None:
raise BadRequestError('Invalid value type in chart object: ' + trace_type)
else:
raise BadRequestError('No trace type provided.')
# If there is a link to an about:tracing trace in cloud storage for this
# test trace_name, cache it.
tracing_uri = None
if (tracing_links and
trace_name in tracing_links and
'cloud_url' in tracing_links[trace_name]):
tracing_uri = tracing_links[trace_name]['cloud_url'].replace('\\/', '/')
trace_name = _EscapeName(trace_name)
if trace_name == 'summary':
subtest_name = chart_name
else:
subtest_name = chart_name + '/' + trace_name
name = test_suite_name + '/' + subtest_name
if trace_name == 'summary' and is_ref:
name += '/ref'
elif trace_name != 'summary' and is_ref:
name += '_ref'
row_dict = {
'test': name,
'value': value,
'error': error,
'units': trace['units'],
'tracing_uri': tracing_uri,
'benchmark_description': benchmark_description,
}
if 'improvement_direction' in trace:
improvement_direction_str = trace['improvement_direction']
if improvement_direction_str is None:
raise BadRequestError('improvement_direction must not be None')
row_dict['higher_is_better'] = _ImprovementDirectionToHigherIsBetter(
improvement_direction_str)
return row_dict
def testMean_ShortList_SameAsAlternativeImplementation(self):
alternate_mean = lambda xs: sum(xs) / float(len(xs))
self.assertEqual(
alternate_mean([-1, 0.12, 0.72, 3.3, 8, 32, 439]),
math_utils.Mean([-1, 0.12, 0.72, 3.3, 8, 32, 439]))
def testMean_ShortList(self): self.assertEqual(0.5, math_utils.Mean([-3, 0, 1, 4]))
def testMean_OneValue(self): self.assertEqual(3.0, math_utils.Mean([3]))
def testMean_EmptyInput_ReturnsNan(self): self.assertTrue(math.isnan(math_utils.Mean([])))
def _MakeSampleStats(sample):
"""Calculates relevant stats for a sample and makes a SampleStats object."""
return SampleStats(math_utils.Mean(sample), math_utils.Variance(sample),
len(sample))
