def test_compare_metrics_reports_no_difference(self): data_points = mcu.get_data_points_from_metrics_reports( [_REPORT_3, _REPORT_3, _REPORT_3]) metrics_difference_report = mcu.compare_metrics( data_points, _REPORT_3, config={'base_expression': 'v == v_mean'}) # The latest metrics match the previous ones exactly, so the difference # report should be empty. self.assertEqual(metrics_difference_report, '')
def test_get_data_points_from_metrics_reports(self): correct_dict = { 'InboundData__TotalSamples': [1728.0, 73216.0, 73216.0], 'InboundData__Accumulator_mb': [10000.0, 64750000.0, 64750.0], 'InboundData__Percentile_1_mb': [0.393, 0.393, 0.393], 'InboundData__Percentile_5_mb': [0.393, 0.393, 0.393], 'InboundData__Percentile_10_mb': [0.786, 0.786, 0.786], 'InboundData__Percentile_20_mb': [1.54, 1.54, 1.54], 'InboundData__Percentile_50_mb': [1.54, 1.54, 1.54], 'InboundData__Percentile_80_mb': [1.54, 1.54, 1.54], 'InboundData__Percentile_90_mb': [1.54, 1.54, 1.54], 'InboundData__Percentile_95_mb': [1.54, 1.54, 1.54], 'InboundData__Percentile_99_mb': [1.54, 1.54, 1.54], 'TransferToServerTime__TotalSamples': [2616.0, 247016.0, 247016.0], 'TransferToServerTime__Accumulator_sec': [89.615, 407467.495546299, 1.0], 'TransferToServerTime__Percentile_1_sec': [300.003, 300.003, 300.003], 'TransferToServerTime__Percentile_5_sec': [300.004, 300.004, 300.004], 'TransferToServerTime__Percentile_10_sec': [300.01, 300.01, 300.01], 'TransferToServerTime__Percentile_20_sec': [300.015, 300.015, 300.015], 'TransferToServerTime__Percentile_50_sec': [300.026, 300.026, 300.026], 'TransferToServerTime__Percentile_80_sec': [300.035, 300.035, 300.035], 'TransferToServerTime__Percentile_90_sec': [300.082, 300.082, 300.082], 'TransferToServerTime__Percentile_95_sec': [300.108, 300.108, 300.108], 'TransferToServerTime__Percentile_99_sec': [300.129, 300.129, 300.129], 'UniqueMetric__TotalSamples': [None, None, 9000.0], 'UniqueMetric__Accumulator': [None, None, 9000.0], 'UniqueMetric__Percentile_1': [None, None, 8902.0], 'UniqueMetric__Percentile_5': [None, None, 89010.0], 'UniqueMetric__Percentile_10': [None, None, 8920.0], 'UniqueMetric__Percentile_20': [None, None, 8940.0], 'UniqueMetric__Percentile_50': [None, None, 9000.0], 'UniqueMetric__Percentile_80': [None, None, 9060.0], 'UniqueMetric__Percentile_90': [None, None, 9080.0], 'UniqueMetric__Percentile_95': [None, None, 9090.0], 'UniqueMetric__Percentile_99': [None, None, 9098.0], 'CachedSyncTensors__Value': [11336, 1022168, 1022168], 'CreateCompileHandles__Value': [40, 40, 40], 'CreateDataHandles__Value': [407992, 576462152, 576462152], 'UniqueCounter__Value': [None, None, 9000] } self.assertTrue( self._dict_almost_equal( mcu.get_data_points_from_metrics_reports( [_REPORT_1, _REPORT_2, _REPORT_3]), correct_dict))
def test_compare_metrics_reports_value_difference_tolerance_loose(self): data_points = mcu.get_data_points_from_metrics_reports( [_REPORT_3, _REPORT_3, _REPORT_3_SLIGHTLY_DIFFERENT_VALUES]) metrics_difference_report = mcu.compare_metrics( data_points, _REPORT_3_SLIGHTLY_DIFFERENT_VALUES, config={'base_expression': 'v <= v_mean + (v_stddev * 2.0)'}) # Since the tolerance is 2.0, the small differences in values are not # big enough to trigger lines in the difference report. self.assertEqual(metrics_difference_report, '')
def test_compare_metrics_reports_new_counters(self): data_points = mcu.get_data_points_from_metrics_reports( [_REPORT_3, _REPORT_3, _REPORT_3_SLIGHTLY_DIFFERENT_VALUES]) metrics_difference_report = mcu.compare_metrics( data_points, _REPORT_3_WITH_NEW_COUNTERS, config={'base_expression': 'v <= v_mean + (v_stddev * 2.0)'}) # Since the tolerance is 2.0, the small differences in values are not # big enough to trigger lines in the difference report. expected_report = 'Found new aten counter: aten::_local_scalar_dense__Value: 73216\n' self.assertEqual(metrics_difference_report, expected_report)
def test_compare_metrics_reports_value_difference_tolerance_strict(self): data_points = mcu.get_data_points_from_metrics_reports( [_REPORT_3, _REPORT_3, _REPORT_3_SLIGHTLY_DIFFERENT_VALUES]) metrics_difference_report = mcu.compare_metrics( data_points, _REPORT_3_SLIGHTLY_DIFFERENT_VALUES, config={'base_expression': 'v == v_mean'}) # Since the tolerance is 0.0, even a tiny difference leads to a line in # the metrics_difference_report. expected_report = 'InboundData__Accumulator_mb failed its expression check. Expression: v == v_mean. Mean: 68083.33333333333. Stddev: 4714.045207910317. Actual Value: 74750.0\nInboundData__TotalSamples failed its expression check. Expression: v == v_mean. Mean: 72144.0. Stddev: 1516.0369388639579. Actual Value: 70000.0\nUniqueCounter__Value failed its expression check. Expression: v == v_mean. Mean: 9333.0. Stddev: 470.93311627024065. Actual Value: 9999\n' self.assertEqual(metrics_difference_report, expected_report)
def test_parse_real_metrics(self): print( "Testing against TPU. If this hangs, check that $XRT_TPU_CONFIG is set" ) x = torch.rand(3, 5, device=xm.xla_device()) x = torch.flatten(x, 1) x = torch.roll(x, 1, 0) x = torch.flip(x, [0, 1]) self.assertEqual(x.device.type, 'xla') metrics = met.metrics_report() self.assertTrue(metrics) data_points = mcu.get_data_points_from_metrics_reports([metrics]) self.assertIn('CompileTime__Percentile_99_sec', data_points.keys()) self.assertIn('CompileTime__TotalSamples', data_points.keys())
def test_compare_metrics_reports_value_difference_tolerance_custom(self): data_points = mcu.get_data_points_from_metrics_reports( [_REPORT_3, _REPORT_3, _REPORT_3_SLIGHTLY_DIFFERENT_VALUES]) metrics_difference_report = mcu.compare_metrics( data_points, _REPORT_3_SLIGHTLY_DIFFERENT_VALUES, config={ 'base_expression': 'True', 'InboundData__Accumulator_mb_expression': 'v < v_mean', 'UniqueCounter__Value_expression': 'v < v_mean', }) # 2 of 3 differing values have custom tolerances and therefore should pass. # The third uses the default tolerance of 0.0, so it will generate a line. expected_report = 'InboundData__Accumulator_mb failed its expression check. Expression: v < v_mean. Mean: 68083.33333333333. Stddev: 4714.045207910317. Actual Value: 74750.0\nUniqueCounter__Value failed its expression check. Expression: v < v_mean. Mean: 9333.0. Stddev: 470.93311627024065. Actual Value: 9999\n' self.assertEqual(metrics_difference_report, expected_report)
config.update(c) # Collect historical metrics for this test and check for any regressions in # the current run vs. the averages from previous runs. metrics_storage_dir = os.path.join( FLAGS.root, FLAGS.test_folder_name, _METRICS_HISTORY_DIR_NAME) metrics_storage_dir += '/' regression_test_config = config.get('regression_test_config', None) if regression_test_config: metrics_file_pattern = re.compile(_METRICS_FILE_PATTERN) prev_metrics_files = [f for f in gcsfs.list( metrics_storage_dir) if metrics_file_pattern.match(f.path)] prev_metrics_strings = [gcsfs.open( os.path.join(FLAGS.root, f.path), mode='rt').read() for f in prev_metrics_files] data_points = mcu.get_data_points_from_metrics_reports( prev_metrics_strings) regression_report = mcu.compare_metrics( data_points, metrics, config=regression_test_config) else: print('Unable to check for metrics regressions. Config should contain ' '"regression_test_config" key -- see example at the top of ' 'metrics_test_wrapper.py.', file=sys.stderr) regression_report = '' # Write the metrics from the current run to disk unless disabled by config. if config.get('write_metrics_to_disk', True): # Include the params for this invocation when saving metrics. output_string = '{}\n\n{}'.format(FLAGS, metrics) output_filename = os.path.join( metrics_storage_dir, datetime.datetime.utcnow().strftime('%Y_%m_%d')) _write_to_disk(output_string, output_filename)