def _dump_chart_json(output_dir, chartjson): """Writes chart histogram to JSON files. Output files: results-chart.json contains the chart JSON. perf_results.json contains histogram JSON for Catapult. Args: output_dir: Directory to place the JSON files. chartjson: Source JSON data for output files. """ results_path = os.path.join(output_dir, 'results-chart.json') logging.critical('Dumping chartjson to %s', results_path) with open(results_path, 'w') as json_file: json.dump(chartjson, json_file, indent=2) # We would ideally generate a histogram set directly instead of generating # chartjson then converting. However, perf_tests_results_helper is in # //build, which doesn't seem to have any precedent for depending on # anything in Catapult. This can probably be fixed, but since this doesn't # need to be super fast or anything, converting is a good enough solution # for the time being. histogram_result = convert_chart_json.ConvertChartJson(results_path) if histogram_result.returncode != 0: raise Exception('chartjson conversion failed with error: ' + histogram_result.stdout) histogram_path = os.path.join(output_dir, 'perf_results.json') logging.critical('Dumping histograms to %s', histogram_path) with open(histogram_path, 'w') as json_file: json_file.write(histogram_result.stdout)
def _DumpChartJson(args, chartjson): if args.output_file == '-': json_file = sys.stdout elif args.output_file: json_file = open(args.output_file, 'w') else: results_path = os.path.join(args.output_dir, 'results-chart.json') logging.critical('Dumping chartjson to %s', results_path) json_file = open(results_path, 'w') json.dump(chartjson, json_file, indent=2) if json_file is not sys.stdout: json_file.close() # We would ideally generate a histogram set directly instead of generating # chartjson then converting. However, perf_tests_results_helper is in # //build, which doesn't seem to have any precedent for depending on # anything in Catapult. This can probably be fixed, but since this doesn't # need to be super fast or anything, converting is a good enough solution # for the time being. if args.output_format == 'histograms': histogram_result = convert_chart_json.ConvertChartJson(results_path) if histogram_result.returncode != 0: raise Exception('chartjson conversion failed with error: ' + histogram_result.stdout) histogram_path = os.path.join(args.output_dir, 'perf_results.json') logging.critical('Dumping histograms to %s', histogram_path) with open(histogram_path, 'w') as json_file: json_file.write(histogram_result.stdout)
def PopulateHistogramSet(self, benchmark_metadata): if len(self._histograms): return chart_json = chart_json_output_formatter.ResultsAsChartDict( benchmark_metadata, self.all_page_specific_values, self.all_summary_values) info = self.telemetry_info chart_json['label'] = info.label chart_json['benchmarkStartMs'] = info.benchmark_start_epoch * 1000.0 file_descriptor, chart_json_path = tempfile.mkstemp() os.close(file_descriptor) json.dump(chart_json, file(chart_json_path, 'w')) vinn_result = convert_chart_json.ConvertChartJson(chart_json_path) os.remove(chart_json_path) if vinn_result.returncode != 0: logging.error('Error converting chart json to Histograms:\n' + vinn_result.stdout) return [] self._histograms.ImportDicts(json.loads(vinn_result.stdout)) self._histograms.ResolveRelatedHistograms()
def main(): args, unknown_args = _CreateArgparser().parse_known_args() # TODO(bsheedy): Remove this once all uses of --chartjson are removed. if args.chartjson: args.output_format = 'chartjson' chartjson = _BASE_CHART.copy() if args.output_format else None with build_utils.TempDir() as base_dir, build_utils.TempDir() as diff_dir: # Run resource_sizes.py on the two APKs resource_sizes_path = os.path.join(_ANDROID_DIR, 'resource_sizes.py') shared_args = (['python', resource_sizes_path, '--output-format=chartjson'] + unknown_args) base_args = shared_args + ['--output-dir', base_dir, args.base_apk] if args.out_dir_base: base_args += ['--chromium-output-directory', args.out_dir_base] try: subprocess.check_output(base_args, stderr=subprocess.STDOUT) except subprocess.CalledProcessError as e: print e.output raise diff_args = shared_args + ['--output-dir', diff_dir, args.diff_apk] if args.out_dir_diff: diff_args += ['--chromium-output-directory', args.out_dir_diff] try: subprocess.check_output(diff_args, stderr=subprocess.STDOUT) except subprocess.CalledProcessError as e: print e.output raise # Combine the separate results base_file = os.path.join(base_dir, _CHARTJSON_FILENAME) diff_file = os.path.join(diff_dir, _CHARTJSON_FILENAME) base_results = shared_preference_utils.ExtractSettingsFromJson(base_file) diff_results = shared_preference_utils.ExtractSettingsFromJson(diff_file) DiffResults(chartjson, base_results, diff_results) if args.include_intermediate_results: AddIntermediateResults(chartjson, base_results, diff_results) if args.output_format: chartjson_path = os.path.join(os.path.abspath(args.output_dir), _CHARTJSON_FILENAME) logging.critical('Dumping diff chartjson to %s', chartjson_path) with open(chartjson_path, 'w') as outfile: json.dump(chartjson, outfile) if args.output_format == 'histograms': histogram_result = convert_chart_json.ConvertChartJson(chartjson_path) if histogram_result.returncode != 0: logging.error('chartjson conversion failed with error: %s', histogram_result.stdout) return 1 histogram_path = os.path.join(os.path.abspath(args.output_dir), 'perf_results.json') logging.critical('Dumping diff histograms to %s', histogram_path) with open(histogram_path, 'w') as json_file: json_file.write(histogram_result.stdout)
def PopulateHistogramSet(self): if len(self._histograms): return # We ensure that html traces are serialized and uploaded if necessary results_processor.SerializeAndUploadHtmlTraces(self) chart_json = chart_json_output_formatter.ResultsAsChartDict(self) chart_json['label'] = self.label chart_json['benchmarkStartMs'] = self.benchmark_start_us / 1000.0 file_descriptor, chart_json_path = tempfile.mkstemp() os.close(file_descriptor) json.dump(chart_json, file(chart_json_path, 'w')) vinn_result = convert_chart_json.ConvertChartJson(chart_json_path) os.remove(chart_json_path) if vinn_result.returncode != 0: logging.error('Error converting chart json to Histograms:\n' + vinn_result.stdout) return [] self._histograms.ImportDicts(json.loads(vinn_result.stdout)) self._histograms.ImportDicts(self._histogram_dicts_to_add)
def ResourceSizes(args): chartjson = _BASE_CHART.copy() if args.output_format else None if args.input.endswith('.apk'): _Analyze(args.input, chartjson, args) elif args.input.endswith('.apks'): with tempfile.NamedTemporaryFile(suffix='.apk') as f: with zipfile.ZipFile(args.input) as z: # Currently bundletool is creating two apks when .apks is created # without specifying an sdkVersion. Always measure the one with an # uncompressed shared library. try: info = z.getinfo('splits/base-master_2.apk') except KeyError: info = z.getinfo('splits/base-master.apk') f.write(z.read(info)) f.flush() _Analyze(f.name, chartjson, args) else: raise Exception('Unknown file type: ' + args.input) if chartjson: if args.output_file == '-': json_file = sys.stdout elif args.output_file: json_file = open(args.output_file, 'w') else: results_path = os.path.join(args.output_dir, 'results-chart.json') logging.critical('Dumping chartjson to %s', results_path) json_file = open(results_path, 'w') json.dump(chartjson, json_file, indent=2) if json_file is not sys.stdout: json_file.close() # We would ideally generate a histogram set directly instead of generating # chartjson then converting. However, perf_tests_results_helper is in # //build, which doesn't seem to have any precedent for depending on # anything in Catapult. This can probably be fixed, but since this doesn't # need to be super fast or anything, converting is a good enough solution # for the time being. if args.output_format == 'histograms': histogram_result = convert_chart_json.ConvertChartJson( results_path) if histogram_result.returncode != 0: logging.error('chartjson conversion failed with error: %s', histogram_result.stdout) return 1 histogram_path = os.path.join(args.output_dir, 'perf_results.json') logging.critical('Dumping histograms to %s', histogram_path) with open(histogram_path, 'w') as json_file: json_file.write(histogram_result.stdout) return 0
def main(): parser = argparse.ArgumentParser( description='Converts a chartjson file to HistogramSet JSON.', add_help=False) parser.add_argument('chartjson_path', help='chartjson file path (input).') parser.add_argument('histograms_path', help='HistogramSet JSON file path (output).') parser.add_argument('-h', '--help', action='help', help='Show this help message and exit.') args = parser.parse_args() result = convert_chart_json.ConvertChartJson(args.chartjson_path) if result.returncode != 0: sys.stderr.write(result.stdout) else: file(args.histograms_path, 'w').write(result.stdout) return result.returncode
def _ConvertChartJson(self, page_test_results): chart_json = chart_json_output_formatter.ResultsAsChartDict( self._metadata, page_test_results.all_page_specific_values, page_test_results.all_summary_values) info = page_test_results.telemetry_info chart_json['label'] = info.label chart_json['benchmarkStartMs'] = info.benchmark_start_epoch * 1000.0 file_descriptor, chart_json_path = tempfile.mkstemp() os.close(file_descriptor) json.dump(chart_json, file(chart_json_path, 'w')) vinn_result = convert_chart_json.ConvertChartJson(chart_json_path) os.remove(chart_json_path) if vinn_result.returncode != 0: logging.error('Error converting chart json to Histograms:\n' + vinn_result.stdout) return [] return json.loads(vinn_result.stdout)
def PopulateHistogramSet(self): if len(self._histograms): return chart_json = chart_json_output_formatter.ResultsAsChartDict(self) info = self.telemetry_info chart_json['label'] = info.label chart_json['benchmarkStartMs'] = info.benchmark_start_us / 1000.0 file_descriptor, chart_json_path = tempfile.mkstemp() os.close(file_descriptor) json.dump(chart_json, file(chart_json_path, 'w')) vinn_result = convert_chart_json.ConvertChartJson(chart_json_path) os.remove(chart_json_path) if vinn_result.returncode != 0: logging.error('Error converting chart json to Histograms:\n' + vinn_result.stdout) return [] self._histograms.ImportDicts(json.loads(vinn_result.stdout)) self._histograms.ImportDicts(self._histogram_dicts_to_add)
def main(): if sys.platform in ('win32', 'cygwin'): default_platform = 'win' elif sys.platform.startswith('darwin'): default_platform = 'mac' elif sys.platform == 'linux2': default_platform = 'linux' else: default_platform = None main_map = { 'android' : main_android, 'android-webview' : main_android_webview, 'android-cronet' : main_android_cronet, 'linux' : main_linux, 'mac' : main_mac, 'win' : main_win, } platforms = sorted(main_map.keys()) option_parser = optparse.OptionParser() option_parser.add_option('--target', default='Release', help='build target (Debug, Release) ' '[default: %default]') option_parser.add_option('--target-dir', help='ignored') option_parser.add_option('--build-dir', help='ignored') option_parser.add_option('--platform', default=default_platform, help='specify platform (%s) [default: %%default]' % ', '.join(platforms)) option_parser.add_option('--json', help='Path to JSON output file') # This needs to be --output-dir (and not something like --output-directory) in # order to work properly with the build-side runtest.py script that's # currently used for dashboard uploading results from this script. option_parser.add_option('--output-dir', help='Directory to dump data in the HistogramSet ' 'format') options, args = option_parser.parse_args() real_main = main_map.get(options.platform) if not real_main: if options.platform is None: sys.stderr.write('Unsupported sys.platform %s.\n' % repr(sys.platform)) else: sys.stderr.write('Unknown platform %s.\n' % repr(options.platform)) msg = 'Use the --platform= option to specify a supported platform:\n' sys.stderr.write(msg + ' ' + ' '.join(platforms) + '\n') return 2 results_collector = ResultsCollector() rc = real_main(options, args, results_collector) if options.json: with open(options.json, 'w') as f: json.dump(results_collector.results, f) if options.output_dir: histogram_path = os.path.join(options.output_dir, 'perf_results.json') # We need to add a bit more data to the results and rearrange some things, # otherwise the conversion fails due to the provided data being malformed. updated_results = format_for_histograms_conversion( results_collector.results) with open(histogram_path, 'w') as f: json.dump(updated_results, f) histogram_result = convert_chart_json.ConvertChartJson(histogram_path) if histogram_result.returncode != 0: sys.stderr.write( 'chartjson conversion failed: %s\n' % histogram_result.stdout) return histogram_result.returncode with open(histogram_path, 'w') as f: f.write(histogram_result.stdout) return rc
def Run(self, finder_options): """We shouldn't be overriding this according to telemetry.benchmark.Benchmark""" assert 'histograms' in finder_options.output_formats, ( 'loading.desktop.network_service requires --output-format=histograms.' ) # feed the story_runner with 'chartjson' output formats. # TODO(https://crbug.com/929765): Make loading.desktop.network_service # benchmark produce histograms natively. while 'histograms' in finder_options.output_formats: finder_options.output_formats.remove('histograms') finder_options.output_formats.append('chartjson') assert finder_options.output_dir output_dir = finder_options.output_dir temp_file_path = os.path.join(output_dir, 'results-chart.json') # Run test with feature disabled. self.enable_feature = False control_return_code = story_runner.RunBenchmark(self, finder_options) if control_return_code != 0: return control_return_code control_chart_json = json.load(open(temp_file_path)) # Run test again with feature enabled. self.enable_feature = True enabled_return_code = story_runner.RunBenchmark(self, finder_options) if enabled_return_code != 0: return enabled_return_code enabled_chart_json = json.load(open(temp_file_path)) logging.info( 'Starting to merge control chartjson into enabled chartjson') try: # Merge the result and compute the difference. _MergeControlChartJsonIntoEnabled(enabled_chart_json, control_chart_json) except Exception as e: logging.error('exception merging two chart json: %s', repr(e)) traceback.print_exc() with open(temp_file_path, 'w') as f: json.dump( { 'control_chart_json': control_chart_json, 'enabled_chart_json': enabled_chart_json }, f, indent=2, separators=(',', ': ')) f.write('\n') return 1 else: logging.info('Finished merging chartjsons, writing back to disk') with open(temp_file_path, 'w') as f: json.dump(enabled_chart_json, f, indent=2, separators=(',', ': ')) f.write('\n') logging.info('Converting chartjsons to histograms') histogram_result = convert_chart_json.ConvertChartJson( temp_file_path) if histogram_result.returncode != 0: logging.error('Error converting chart json to Histograms:\n' + histogram_result.stdout) return 1 temp_file_path = os.path.join(output_dir, 'histograms.json') with open(temp_file_path, 'w') as f: f.write(histogram_result.stdout) return 0
def main(): if sys.platform in ('win32', 'cygwin'): default_platform = 'win' elif sys.platform.startswith('darwin'): default_platform = 'mac' elif sys.platform == 'linux2': default_platform = 'linux' else: default_platform = None main_map = { 'android-cronet': main_android_cronet, 'linux': main_linux, 'mac': main_mac, 'win': main_win, } platforms = sorted(main_map.keys()) parser = argparse.ArgumentParser() parser.add_argument( '--output-directory', type=os.path.realpath, help='Chromium output directory, e.g. /path/to/src/out/Debug') parser.add_argument('--platform', default=default_platform, help='specify platform (%s) [default: %%(default)s]' % ', '.join(platforms)) parser.add_argument('--size-path', default=None, help='Path to size binary') # Accepted to conform to the isolated script interface, but ignored. parser.add_argument('--isolated-script-test-filter', help=argparse.SUPPRESS) parser.add_argument('--isolated-script-test-perf-output', help=argparse.SUPPRESS) parser.add_argument( '--isolated-script-test-output', type=os.path.realpath, help='File to which simplified JSON results will be written.') args = parser.parse_args() real_main = main_map.get(args.platform) if not real_main: if args.platform is None: sys.stderr.write('Unsupported sys.platform %s.\n' % repr(sys.platform)) else: sys.stderr.write('Unknown platform %s.\n' % repr(args.platform)) msg = 'Use the --platform= option to specify a supported platform:\n' sys.stderr.write(msg + ' ' + ' '.join(platforms) + '\n') return 2 isolated_script_output = { 'valid': False, 'failures': [], 'version': 'simplified' } test_name = 'sizes' results_directory = None if args.isolated_script_test_output: results_directory = os.path.join( os.path.dirname(args.isolated_script_test_output), test_name) if not os.path.exists(results_directory): os.makedirs(results_directory) results_collector = ResultsCollector() try: rc = real_main(args.output_directory, results_collector, args.size_path) isolated_script_output = { 'valid': True, 'failures': [test_name] if rc else [], 'version': 'simplified', } finally: if results_directory: results_path = os.path.join(results_directory, 'test_results.json') with open(results_path, 'w') as output_file: json.dump(isolated_script_output, output_file) histogram_path = os.path.join(results_directory, 'perf_results.json') # We need to add a bit more data to the results and rearrange some things, # otherwise the conversion fails due to the provided data being malformed. updated_results = format_for_histograms_conversion( results_collector.results) with open(histogram_path, 'w') as f: json.dump(updated_results, f) histogram_result = convert_chart_json.ConvertChartJson( histogram_path) if histogram_result.returncode != 0: sys.stderr.write('chartjson conversion failed: %s\n' % histogram_result.stdout) rc = rc or histogram_result.returncode else: with open(histogram_path, 'w') as f: f.write(histogram_result.stdout) return rc
def main(): argparser = argparse.ArgumentParser(description='Print APK size metrics.') argparser.add_argument( '--min-pak-resource-size', type=int, default=20 * 1024, help='Minimum byte size of displayed pak resources.') argparser.add_argument('--chromium-output-directory', dest='out_dir', help='Location of the build artifacts.') argparser.add_argument('--chartjson', action='store_true', help='DEPRECATED. Use --output-format=chartjson ' 'instead.') argparser.add_argument('--output-format', choices=['chartjson', 'histograms'], help='Output the results to a file in the given ' 'format instead of printing the results.') argparser.add_argument('--output-dir', default='.', help='Directory to save chartjson to.') argparser.add_argument('--loadable_module', help='Obsolete (ignored).') argparser.add_argument( '--estimate-patch-size', action='store_true', help='Include patch size estimates. Useful for perf ' 'builders where a reference APK is available but adds ' '~3 mins to run time.') argparser.add_argument( '--reference-apk-builder', default=apk_downloader.DEFAULT_BUILDER, help='Builder name to use for reference APK for patch ' 'size estimates.') argparser.add_argument('--reference-apk-bucket', default=apk_downloader.DEFAULT_BUCKET, help='Storage bucket holding reference APKs.') argparser.add_argument('input', help='Path to .apk or .apks file to measure.') args = argparser.parse_args() # TODO(bsheedy): Remove this once uses of --chartjson have been removed. if args.chartjson: args.output_format = 'chartjson' chartjson = _BASE_CHART.copy() if args.output_format else None if args.input.endswith('.apk'): _Analyze(args.input, chartjson, args) elif args.input.endswith('.apks'): with tempfile.NamedTemporaryFile(suffix='.apk') as f: with zipfile.ZipFile(args.input) as z: f.write(z.read('splits/base-master.apk')) f.flush() _Analyze(f.name, chartjson, args) else: raise Exception('Unknown file type: ' + args.input) if chartjson: results_path = os.path.join(args.output_dir, 'results-chart.json') logging.critical('Dumping chartjson to %s', results_path) with open(results_path, 'w') as json_file: json.dump(chartjson, json_file) # We would ideally generate a histogram set directly instead of generating # chartjson then converting. However, perf_tests_results_helper is in # //build, which doesn't seem to have any precedent for depending on # anything in Catapult. This can probably be fixed, but since this doesn't # need to be super fast or anything, converting is a good enough solution # for the time being. if args.output_format == 'histograms': histogram_result = convert_chart_json.ConvertChartJson( results_path) if histogram_result.returncode != 0: logging.error('chartjson conversion failed with error: %s', histogram_result.stdout) return 1 histogram_path = os.path.join(args.output_dir, 'perf_results.json') logging.critical('Dumping histograms to %s', histogram_path) with open(histogram_path, 'w') as json_file: json_file.write(histogram_result.stdout)
def main(): argparser = argparse.ArgumentParser(description='Print APK size metrics.') argparser.add_argument( '--min-pak-resource-size', type=int, default=20 * 1024, help='Minimum byte size of displayed pak resources.') argparser.add_argument('--chromium-output-directory', dest='out_dir', help='Location of the build artifacts.') argparser.add_argument('--chartjson', action='store_true', help='DEPRECATED. Use --output-format=chartjson ' 'instead.') argparser.add_argument('--output-format', choices=['chartjson', 'histograms'], help='Output the results to a file in the given ' 'format instead of printing the results.') argparser.add_argument('--output-dir', default='.', help='Directory to save chartjson to.') argparser.add_argument( '--dump-static-initializers', action='store_true', dest='dump_sis', help='Run dump-static-initializers.py to get the list' 'of static initializers (slow).') argparser.add_argument('--loadable_module', action='append', help='Use for libraries added via loadable_modules') argparser.add_argument( '--estimate-patch-size', action='store_true', help='Include patch size estimates. Useful for perf ' 'builders where a reference APK is available but adds ' '~3 mins to run time.') argparser.add_argument( '--reference-apk-builder', default=apk_downloader.DEFAULT_BUILDER, help='Builder name to use for reference APK for patch ' 'size estimates.') argparser.add_argument('--reference-apk-bucket', default=apk_downloader.DEFAULT_BUCKET, help='Storage bucket holding reference APKs.') argparser.add_argument('apk', help='APK file path.') args = argparser.parse_args() # TODO(bsheedy): Remove this once uses of --chartjson have been removed. if args.chartjson: args.output_format = 'chartjson' chartjson = _BASE_CHART.copy() if args.output_format else None out_dir, tool_prefix = _ConfigOutDirAndToolsPrefix(args.out_dir) if args.dump_sis and not out_dir: argparser.error( '--dump-static-initializers requires --chromium-output-directory') # Do not add any new metrics without also documenting them in: # //docs/speed/binary_size/metrics.md. PrintApkAnalysis(args.apk, tool_prefix, out_dir, chartjson=chartjson) _PrintDexAnalysis(args.apk, chartjson=chartjson) ignored_libs = args.loadable_module if args.loadable_module else [] si_count = AnalyzeStaticInitializers(args.apk, tool_prefix, args.dump_sis, out_dir, ignored_libs) perf_tests_results_helper.ReportPerfResult(chartjson, 'StaticInitializersCount', 'count', si_count, 'count') if args.estimate_patch_size: _PrintPatchSizeEstimate(args.apk, args.reference_apk_builder, args.reference_apk_bucket, chartjson=chartjson) if chartjson: results_path = os.path.join(args.output_dir, 'results-chart.json') logging.critical('Dumping chartjson to %s', results_path) with open(results_path, 'w') as json_file: json.dump(chartjson, json_file) # We would ideally generate a histogram set directly instead of generating # chartjson then converting. However, perf_tests_results_helper is in # //build, which doesn't seem to have any precedent for depending on # anything in Catapult. This can probably be fixed, but since this doesn't # need to be super fast or anything, converting is a good enough solution # for the time being. if args.output_format == 'histograms': histogram_result = convert_chart_json.ConvertChartJson( results_path) if histogram_result.returncode != 0: logging.error('chartjson conversion failed with error: %s', histogram_result.stdout) return 1 histogram_path = os.path.join(args.output_dir, 'perf_results.json') logging.critical('Dumping histograms to %s', histogram_path) with open(histogram_path, 'w') as json_file: json_file.write(histogram_result.stdout)