def _dump_chart_json(output_dir, chartjson):
"""Writes chart histogram to JSON files.
Output files:
results-chart.json contains the chart JSON.
perf_results.json contains histogram JSON for Catapult.
Args:
output_dir: Directory to place the JSON files.
chartjson: Source JSON data for output files.
"""
results_path = os.path.join(output_dir, 'results-chart.json')
logging.critical('Dumping chartjson to %s', results_path)
with open(results_path, 'w') as json_file:
json.dump(chartjson, json_file, indent=2)
# We would ideally generate a histogram set directly instead of generating
# chartjson then converting. However, perf_tests_results_helper is in
# //build, which doesn't seem to have any precedent for depending on
# anything in Catapult. This can probably be fixed, but since this doesn't
# need to be super fast or anything, converting is a good enough solution
# for the time being.
histogram_result = convert_chart_json.ConvertChartJson(results_path)
if histogram_result.returncode != 0:
raise Exception('chartjson conversion failed with error: ' +
histogram_result.stdout)
histogram_path = os.path.join(output_dir, 'perf_results.json')
logging.critical('Dumping histograms to %s', histogram_path)
with open(histogram_path, 'w') as json_file:
json_file.write(histogram_result.stdout)
def _DumpChartJson(args, chartjson):
if args.output_file == '-':
json_file = sys.stdout
elif args.output_file:
json_file = open(args.output_file, 'w')
else:
results_path = os.path.join(args.output_dir, 'results-chart.json')
logging.critical('Dumping chartjson to %s', results_path)
json_file = open(results_path, 'w')
json.dump(chartjson, json_file, indent=2)
if json_file is not sys.stdout:
json_file.close()
# We would ideally generate a histogram set directly instead of generating
# chartjson then converting. However, perf_tests_results_helper is in
# //build, which doesn't seem to have any precedent for depending on
# anything in Catapult. This can probably be fixed, but since this doesn't
# need to be super fast or anything, converting is a good enough solution
# for the time being.
if args.output_format == 'histograms':
histogram_result = convert_chart_json.ConvertChartJson(results_path)
if histogram_result.returncode != 0:
raise Exception('chartjson conversion failed with error: ' +
histogram_result.stdout)
histogram_path = os.path.join(args.output_dir, 'perf_results.json')
logging.critical('Dumping histograms to %s', histogram_path)
with open(histogram_path, 'w') as json_file:
json_file.write(histogram_result.stdout)
def PopulateHistogramSet(self, benchmark_metadata):
if len(self._histograms):
return
chart_json = chart_json_output_formatter.ResultsAsChartDict(
benchmark_metadata, self.all_page_specific_values,
self.all_summary_values)
info = self.telemetry_info
chart_json['label'] = info.label
chart_json['benchmarkStartMs'] = info.benchmark_start_epoch * 1000.0
file_descriptor, chart_json_path = tempfile.mkstemp()
os.close(file_descriptor)
json.dump(chart_json, file(chart_json_path, 'w'))
vinn_result = convert_chart_json.ConvertChartJson(chart_json_path)
os.remove(chart_json_path)
if vinn_result.returncode != 0:
logging.error('Error converting chart json to Histograms:\n' +
vinn_result.stdout)
return []
self._histograms.ImportDicts(json.loads(vinn_result.stdout))
self._histograms.ResolveRelatedHistograms()
def main():
args, unknown_args = _CreateArgparser().parse_known_args()
# TODO(bsheedy): Remove this once all uses of --chartjson are removed.
if args.chartjson:
args.output_format = 'chartjson'
chartjson = _BASE_CHART.copy() if args.output_format else None
with build_utils.TempDir() as base_dir, build_utils.TempDir() as diff_dir:
# Run resource_sizes.py on the two APKs
resource_sizes_path = os.path.join(_ANDROID_DIR, 'resource_sizes.py')
shared_args = (['python', resource_sizes_path, '--output-format=chartjson']
+ unknown_args)
base_args = shared_args + ['--output-dir', base_dir, args.base_apk]
if args.out_dir_base:
base_args += ['--chromium-output-directory', args.out_dir_base]
try:
subprocess.check_output(base_args, stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as e:
print e.output
raise
diff_args = shared_args + ['--output-dir', diff_dir, args.diff_apk]
if args.out_dir_diff:
diff_args += ['--chromium-output-directory', args.out_dir_diff]
try:
subprocess.check_output(diff_args, stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as e:
print e.output
raise
# Combine the separate results
base_file = os.path.join(base_dir, _CHARTJSON_FILENAME)
diff_file = os.path.join(diff_dir, _CHARTJSON_FILENAME)
base_results = shared_preference_utils.ExtractSettingsFromJson(base_file)
diff_results = shared_preference_utils.ExtractSettingsFromJson(diff_file)
DiffResults(chartjson, base_results, diff_results)
if args.include_intermediate_results:
AddIntermediateResults(chartjson, base_results, diff_results)
if args.output_format:
chartjson_path = os.path.join(os.path.abspath(args.output_dir),
_CHARTJSON_FILENAME)
logging.critical('Dumping diff chartjson to %s', chartjson_path)
with open(chartjson_path, 'w') as outfile:
json.dump(chartjson, outfile)
if args.output_format == 'histograms':
histogram_result = convert_chart_json.ConvertChartJson(chartjson_path)
if histogram_result.returncode != 0:
logging.error('chartjson conversion failed with error: %s',
histogram_result.stdout)
return 1
histogram_path = os.path.join(os.path.abspath(args.output_dir),
'perf_results.json')
logging.critical('Dumping diff histograms to %s', histogram_path)
with open(histogram_path, 'w') as json_file:
json_file.write(histogram_result.stdout)
def PopulateHistogramSet(self):
if len(self._histograms):
return
# We ensure that html traces are serialized and uploaded if necessary
results_processor.SerializeAndUploadHtmlTraces(self)
chart_json = chart_json_output_formatter.ResultsAsChartDict(self)
chart_json['label'] = self.label
chart_json['benchmarkStartMs'] = self.benchmark_start_us / 1000.0
file_descriptor, chart_json_path = tempfile.mkstemp()
os.close(file_descriptor)
json.dump(chart_json, file(chart_json_path, 'w'))
vinn_result = convert_chart_json.ConvertChartJson(chart_json_path)
os.remove(chart_json_path)
if vinn_result.returncode != 0:
logging.error('Error converting chart json to Histograms:\n' +
vinn_result.stdout)
return []
self._histograms.ImportDicts(json.loads(vinn_result.stdout))
self._histograms.ImportDicts(self._histogram_dicts_to_add)
def ResourceSizes(args):
chartjson = _BASE_CHART.copy() if args.output_format else None
if args.input.endswith('.apk'):
_Analyze(args.input, chartjson, args)
elif args.input.endswith('.apks'):
with tempfile.NamedTemporaryFile(suffix='.apk') as f:
with zipfile.ZipFile(args.input) as z:
# Currently bundletool is creating two apks when .apks is created
# without specifying an sdkVersion. Always measure the one with an
# uncompressed shared library.
try:
info = z.getinfo('splits/base-master_2.apk')
except KeyError:
info = z.getinfo('splits/base-master.apk')
f.write(z.read(info))
f.flush()
_Analyze(f.name, chartjson, args)
else:
raise Exception('Unknown file type: ' + args.input)
if chartjson:
if args.output_file == '-':
json_file = sys.stdout
elif args.output_file:
json_file = open(args.output_file, 'w')
else:
results_path = os.path.join(args.output_dir, 'results-chart.json')
logging.critical('Dumping chartjson to %s', results_path)
json_file = open(results_path, 'w')
json.dump(chartjson, json_file, indent=2)
if json_file is not sys.stdout:
json_file.close()
# We would ideally generate a histogram set directly instead of generating
# chartjson then converting. However, perf_tests_results_helper is in
# //build, which doesn't seem to have any precedent for depending on
# anything in Catapult. This can probably be fixed, but since this doesn't
# need to be super fast or anything, converting is a good enough solution
# for the time being.
if args.output_format == 'histograms':
histogram_result = convert_chart_json.ConvertChartJson(
results_path)
if histogram_result.returncode != 0:
logging.error('chartjson conversion failed with error: %s',
histogram_result.stdout)
return 1
histogram_path = os.path.join(args.output_dir, 'perf_results.json')
logging.critical('Dumping histograms to %s', histogram_path)
with open(histogram_path, 'w') as json_file:
json_file.write(histogram_result.stdout)
return 0
def main():
parser = argparse.ArgumentParser(
description='Converts a chartjson file to HistogramSet JSON.',
add_help=False)
parser.add_argument('chartjson_path', help='chartjson file path (input).')
parser.add_argument('histograms_path',
help='HistogramSet JSON file path (output).')
parser.add_argument('-h',
'--help',
action='help',
help='Show this help message and exit.')
args = parser.parse_args()
result = convert_chart_json.ConvertChartJson(args.chartjson_path)
if result.returncode != 0:
sys.stderr.write(result.stdout)
else:
file(args.histograms_path, 'w').write(result.stdout)
return result.returncode
def _ConvertChartJson(self, page_test_results):
chart_json = chart_json_output_formatter.ResultsAsChartDict(
self._metadata, page_test_results.all_page_specific_values,
page_test_results.all_summary_values)
info = page_test_results.telemetry_info
chart_json['label'] = info.label
chart_json['benchmarkStartMs'] = info.benchmark_start_epoch * 1000.0
file_descriptor, chart_json_path = tempfile.mkstemp()
os.close(file_descriptor)
json.dump(chart_json, file(chart_json_path, 'w'))
vinn_result = convert_chart_json.ConvertChartJson(chart_json_path)
os.remove(chart_json_path)
if vinn_result.returncode != 0:
logging.error('Error converting chart json to Histograms:\n' +
vinn_result.stdout)
return []
return json.loads(vinn_result.stdout)
def PopulateHistogramSet(self):
if len(self._histograms):
return
chart_json = chart_json_output_formatter.ResultsAsChartDict(self)
info = self.telemetry_info
chart_json['label'] = info.label
chart_json['benchmarkStartMs'] = info.benchmark_start_us / 1000.0
file_descriptor, chart_json_path = tempfile.mkstemp()
os.close(file_descriptor)
json.dump(chart_json, file(chart_json_path, 'w'))
vinn_result = convert_chart_json.ConvertChartJson(chart_json_path)
os.remove(chart_json_path)
if vinn_result.returncode != 0:
logging.error('Error converting chart json to Histograms:\n' +
vinn_result.stdout)
return []
self._histograms.ImportDicts(json.loads(vinn_result.stdout))
self._histograms.ImportDicts(self._histogram_dicts_to_add)
def main():
if sys.platform in ('win32', 'cygwin'):
default_platform = 'win'
elif sys.platform.startswith('darwin'):
default_platform = 'mac'
elif sys.platform == 'linux2':
default_platform = 'linux'
else:
default_platform = None
main_map = {
'android' : main_android,
'android-webview' : main_android_webview,
'android-cronet' : main_android_cronet,
'linux' : main_linux,
'mac' : main_mac,
'win' : main_win,
}
platforms = sorted(main_map.keys())
option_parser = optparse.OptionParser()
option_parser.add_option('--target',
default='Release',
help='build target (Debug, Release) '
'[default: %default]')
option_parser.add_option('--target-dir', help='ignored')
option_parser.add_option('--build-dir', help='ignored')
option_parser.add_option('--platform',
default=default_platform,
help='specify platform (%s) [default: %%default]'
% ', '.join(platforms))
option_parser.add_option('--json', help='Path to JSON output file')
# This needs to be --output-dir (and not something like --output-directory) in
# order to work properly with the build-side runtest.py script that's
# currently used for dashboard uploading results from this script.
option_parser.add_option('--output-dir',
help='Directory to dump data in the HistogramSet '
'format')
options, args = option_parser.parse_args()
real_main = main_map.get(options.platform)
if not real_main:
if options.platform is None:
sys.stderr.write('Unsupported sys.platform %s.\n' % repr(sys.platform))
else:
sys.stderr.write('Unknown platform %s.\n' % repr(options.platform))
msg = 'Use the --platform= option to specify a supported platform:\n'
sys.stderr.write(msg + ' ' + ' '.join(platforms) + '\n')
return 2
results_collector = ResultsCollector()
rc = real_main(options, args, results_collector)
if options.json:
with open(options.json, 'w') as f:
json.dump(results_collector.results, f)
if options.output_dir:
histogram_path = os.path.join(options.output_dir, 'perf_results.json')
# We need to add a bit more data to the results and rearrange some things,
# otherwise the conversion fails due to the provided data being malformed.
updated_results = format_for_histograms_conversion(
results_collector.results)
with open(histogram_path, 'w') as f:
json.dump(updated_results, f)
histogram_result = convert_chart_json.ConvertChartJson(histogram_path)
if histogram_result.returncode != 0:
sys.stderr.write(
'chartjson conversion failed: %s\n' % histogram_result.stdout)
return histogram_result.returncode
with open(histogram_path, 'w') as f:
f.write(histogram_result.stdout)
return rc
def Run(self, finder_options):
"""We shouldn't be overriding this according to
telemetry.benchmark.Benchmark"""
assert 'histograms' in finder_options.output_formats, (
'loading.desktop.network_service requires --output-format=histograms.'
)
# feed the story_runner with 'chartjson' output formats.
# TODO(https://crbug.com/929765): Make loading.desktop.network_service
# benchmark produce histograms natively.
while 'histograms' in finder_options.output_formats:
finder_options.output_formats.remove('histograms')
finder_options.output_formats.append('chartjson')
assert finder_options.output_dir
output_dir = finder_options.output_dir
temp_file_path = os.path.join(output_dir, 'results-chart.json')
# Run test with feature disabled.
self.enable_feature = False
control_return_code = story_runner.RunBenchmark(self, finder_options)
if control_return_code != 0:
return control_return_code
control_chart_json = json.load(open(temp_file_path))
# Run test again with feature enabled.
self.enable_feature = True
enabled_return_code = story_runner.RunBenchmark(self, finder_options)
if enabled_return_code != 0:
return enabled_return_code
enabled_chart_json = json.load(open(temp_file_path))
logging.info(
'Starting to merge control chartjson into enabled chartjson')
try:
# Merge the result and compute the difference.
_MergeControlChartJsonIntoEnabled(enabled_chart_json,
control_chart_json)
except Exception as e:
logging.error('exception merging two chart json: %s', repr(e))
traceback.print_exc()
with open(temp_file_path, 'w') as f:
json.dump(
{
'control_chart_json': control_chart_json,
'enabled_chart_json': enabled_chart_json
},
f,
indent=2,
separators=(',', ': '))
f.write('\n')
return 1
else:
logging.info('Finished merging chartjsons, writing back to disk')
with open(temp_file_path, 'w') as f:
json.dump(enabled_chart_json,
f,
indent=2,
separators=(',', ': '))
f.write('\n')
logging.info('Converting chartjsons to histograms')
histogram_result = convert_chart_json.ConvertChartJson(
temp_file_path)
if histogram_result.returncode != 0:
logging.error('Error converting chart json to Histograms:\n' +
histogram_result.stdout)
return 1
temp_file_path = os.path.join(output_dir, 'histograms.json')
with open(temp_file_path, 'w') as f:
f.write(histogram_result.stdout)
return 0
def main():
if sys.platform in ('win32', 'cygwin'):
default_platform = 'win'
elif sys.platform.startswith('darwin'):
default_platform = 'mac'
elif sys.platform == 'linux2':
default_platform = 'linux'
else:
default_platform = None
main_map = {
'android-cronet': main_android_cronet,
'linux': main_linux,
'mac': main_mac,
'win': main_win,
}
platforms = sorted(main_map.keys())
parser = argparse.ArgumentParser()
parser.add_argument(
'--output-directory',
type=os.path.realpath,
help='Chromium output directory, e.g. /path/to/src/out/Debug')
parser.add_argument('--platform',
default=default_platform,
help='specify platform (%s) [default: %%(default)s]' %
', '.join(platforms))
parser.add_argument('--size-path',
default=None,
help='Path to size binary')
# Accepted to conform to the isolated script interface, but ignored.
parser.add_argument('--isolated-script-test-filter',
help=argparse.SUPPRESS)
parser.add_argument('--isolated-script-test-perf-output',
help=argparse.SUPPRESS)
parser.add_argument(
'--isolated-script-test-output',
type=os.path.realpath,
help='File to which simplified JSON results will be written.')
args = parser.parse_args()
real_main = main_map.get(args.platform)
if not real_main:
if args.platform is None:
sys.stderr.write('Unsupported sys.platform %s.\n' %
repr(sys.platform))
else:
sys.stderr.write('Unknown platform %s.\n' % repr(args.platform))
msg = 'Use the --platform= option to specify a supported platform:\n'
sys.stderr.write(msg + ' ' + ' '.join(platforms) + '\n')
return 2
isolated_script_output = {
'valid': False,
'failures': [],
'version': 'simplified'
}
test_name = 'sizes'
results_directory = None
if args.isolated_script_test_output:
results_directory = os.path.join(
os.path.dirname(args.isolated_script_test_output), test_name)
if not os.path.exists(results_directory):
os.makedirs(results_directory)
results_collector = ResultsCollector()
try:
rc = real_main(args.output_directory, results_collector,
args.size_path)
isolated_script_output = {
'valid': True,
'failures': [test_name] if rc else [],
'version': 'simplified',
}
finally:
if results_directory:
results_path = os.path.join(results_directory, 'test_results.json')
with open(results_path, 'w') as output_file:
json.dump(isolated_script_output, output_file)
histogram_path = os.path.join(results_directory,
'perf_results.json')
# We need to add a bit more data to the results and rearrange some things,
# otherwise the conversion fails due to the provided data being malformed.
updated_results = format_for_histograms_conversion(
results_collector.results)
with open(histogram_path, 'w') as f:
json.dump(updated_results, f)
histogram_result = convert_chart_json.ConvertChartJson(
histogram_path)
if histogram_result.returncode != 0:
sys.stderr.write('chartjson conversion failed: %s\n' %
histogram_result.stdout)
rc = rc or histogram_result.returncode
else:
with open(histogram_path, 'w') as f:
f.write(histogram_result.stdout)
return rc
def main():
argparser = argparse.ArgumentParser(description='Print APK size metrics.')
argparser.add_argument(
'--min-pak-resource-size',
type=int,
default=20 * 1024,
help='Minimum byte size of displayed pak resources.')
argparser.add_argument('--chromium-output-directory',
dest='out_dir',
help='Location of the build artifacts.')
argparser.add_argument('--chartjson',
action='store_true',
help='DEPRECATED. Use --output-format=chartjson '
'instead.')
argparser.add_argument('--output-format',
choices=['chartjson', 'histograms'],
help='Output the results to a file in the given '
'format instead of printing the results.')
argparser.add_argument('--output-dir',
default='.',
help='Directory to save chartjson to.')
argparser.add_argument('--loadable_module', help='Obsolete (ignored).')
argparser.add_argument(
'--estimate-patch-size',
action='store_true',
help='Include patch size estimates. Useful for perf '
'builders where a reference APK is available but adds '
'~3 mins to run time.')
argparser.add_argument(
'--reference-apk-builder',
default=apk_downloader.DEFAULT_BUILDER,
help='Builder name to use for reference APK for patch '
'size estimates.')
argparser.add_argument('--reference-apk-bucket',
default=apk_downloader.DEFAULT_BUCKET,
help='Storage bucket holding reference APKs.')
argparser.add_argument('input',
help='Path to .apk or .apks file to measure.')
args = argparser.parse_args()
# TODO(bsheedy): Remove this once uses of --chartjson have been removed.
if args.chartjson:
args.output_format = 'chartjson'
chartjson = _BASE_CHART.copy() if args.output_format else None
if args.input.endswith('.apk'):
_Analyze(args.input, chartjson, args)
elif args.input.endswith('.apks'):
with tempfile.NamedTemporaryFile(suffix='.apk') as f:
with zipfile.ZipFile(args.input) as z:
f.write(z.read('splits/base-master.apk'))
f.flush()
_Analyze(f.name, chartjson, args)
else:
raise Exception('Unknown file type: ' + args.input)
if chartjson:
results_path = os.path.join(args.output_dir, 'results-chart.json')
logging.critical('Dumping chartjson to %s', results_path)
with open(results_path, 'w') as json_file:
json.dump(chartjson, json_file)
# We would ideally generate a histogram set directly instead of generating
# chartjson then converting. However, perf_tests_results_helper is in
# //build, which doesn't seem to have any precedent for depending on
# anything in Catapult. This can probably be fixed, but since this doesn't
# need to be super fast or anything, converting is a good enough solution
# for the time being.
if args.output_format == 'histograms':
histogram_result = convert_chart_json.ConvertChartJson(
results_path)
if histogram_result.returncode != 0:
logging.error('chartjson conversion failed with error: %s',
histogram_result.stdout)
return 1
histogram_path = os.path.join(args.output_dir, 'perf_results.json')
logging.critical('Dumping histograms to %s', histogram_path)
with open(histogram_path, 'w') as json_file:
json_file.write(histogram_result.stdout)
def main():
argparser = argparse.ArgumentParser(description='Print APK size metrics.')
argparser.add_argument(
'--min-pak-resource-size',
type=int,
default=20 * 1024,
help='Minimum byte size of displayed pak resources.')
argparser.add_argument('--chromium-output-directory',
dest='out_dir',
help='Location of the build artifacts.')
argparser.add_argument('--chartjson',
action='store_true',
help='DEPRECATED. Use --output-format=chartjson '
'instead.')
argparser.add_argument('--output-format',
choices=['chartjson', 'histograms'],
help='Output the results to a file in the given '
'format instead of printing the results.')
argparser.add_argument('--output-dir',
default='.',
help='Directory to save chartjson to.')
argparser.add_argument(
'--dump-static-initializers',
action='store_true',
dest='dump_sis',
help='Run dump-static-initializers.py to get the list'
'of static initializers (slow).')
argparser.add_argument('--loadable_module',
action='append',
help='Use for libraries added via loadable_modules')
argparser.add_argument(
'--estimate-patch-size',
action='store_true',
help='Include patch size estimates. Useful for perf '
'builders where a reference APK is available but adds '
'~3 mins to run time.')
argparser.add_argument(
'--reference-apk-builder',
default=apk_downloader.DEFAULT_BUILDER,
help='Builder name to use for reference APK for patch '
'size estimates.')
argparser.add_argument('--reference-apk-bucket',
default=apk_downloader.DEFAULT_BUCKET,
help='Storage bucket holding reference APKs.')
argparser.add_argument('apk', help='APK file path.')
args = argparser.parse_args()
# TODO(bsheedy): Remove this once uses of --chartjson have been removed.
if args.chartjson:
args.output_format = 'chartjson'
chartjson = _BASE_CHART.copy() if args.output_format else None
out_dir, tool_prefix = _ConfigOutDirAndToolsPrefix(args.out_dir)
if args.dump_sis and not out_dir:
argparser.error(
'--dump-static-initializers requires --chromium-output-directory')
# Do not add any new metrics without also documenting them in:
# //docs/speed/binary_size/metrics.md.
PrintApkAnalysis(args.apk, tool_prefix, out_dir, chartjson=chartjson)
_PrintDexAnalysis(args.apk, chartjson=chartjson)
ignored_libs = args.loadable_module if args.loadable_module else []
si_count = AnalyzeStaticInitializers(args.apk, tool_prefix, args.dump_sis,
out_dir, ignored_libs)
perf_tests_results_helper.ReportPerfResult(chartjson,
'StaticInitializersCount',
'count', si_count, 'count')
if args.estimate_patch_size:
_PrintPatchSizeEstimate(args.apk,
args.reference_apk_builder,
args.reference_apk_bucket,
chartjson=chartjson)
if chartjson:
results_path = os.path.join(args.output_dir, 'results-chart.json')
logging.critical('Dumping chartjson to %s', results_path)
with open(results_path, 'w') as json_file:
json.dump(chartjson, json_file)
# We would ideally generate a histogram set directly instead of generating
# chartjson then converting. However, perf_tests_results_helper is in
# //build, which doesn't seem to have any precedent for depending on
# anything in Catapult. This can probably be fixed, but since this doesn't
# need to be super fast or anything, converting is a good enough solution
# for the time being.
if args.output_format == 'histograms':
histogram_result = convert_chart_json.ConvertChartJson(
results_path)
if histogram_result.returncode != 0:
logging.error('chartjson conversion failed with error: %s',
histogram_result.stdout)
return 1
histogram_path = os.path.join(args.output_dir, 'perf_results.json')
logging.critical('Dumping histograms to %s', histogram_path)
with open(histogram_path, 'w') as json_file:
json_file.write(histogram_result.stdout)
