def _handle_perf_results(
benchmark_enabled_map, benchmark_directory_map, configuration_name,
build_properties, service_account_file, extra_links):
"""
Upload perf results to the perf dashboard.
This method also upload the perf results to logdog and augment it to
|extra_links|.
Returns:
0 if this upload to perf dashboard succesfully, 1 otherwise.
"""
tmpfile_dir = tempfile.mkdtemp('resultscache')
try:
# Upload all eligible benchmarks to the perf dashboard
logdog_dict = {}
logdog_stream = None
logdog_label = 'Results Dashboard'
upload_fail = False
with oauth_api.with_access_token(service_account_file) as oauth_file:
for benchmark_name, directories in benchmark_directory_map.iteritems():
if not benchmark_enabled_map[benchmark_name]:
continue
# There are potentially multiple directores with results, re-write and
# merge them if necessary
results_filename = None
if len(directories) > 1:
merge_perf_dir = os.path.join(
os.path.abspath(tmpfile_dir), benchmark_name)
if not os.path.exists(merge_perf_dir):
os.makedirs(merge_perf_dir)
results_filename = os.path.join(
merge_perf_dir, 'merged_perf_results.json')
_merge_perf_results(results_filename, directories)
else:
# It was only written to one shard, use that shards data
results_filename = join(directories[0], 'perf_results.json')
print 'Uploading perf results from %s benchmark' % benchmark_name
upload_fail = _upload_and_write_perf_data_to_logfile(
benchmark_name, results_filename, configuration_name,
build_properties, oauth_file, tmpfile_dir, logdog_dict,
('.reference' in benchmark_name))
logdog_file_name = _generate_unique_logdog_filename('Results_Dashboard_')
logdog_stream = logdog_helper.text(logdog_file_name,
json.dumps(logdog_dict, sort_keys=True,
indent=4, separators=(',', ': ')),
content_type=JSON_CONTENT_TYPE)
if upload_fail:
logdog_label += ' Upload Failure'
extra_links[logdog_label] = logdog_stream
if upload_fail:
return 1
return 0
finally:
shutil.rmtree(tmpfile_dir)
def _process_perf_results(output_json, configuration_name,
service_account_file,
build_properties, task_output_dir):
"""Process one or more perf JSON results.
Consists of merging the json-test-format output and uploading the perf test
output (chartjson and histogram).
Each directory in the task_output_dir represents one benchmark
that was run. Within this directory, there is a subdirectory with the name
of the benchmark that was run. In that subdirectory, there is a
perftest-output.json file containing the performance results in histogram
or dashboard json format and an output.json file containing the json test
results for the benchmark.
"""
directory_list = [
f for f in listdir(task_output_dir)
if not isfile(join(task_output_dir, f))
]
benchmark_directory_list = []
for directory in directory_list:
benchmark_directory_list += [
join(task_output_dir, directory, f)
for f in listdir(join(task_output_dir, directory))
]
test_results_list = []
tmpfile_dir = tempfile.mkdtemp('resultscache')
try:
for directory in benchmark_directory_list:
if '.reference' in directory:
# We don't need to upload reference build data to the
# flakiness dashboard since we don't monitor the ref build
continue
with open(join(directory, 'test_results.json')) as json_data:
test_results_list.append(json.load(json_data))
_merge_json_output(output_json, test_results_list)
with oauth_api.with_access_token(service_account_file) as oauth_file:
for directory in benchmark_directory_list:
_upload_perf_results(join(directory, 'perf_results.json'),
directory, configuration_name, build_properties,
oauth_file, tmpfile_dir)
finally:
shutil.rmtree(tmpfile_dir)
return 0
def _handle_perf_results(benchmark_enabled_map, benchmark_directory_list,
configuration_name, build_properties,
service_account_file, extra_links):
"""
Upload perf results to the perf dashboard.
This method also upload the perf results to logdog and augment it to
|extra_links|.
Returns:
0 if this upload to perf dashboard succesfully, 1 otherwise.
"""
tmpfile_dir = tempfile.mkdtemp('resultscache')
try:
# Upload all eligible benchmarks to the perf dashboard
logdog_dict = {}
logdog_stream = None
logdog_label = 'Results Dashboard'
upload_fail = False
with oauth_api.with_access_token(service_account_file) as oauth_file:
for directory in benchmark_directory_list:
benchmark_name = _get_benchmark_name(directory)
if not benchmark_enabled_map[benchmark_name]:
continue
print 'Uploading perf results from %s benchmark' % benchmark_name
upload_fail = _upload_and_write_perf_data_to_logfile(
benchmark_name, directory, configuration_name,
build_properties, oauth_file, tmpfile_dir, logdog_dict,
('.reference' in benchmark_name))
logdog_file_name = _generate_unique_logdog_filename(
'Results_Dashboard_')
logdog_stream = logdog_helper.text(
logdog_file_name,
json.dumps(logdog_dict,
sort_keys=True,
indent=4,
separators=(',', ': ')))
if upload_fail:
logdog_label += ' Upload Failure'
extra_links[logdog_label] = logdog_stream
if upload_fail:
return 1
return 0
finally:
shutil.rmtree(tmpfile_dir)
def _upload_individual(benchmark_name, directories, configuration_name,
build_properties, output_json_file,
service_account_file):
tmpfile_dir = tempfile.mkdtemp('resultscache')
try:
upload_begin_time = time.time()
# There are potentially multiple directores with results, re-write and
# merge them if necessary
results_filename = None
if len(directories) > 1:
merge_perf_dir = os.path.join(os.path.abspath(tmpfile_dir),
benchmark_name)
if not os.path.exists(merge_perf_dir):
os.makedirs(merge_perf_dir)
results_filename = os.path.join(merge_perf_dir,
'merged_perf_results.json')
_merge_perf_results(benchmark_name, results_filename, directories)
else:
# It was only written to one shard, use that shards data
results_filename = join(directories[0], 'perf_results.json')
print 'Uploading perf results from %s benchmark' % benchmark_name
# We generate an oauth token for every benchmark upload in the event
# the token could time out, see crbug.com/854162
with oauth_api.with_access_token(
service_account_file,
("%s_tok" % benchmark_name)) as oauth_file:
with open(output_json_file, 'w') as oj:
upload_fail = _upload_perf_results(results_filename,
benchmark_name,
configuration_name,
build_properties,
oauth_file, tmpfile_dir, oj)
upload_end_time = time.time()
print_duration(('%s upload time' % (benchmark_name)),
upload_begin_time, upload_end_time)
return (benchmark_name, upload_fail)
finally:
shutil.rmtree(tmpfile_dir)
def _process_perf_results(output_json, configuration_name,
service_account_file, build_properties,
task_output_dir):
"""Process one or more perf JSON results.
Consists of merging the json-test-format output and uploading the perf test
output (chartjson and histogram).
Each directory in the task_output_dir represents one benchmark
that was run. Within this directory, there is a subdirectory with the name
of the benchmark that was run. In that subdirectory, there is a
perftest-output.json file containing the performance results in histogram
or dashboard json format and an output.json file containing the json test
results for the benchmark.
"""
directory_list = [
f for f in listdir(task_output_dir)
if not isfile(join(task_output_dir, f))
]
benchmark_directory_list = []
for directory in directory_list:
benchmark_directory_list += [
join(task_output_dir, directory, f)
for f in listdir(join(task_output_dir, directory))
]
# We need to keep track of disabled benchmarks so we don't try to
# upload the results.
test_results_list = []
tmpfile_dir = tempfile.mkdtemp('resultscache')
try:
with oauth_api.with_access_token(service_account_file) as oauth_file:
for directory in benchmark_directory_list:
# Obtain the test name we are running
benchmark_name = basename(directory)
disabled = False
with open(join(directory, 'test_results.json')) as json_data:
json_results = json.load(json_data)
if not json_results:
# Output is null meaning the test didn't produce any results.
# Want to output an error and continue loading the rest of the
# test results.
print 'No results produced for %s, skipping upload' % directory
continue
if json_results.get('version') == 3:
# Non-telemetry tests don't have written json results but
# if they are executing then they are enabled and will generate
# chartjson results.
if not bool(json_results.get('tests')):
disabled = True
if not '.reference' in directory:
# We don't need to upload reference build data to the
# flakiness dashboard since we don't monitor the ref build
test_results_list.append(json_results)
if disabled:
# We don't upload disabled benchmarks
print 'Benchmark %s disabled' % benchmark_name
continue
print 'Uploading perf results from %s benchmark' % benchmark_name
_upload_perf_results(join(directory, 'perf_results.json'),
benchmark_name, configuration_name,
build_properties, oauth_file, tmpfile_dir)
_merge_json_output(output_json, test_results_list)
finally:
shutil.rmtree(tmpfile_dir)
return 0
def _process_perf_results(output_json, configuration_name,
service_account_file, build_properties,
task_output_dir):
"""Process one or more perf JSON results.
Consists of merging the json-test-format output and uploading the perf test
output (chartjson and histogram).
Each directory in the task_output_dir represents one benchmark
that was run. Within this directory, there is a subdirectory with the name
of the benchmark that was run. In that subdirectory, there is a
perftest-output.json file containing the performance results in histogram
or dashboard json format and an output.json file containing the json test
results for the benchmark.
"""
directory_list = [
f for f in listdir(task_output_dir)
if not isfile(join(task_output_dir, f))
]
benchmark_directory_list = []
for directory in directory_list:
benchmark_directory_list += [
join(task_output_dir, directory, f)
for f in listdir(join(task_output_dir, directory))
]
# We need to keep track of disabled benchmarks so we don't try to
# upload the results.
test_results_list = []
tmpfile_dir = tempfile.mkdtemp('resultscache')
upload_failure = False
build_properties = json.loads(build_properties)
if not configuration_name:
# we are deprecating perf-id crbug.com/817823
configuration_name = build_properties['buildername']
try:
logdog_dict = {}
with oauth_api.with_access_token(service_account_file) as oauth_file:
for directory in benchmark_directory_list:
# Obtain the test name we are running
benchmark_name = basename(directory).replace(" benchmark", "")
is_ref = '.reference' in benchmark_name
disabled = False
with open(join(directory, 'test_results.json')) as json_data:
json_results = json.load(json_data)
if not json_results:
# Output is null meaning the test didn't produce any results.
# Want to output an error and continue loading the rest of the
# test results.
print 'No results produced for %s, skipping upload' % directory
continue
if json_results.get('version') == 3:
# Non-telemetry tests don't have written json results but
# if they are executing then they are enabled and will generate
# chartjson results.
if not bool(json_results.get('tests')):
disabled = True
if not is_ref:
# We don't need to upload reference build data to the
# flakiness dashboard since we don't monitor the ref build
test_results_list.append(json_results)
if disabled:
# We don't upload disabled benchmarks
print 'Benchmark %s disabled' % benchmark_name
continue
print 'Uploading perf results from %s benchmark' % benchmark_name
upload_fail = _upload_and_write_perf_data_to_logfile(
benchmark_name, directory, configuration_name,
build_properties, oauth_file, tmpfile_dir, logdog_dict,
is_ref)
upload_failure = upload_failure or upload_fail
logdog_label = 'Results Dashboard'
logdog_file_name = 'Results_Dashboard_' + str(uuid.uuid4())
if upload_failure:
logdog_label += ' Upload Failure'
_merge_json_output(
output_json, test_results_list,
logdog_helper.text(
logdog_file_name,
json.dumps(logdog_dict,
sort_keys=True,
indent=4,
separators=(',', ':'))), logdog_label)
finally:
shutil.rmtree(tmpfile_dir)
return upload_failure
def _process_perf_results(output_json, configuration_name,
service_account_file,
build_properties, task_output_dir,
smoke_test_mode):
"""Process one or more perf JSON results.
Consists of merging the json-test-format output and uploading the perf test
output (chartjson and histogram).
Each directory in the task_output_dir represents one benchmark
that was run. Within this directory, there is a subdirectory with the name
of the benchmark that was run. In that subdirectory, there is a
perftest-output.json file containing the performance results in histogram
or dashboard json format and an output.json file containing the json test
results for the benchmark.
"""
directory_list = [
f for f in listdir(task_output_dir)
if not isfile(join(task_output_dir, f))
]
benchmark_directory_list = []
for directory in directory_list:
benchmark_directory_list += [
join(task_output_dir, directory, f)
for f in listdir(join(task_output_dir, directory))
]
test_results_list = []
tmpfile_dir = tempfile.mkdtemp('resultscache')
upload_failure = False
build_properties = json.loads(build_properties)
if not configuration_name:
# we are deprecating perf-id crbug.com/817823
configuration_name = build_properties['buildername']
try:
# First obtain the list of json test results to merge
# and determine the status of each benchmark
benchmark_enabled_map = _handle_perf_json_test_results(
benchmark_directory_list, test_results_list)
# Upload all eligible benchmarks to the perf dashboard
logdog_dict = {}
logdog_stream = None
logdog_label = 'Results Dashboard'
if not smoke_test_mode:
with oauth_api.with_access_token(service_account_file) as oauth_file:
for directory in benchmark_directory_list:
benchmark_name = _get_benchmark_name(directory)
if not benchmark_enabled_map[benchmark_name]:
continue
print 'Uploading perf results from %s benchmark' % benchmark_name
upload_fail = _upload_and_write_perf_data_to_logfile(
benchmark_name, directory, configuration_name, build_properties,
oauth_file, tmpfile_dir, logdog_dict,
('.reference' in benchmark_name))
upload_failure = upload_failure or upload_fail
logdog_file_name = 'Results_Dashboard_' + str(uuid.uuid4())
logdog_stream = logdog_helper.text(logdog_file_name,
json.dumps(logdog_dict, sort_keys=True,
indent=4, separators=(',', ':')))
if upload_failure:
logdog_label += ' Upload Failure'
# Finally, merge all test results json and write out to output location
_merge_json_output(output_json, test_results_list,
logdog_stream, logdog_label)
finally:
shutil.rmtree(tmpfile_dir)
return upload_failure