def _handle_perf_logs(benchmark_directory_map, extra_links):
""" Upload benchmark logs to logdog and add a page entry for them. """
begin_time = time.time()
benchmark_logs_links = collections.defaultdict(list)
for benchmark_name, directories in benchmark_directory_map.iteritems():
for directory in directories:
benchmark_log_file = os.path.join(directory, 'benchmark_log.txt')
if os.path.exists(benchmark_log_file):
with open(benchmark_log_file) as f:
uploaded_link = logdog_helper.text(
name=_generate_unique_logdog_filename(benchmark_name),
data=f.read())
benchmark_logs_links[benchmark_name].append(uploaded_link)
logdog_file_name = _generate_unique_logdog_filename('Benchmarks_Logs')
logdog_stream = logdog_helper.text(logdog_file_name,
json.dumps(benchmark_logs_links,
sort_keys=True,
indent=4,
separators=(',', ': ')),
content_type=JSON_CONTENT_TYPE)
extra_links['Benchmarks logs'] = logdog_stream
end_time = time.time()
print_duration('Generating perf log streams', begin_time, end_time)
def _handle_perf_results(
benchmark_enabled_map, benchmark_directory_map, configuration_name,
build_properties, service_account_file, extra_links):
"""
Upload perf results to the perf dashboard.
This method also upload the perf results to logdog and augment it to
|extra_links|.
Returns:
0 if this upload to perf dashboard succesfully, 1 otherwise.
"""
tmpfile_dir = tempfile.mkdtemp('resultscache')
try:
# Upload all eligible benchmarks to the perf dashboard
logdog_dict = {}
logdog_stream = None
logdog_label = 'Results Dashboard'
upload_fail = False
with oauth_api.with_access_token(service_account_file) as oauth_file:
for benchmark_name, directories in benchmark_directory_map.iteritems():
if not benchmark_enabled_map[benchmark_name]:
continue
# There are potentially multiple directores with results, re-write and
# merge them if necessary
results_filename = None
if len(directories) > 1:
merge_perf_dir = os.path.join(
os.path.abspath(tmpfile_dir), benchmark_name)
if not os.path.exists(merge_perf_dir):
os.makedirs(merge_perf_dir)
results_filename = os.path.join(
merge_perf_dir, 'merged_perf_results.json')
_merge_perf_results(results_filename, directories)
else:
# It was only written to one shard, use that shards data
results_filename = join(directories[0], 'perf_results.json')
print 'Uploading perf results from %s benchmark' % benchmark_name
upload_fail = _upload_and_write_perf_data_to_logfile(
benchmark_name, results_filename, configuration_name,
build_properties, oauth_file, tmpfile_dir, logdog_dict,
('.reference' in benchmark_name))
logdog_file_name = _generate_unique_logdog_filename('Results_Dashboard_')
logdog_stream = logdog_helper.text(logdog_file_name,
json.dumps(logdog_dict, sort_keys=True,
indent=4, separators=(',', ': ')),
content_type=JSON_CONTENT_TYPE)
if upload_fail:
logdog_label += ' Upload Failure'
extra_links[logdog_label] = logdog_stream
if upload_fail:
return 1
return 0
finally:
shutil.rmtree(tmpfile_dir)
def _handle_benchmarks_shard_map(benchmarks_shard_map_file, extra_links):
begin_time = time.time()
with open(benchmarks_shard_map_file) as f:
benchmarks_shard_data = f.read()
logdog_file_name = _generate_unique_logdog_filename('Benchmarks_Shard_Map')
logdog_stream = logdog_helper.text(
logdog_file_name, benchmarks_shard_data, content_type=JSON_CONTENT_TYPE)
extra_links['Benchmarks shard map'] = logdog_stream
end_time = time.time()
print_duration('Generating benchmark shard map stream', begin_time, end_time)
def _handle_benchmarks_shard_map(benchmarks_shard_map_file, extra_links):
begin_time = time.time()
with open(benchmarks_shard_map_file) as f:
benchmarks_shard_data = json.load(f)
logdog_file_name = _generate_unique_logdog_filename('Benchmarks_Shard_Map')
logdog_stream = logdog_helper.text(
logdog_file_name, json.dumps(benchmarks_shard_data, sort_keys=True,
indent=4, separators=(',', ': ')),
content_type=JSON_CONTENT_TYPE)
extra_links['Benchmarks shard map'] = logdog_stream
end_time = time.time()
print_duration('Generating benchmark shard map stream', begin_time, end_time)
def _handle_perf_logs(benchmark_directory_list, extra_links):
""" Upload benchmark logs to logdog and add a page entry for them. """
benchmark_logs_links = {}
for directory in benchmark_directory_list:
# Obtain the test name we are running
benchmark_name = _get_benchmark_name(directory)
with open(join(directory, 'benchmark_log.txt')) as f:
uploaded_link = logdog_helper.text(
name=_generate_unique_logdog_filename(benchmark_name),
data=f.read())
benchmark_logs_links[benchmark_name] = uploaded_link
logdog_file_name = _generate_unique_logdog_filename('Benchmarks_Logs')
logdog_stream = logdog_helper.text(
logdog_file_name,
json.dumps(benchmark_logs_links,
sort_keys=True,
indent=4,
separators=(',', ': ')))
extra_links['Benchmarks logs'] = logdog_stream
def _handle_perf_logs(benchmark_directory_map, extra_links):
""" Upload benchmark logs to logdog and add a page entry for them. """
benchmark_logs_links = {}
for benchmark_name, directories in benchmark_directory_map.iteritems():
for directory in directories:
with open(join(directory, 'benchmark_log.txt')) as f:
uploaded_link = logdog_helper.text(
name=_generate_unique_logdog_filename(benchmark_name),
data=f.read())
if benchmark_name in benchmark_logs_links.keys():
benchmark_logs_links[benchmark_name].append(uploaded_link)
else:
benchmark_logs_links[benchmark_name] = [uploaded_link]
logdog_file_name = _generate_unique_logdog_filename('Benchmarks_Logs')
logdog_stream = logdog_helper.text(
logdog_file_name, json.dumps(benchmark_logs_links, sort_keys=True,
indent=4, separators=(',', ': ')),
content_type=JSON_CONTENT_TYPE)
extra_links['Benchmarks logs'] = logdog_stream
def _handle_perf_results(benchmark_enabled_map, benchmark_directory_list,
configuration_name, build_properties,
service_account_file, extra_links):
"""
Upload perf results to the perf dashboard.
This method also upload the perf results to logdog and augment it to
|extra_links|.
Returns:
0 if this upload to perf dashboard succesfully, 1 otherwise.
"""
tmpfile_dir = tempfile.mkdtemp('resultscache')
try:
# Upload all eligible benchmarks to the perf dashboard
logdog_dict = {}
logdog_stream = None
logdog_label = 'Results Dashboard'
upload_fail = False
with oauth_api.with_access_token(service_account_file) as oauth_file:
for directory in benchmark_directory_list:
benchmark_name = _get_benchmark_name(directory)
if not benchmark_enabled_map[benchmark_name]:
continue
print 'Uploading perf results from %s benchmark' % benchmark_name
upload_fail = _upload_and_write_perf_data_to_logfile(
benchmark_name, directory, configuration_name,
build_properties, oauth_file, tmpfile_dir, logdog_dict,
('.reference' in benchmark_name))
logdog_file_name = _generate_unique_logdog_filename(
'Results_Dashboard_')
logdog_stream = logdog_helper.text(
logdog_file_name,
json.dumps(logdog_dict,
sort_keys=True,
indent=4,
separators=(',', ': ')))
if upload_fail:
logdog_label += ' Upload Failure'
extra_links[logdog_label] = logdog_stream
if upload_fail:
return 1
return 0
finally:
shutil.rmtree(tmpfile_dir)
def _RunTest(self, device, test):
extras = {}
flags = None
test_timeout_scale = None
if self._test_instance.coverage_directory:
coverage_basename = '%s.ec' % ('%s_group' %
test[0]['method'] if isinstance(
test, list) else test['method'])
extras['coverage'] = 'true'
coverage_directory = os.path.join(device.GetExternalStoragePath(),
'chrome', 'test', 'coverage')
coverage_device_file = os.path.join(coverage_directory,
coverage_basename)
extras['coverageFile'] = coverage_device_file
if isinstance(test, list):
if not self._test_instance.driver_apk:
raise Exception('driver_apk does not exist. '
'Please build it and try again.')
if any(t.get('is_junit4') for t in test):
raise Exception('driver apk does not support JUnit4 tests')
def name_and_timeout(t):
n = instrumentation_test_instance.GetTestName(t)
i = self._GetTimeoutFromAnnotations(t['annotations'], n)
return (n, i)
test_names, timeouts = zip(*(name_and_timeout(t) for t in test))
test_name = ','.join(test_names)
test_display_name = test_name
target = '%s/%s' % (self._test_instance.driver_package,
self._test_instance.driver_name)
extras.update(
self._test_instance.GetDriverEnvironmentVars(
test_list=test_names))
timeout = sum(timeouts)
else:
test_name = instrumentation_test_instance.GetTestName(test)
test_display_name = self._GetUniqueTestName(test)
if test['is_junit4']:
target = '%s/%s' % (self._test_instance.test_package,
self._test_instance.test_runner_junit4)
else:
target = '%s/%s' % (self._test_instance.test_package,
self._test_instance.test_runner)
extras['class'] = test_name
if 'flags' in test:
flags = test['flags']
timeout = self._GetTimeoutFromAnnotations(test['annotations'],
test_display_name)
test_timeout_scale = self._GetTimeoutScaleFromAnnotations(
test['annotations'])
if test_timeout_scale and test_timeout_scale != 1:
valgrind_tools.SetChromeTimeoutScale(
device,
test_timeout_scale * self._test_instance.timeout_scale)
logging.info('preparing to run %s: %s', test_display_name, test)
if flags:
self._CreateFlagChangerIfNeeded(device)
self._flag_changers[str(device)].PushFlags(add=flags.add,
remove=flags.remove)
try:
device.RunShellCommand(
['log', '-p', 'i', '-t', _TAG,
'START %s' % test_name],
check_return=True)
logcat_url = None
time_ms = lambda: int(time.time() * 1e3)
start_ms = time_ms()
stream_name = 'logcat_%s_%s_%s' % (test_name.replace(
'#', '.'), time.strftime('%Y%m%dT%H%M%S',
time.localtime()), device.serial)
with contextlib_ext.Optional(
logdog_logcat_monitor.LogdogLogcatMonitor(
device.adb, stream_name),
self._test_instance.should_save_logcat) as logmon:
with contextlib_ext.Optional(trace_event.trace(test_name),
self._env.trace_output):
output = device.StartInstrumentation(target,
raw=True,
extras=extras,
timeout=timeout,
retries=0)
if logmon:
logcat_url = logmon.GetLogcatURL()
finally:
device.RunShellCommand(
['log', '-p', 'i', '-t', _TAG,
'END %s' % test_name],
check_return=True)
duration_ms = time_ms() - start_ms
if flags:
self._flag_changers[str(device)].Restore()
if test_timeout_scale:
valgrind_tools.SetChromeTimeoutScale(
device, self._test_instance.timeout_scale)
# TODO(jbudorick): Make instrumentation tests output a JSON so this
# doesn't have to parse the output.
result_code, result_bundle, statuses = (
self._test_instance.ParseAmInstrumentRawOutput(output))
results = self._test_instance.GenerateTestResults(
result_code, result_bundle, statuses, start_ms, duration_ms)
for result in results:
if logcat_url:
result.SetLink('logcat', logcat_url)
# Update the result name if the test used flags.
if flags:
for r in results:
if r.GetName() == test_name:
r.SetName(test_display_name)
# Add UNKNOWN results for any missing tests.
iterable_test = test if isinstance(test, list) else [test]
test_names = set(self._GetUniqueTestName(t) for t in iterable_test)
results_names = set(r.GetName() for r in results)
results.extend(
base_test_result.BaseTestResult(
u, base_test_result.ResultType.UNKNOWN)
for u in test_names.difference(results_names))
# Update the result type if we detect a crash.
if DidPackageCrashOnDevice(self._test_instance.test_package, device):
for r in results:
if r.GetType() == base_test_result.ResultType.UNKNOWN:
r.SetType(base_test_result.ResultType.CRASH)
# Handle failures by:
# - optionally taking a screenshot
# - logging the raw output at INFO level
# - clearing the application state while persisting permissions
if any(r.GetType() not in (base_test_result.ResultType.PASS,
base_test_result.ResultType.SKIP)
for r in results):
if self._test_instance.screenshot_dir:
file_name = '%s-%s.png' % (
test_display_name,
time.strftime('%Y%m%dT%H%M%S', time.localtime()))
saved_dir = device.TakeScreenshot(
os.path.join(self._test_instance.screenshot_dir,
file_name))
logging.info('Saved screenshot for %s to %s.',
test_display_name, saved_dir)
logging.info('detected failure in %s. raw output:',
test_display_name)
for l in output:
logging.info(' %s', l)
if (not self._env.skip_clear_data
and self._test_instance.package_info):
permissions = (
self._test_instance.apk_under_test.GetPermissions()
if self._test_instance.apk_under_test else None)
device.ClearApplicationState(
self._test_instance.package_info.package,
permissions=permissions)
else:
logging.debug('raw output from %s:', test_display_name)
for l in output:
logging.debug(' %s', l)
if self._test_instance.coverage_directory:
device.PullFile(coverage_directory,
self._test_instance.coverage_directory)
device.RunShellCommand('rm -f %s' %
os.path.join(coverage_directory, '*'))
if self._test_instance.store_tombstones:
tombstones_url = None
for result in results:
if result.GetType() == base_test_result.ResultType.CRASH:
if not tombstones_url:
resolved_tombstones = tombstones.ResolveTombstones(
device,
resolve_all_tombstones=True,
include_stack_symbols=False,
wipe_tombstones=True)
stream_name = 'tombstones_%s_%s' % (time.strftime(
'%Y%m%dT%H%M%S', time.localtime()), device.serial)
tombstones_url = logdog_helper.text(
stream_name, resolved_tombstones)
result.SetLink('tombstones', tombstones_url)
return results, None
def _RunTest(self, device, test):
# Run the test.
timeout = (self._test_instance.shard_timeout *
self.GetTool(device).GetTimeoutScale())
if self._test_instance.wait_for_java_debugger:
timeout = None
if self._test_instance.store_tombstones:
tombstones.ClearAllTombstones(device)
test_perf_output_filename = next(self._test_perf_output_filenames)
with device_temp_file.DeviceTempFile(
adb=device.adb,
dir=self._delegate.ResultsDirectory(device),
suffix='.xml') as device_tmp_results_file:
with contextlib_ext.Optional(
device_temp_file.NamedDeviceTemporaryDirectory(
adb=device.adb, dir='/sdcard/'), self._test_instance.
gs_test_artifacts_bucket) as test_artifacts_dir:
with (contextlib_ext.Optional(
device_temp_file.DeviceTempFile(
adb=device.adb,
dir=self._delegate.ResultsDirectory(device)),
test_perf_output_filename)
) as isolated_script_test_perf_output:
flags = list(self._test_instance.flags)
if self._test_instance.enable_xml_result_parsing:
flags.append('--gtest_output=xml:%s' %
device_tmp_results_file.name)
if self._test_instance.gs_test_artifacts_bucket:
flags.append('--test_artifacts_dir=%s' %
test_artifacts_dir.name)
if test_perf_output_filename:
flags.append('--isolated_script_test_perf_output=%s' %
isolated_script_test_perf_output.name)
logging.info('flags:')
for f in flags:
logging.info(' %s', f)
stream_name = 'logcat_%s_%s_%s' % (
hash(tuple(test)),
time.strftime('%Y%m%dT%H%M%S-UTC',
time.gmtime()), device.serial)
with self._env.output_manager.ArchivedTempfile(
stream_name, 'logcat') as logcat_file:
with logcat_monitor.LogcatMonitor(
device.adb,
filter_specs=local_device_environment.
LOGCAT_FILTERS,
output_file=logcat_file.name,
check_error=False) as logmon:
with contextlib_ext.Optional(
trace_event.trace(str(test)),
self._env.trace_output):
output = self._delegate.Run(
test,
device,
flags=' '.join(flags),
timeout=timeout,
retries=0)
logmon.Close()
if logcat_file.Link():
logging.info('Logcat saved to %s', logcat_file.Link())
if self._test_instance.enable_xml_result_parsing:
try:
gtest_xml = device.ReadFile(
device_tmp_results_file.name, as_root=True)
except device_errors.CommandFailedError as e:
logging.warning(
'Failed to pull gtest results XML file %s: %s',
device_tmp_results_file.name, str(e))
gtest_xml = None
if test_perf_output_filename:
try:
device.PullFile(
isolated_script_test_perf_output.name,
test_perf_output_filename)
except device_errors.CommandFailedError as e:
logging.warning(
'Failed to pull chartjson results %s: %s',
isolated_script_test_perf_output.name, str(e))
test_artifacts_url = self._UploadTestArtifacts(
device, test_artifacts_dir)
for s in self._servers[str(device)]:
s.Reset()
if self._test_instance.app_files:
self._delegate.PullAppFiles(device, self._test_instance.app_files,
self._test_instance.app_file_dir)
if not self._env.skip_clear_data:
self._delegate.Clear(device)
for l in output:
logging.info(l)
# Parse the output.
# TODO(jbudorick): Transition test scripts away from parsing stdout.
if self._test_instance.enable_xml_result_parsing:
results = gtest_test_instance.ParseGTestXML(gtest_xml)
else:
results = gtest_test_instance.ParseGTestOutput(
output, self._test_instance.symbolizer, device.product_cpu_abi)
tombstones_url = None
for r in results:
if logcat_file:
r.SetLink('logcat', logcat_file.Link())
if self._test_instance.gs_test_artifacts_bucket:
r.SetLink('test_artifacts', test_artifacts_url)
if r.GetType() == base_test_result.ResultType.CRASH:
self._crashes.add(r.GetName())
if self._test_instance.store_tombstones:
if not tombstones_url:
resolved_tombstones = tombstones.ResolveTombstones(
device,
resolve_all_tombstones=True,
include_stack_symbols=False,
wipe_tombstones=True)
stream_name = 'tombstones_%s_%s' % (time.strftime(
'%Y%m%dT%H%M%S', time.localtime()), device.serial)
tombstones_url = logdog_helper.text(
stream_name, '\n'.join(resolved_tombstones))
r.SetLink('tombstones', tombstones_url)
tests_stripped_disabled_prefix = set()
for t in test:
tests_stripped_disabled_prefix.add(
gtest_test_instance.TestNameWithoutDisabledPrefix(t))
not_run_tests = tests_stripped_disabled_prefix.difference(
set(r.GetName() for r in results))
return results, list(not_run_tests) if results else None
def _RunTest(self, device, test):
# Run the test.
timeout = (self._test_instance.shard_timeout
* self.GetTool(device).GetTimeoutScale())
if self._test_instance.store_tombstones:
tombstones.ClearAllTombstones(device)
with device_temp_file.DeviceTempFile(
adb=device.adb,
dir=self._delegate.ResultsDirectory(device),
suffix='.xml') as device_tmp_results_file:
flags = list(self._test_instance.flags)
if self._test_instance.enable_xml_result_parsing:
flags.append('--gtest_output=xml:%s' % device_tmp_results_file.name)
logging.info('flags:')
for f in flags:
logging.info(' %s', f)
with contextlib_ext.Optional(
trace_event.trace(str(test)),
self._env.trace_output):
output = self._delegate.Run(
test, device, flags=' '.join(flags),
timeout=timeout, retries=0)
if self._test_instance.enable_xml_result_parsing:
gtest_xml = device.ReadFile(
device_tmp_results_file.name,
as_root=True)
for s in self._servers[str(device)]:
s.Reset()
if self._test_instance.app_files:
self._delegate.PullAppFiles(device, self._test_instance.app_files,
self._test_instance.app_file_dir)
if not self._env.skip_clear_data:
self._delegate.Clear(device)
for l in output:
logging.info(l)
# Parse the output.
# TODO(jbudorick): Transition test scripts away from parsing stdout.
if self._test_instance.enable_xml_result_parsing:
results = gtest_test_instance.ParseGTestXML(gtest_xml)
else:
results = gtest_test_instance.ParseGTestOutput(output)
# Check whether there are any crashed testcases.
self._crashes.update(r.GetName() for r in results
if r.GetType() == base_test_result.ResultType.CRASH)
if self._test_instance.store_tombstones:
tombstones_url = None
for result in results:
if result.GetType() == base_test_result.ResultType.CRASH:
if not tombstones_url:
resolved_tombstones = tombstones.ResolveTombstones(
device,
resolve_all_tombstones=True,
include_stack_symbols=False,
wipe_tombstones=True)
stream_name = 'tombstones_%s_%s' % (
time.strftime('%Y%m%dT%H%M%S', time.localtime()),
device.serial)
tombstones_url = logdog_helper.text(
stream_name, '\n'.join(resolved_tombstones))
result.SetLink('tombstones', tombstones_url)
tests_stripped_disabled_prefix = set()
for t in test:
tests_stripped_disabled_prefix.add(
gtest_test_instance.TestNameWithoutDisabledPrefix(t))
not_run_tests = tests_stripped_disabled_prefix.difference(
set(r.GetName() for r in results))
return results, list(not_run_tests) if results else None
def _RunTest(self, device, test):
# Run the test.
timeout = (self._test_instance.shard_timeout *
self.GetTool(device).GetTimeoutScale())
if self._test_instance.store_tombstones:
tombstones.ClearAllTombstones(device)
with device_temp_file.DeviceTempFile(
adb=device.adb,
dir=self._delegate.ResultsDirectory(device),
suffix='.xml') as device_tmp_results_file:
flags = list(self._test_instance.flags)
if self._test_instance.enable_xml_result_parsing:
flags.append('--gtest_output=xml:%s' %
device_tmp_results_file.name)
logging.info('flags:')
for f in flags:
logging.info(' %s', f)
with contextlib_ext.Optional(trace_event.trace(str(test)),
self._env.trace_output):
output = self._delegate.Run(test,
device,
flags=' '.join(flags),
timeout=timeout,
retries=0)
if self._test_instance.enable_xml_result_parsing:
gtest_xml = device.ReadFile(device_tmp_results_file.name,
as_root=True)
for s in self._servers[str(device)]:
s.Reset()
if self._test_instance.app_files:
self._delegate.PullAppFiles(device, self._test_instance.app_files,
self._test_instance.app_file_dir)
if not self._env.skip_clear_data:
self._delegate.Clear(device)
# Parse the output.
# TODO(jbudorick): Transition test scripts away from parsing stdout.
if self._test_instance.enable_xml_result_parsing:
results = gtest_test_instance.ParseGTestXML(gtest_xml)
else:
results = gtest_test_instance.ParseGTestOutput(output)
# Check whether there are any crashed testcases.
self._crashes.update(
r.GetName() for r in results
if r.GetType() == base_test_result.ResultType.CRASH)
if self._test_instance.store_tombstones:
tombstones_url = None
for result in results:
if result.GetType() == base_test_result.ResultType.CRASH:
if not tombstones_url:
resolved_tombstones = tombstones.ResolveTombstones(
device,
resolve_all_tombstones=True,
include_stack_symbols=False,
wipe_tombstones=True)
stream_name = 'tombstones_%s_%s' % (time.strftime(
'%Y%m%dT%H%M%S', time.localtime()), device.serial)
tombstones_url = logdog_helper.text(
stream_name, resolved_tombstones)
result.SetLink('tombstones', tombstones_url)
tests_stripped_disabled_prefix = set()
for t in test:
tests_stripped_disabled_prefix.add(
gtest_test_instance.TestNameWithoutDisabledPrefix(t))
not_run_tests = tests_stripped_disabled_prefix.difference(
set(r.GetName() for r in results))
return results, list(not_run_tests) if results else None
def _Archive(self):
with open(self.name, 'r') as f:
logdog_helper.text(self._stream_name, f.read())
def _handle_perf_results(benchmark_enabled_map, benchmark_directory_map,
configuration_name, build_properties, extra_links,
output_results_dir):
"""
Upload perf results to the perf dashboard.
This method also upload the perf results to logdog and augment it to
|extra_links|.
Returns:
(return_code, benchmark_upload_result_map)
return_code is 0 if this upload to perf dashboard successfully, 1
otherwise.
benchmark_upload_result_map is a dictionary describes which benchmark
was successfully uploaded.
"""
begin_time = time.time()
# Upload all eligible benchmarks to the perf dashboard
results_dict = {}
invocations = []
for benchmark_name, directories in benchmark_directory_map.iteritems():
if not benchmark_enabled_map.get(benchmark_name, False):
continue
# Create a place to write the perf results that you will write out to
# logdog.
output_json_file = os.path.join(output_results_dir,
(str(uuid.uuid4()) + benchmark_name))
results_dict[benchmark_name] = output_json_file
invocations.append((benchmark_name, directories, configuration_name,
build_properties, output_json_file))
# Kick off the uploads in multiple processes
pool = multiprocessing.Pool(_GetCpuCount())
try:
async_result = pool.map_async(_upload_individual_benchmark,
invocations)
# TODO(crbug.com/947035): What timeout is reasonable?
results = async_result.get(timeout=4000)
except multiprocessing.TimeoutError:
logging.error(
'Failed uploading benchmarks to perf dashboard in parallel')
results = []
for benchmark_name in benchmark_directory_map:
results.append((benchmark_name, False))
finally:
pool.terminate()
# Keep a mapping of benchmarks to their upload results
benchmark_upload_result_map = {}
for r in results:
benchmark_upload_result_map[r[0]] = r[1]
logdog_dict = {}
upload_failures_counter = 0
logdog_stream = None
logdog_label = 'Results Dashboard'
for benchmark_name, output_file in results_dict.iteritems():
upload_succeed = benchmark_upload_result_map[benchmark_name]
if not upload_succeed:
upload_failures_counter += 1
is_reference = '.reference' in benchmark_name
_write_perf_data_to_logfile(benchmark_name,
output_file,
configuration_name,
build_properties,
logdog_dict,
is_reference,
upload_failure=not upload_succeed)
logdog_file_name = _generate_unique_logdog_filename('Results_Dashboard_')
logdog_stream = logdog_helper.text(logdog_file_name,
json.dumps(dict(logdog_dict),
sort_keys=True,
indent=4,
separators=(',', ': ')),
content_type=JSON_CONTENT_TYPE)
if upload_failures_counter > 0:
logdog_label += (' %s merge script perf data upload failures' %
upload_failures_counter)
extra_links[logdog_label] = logdog_stream
end_time = time.time()
print_duration('Uploading results to perf dashboard', begin_time, end_time)
if upload_failures_counter > 0:
return 1, benchmark_upload_result_map
return 0, benchmark_upload_result_map
def _handle_perf_results(benchmark_enabled_map, benchmark_directory_map,
configuration_name, build_properties,
service_account_file, extra_links):
"""
Upload perf results to the perf dashboard.
This method also upload the perf results to logdog and augment it to
|extra_links|.
Returns:
0 if this upload to perf dashboard succesfully, 1 otherwise.
"""
begin_time = time.time()
tmpfile_dir = tempfile.mkdtemp('outputresults')
try:
# Upload all eligible benchmarks to the perf dashboard
results_dict = {}
invocations = []
for benchmark_name, directories in benchmark_directory_map.iteritems():
if not benchmark_enabled_map.get(benchmark_name, False):
continue
# Create a place to write the perf results that you will write out to
# logdog.
output_json_file = os.path.join(
tmpfile_dir, (str(uuid.uuid4()) + benchmark_name))
results_dict[benchmark_name] = output_json_file
invocations.append(
(benchmark_name, directories, configuration_name,
build_properties, output_json_file, service_account_file))
# Kick off the uploads in mutliple processes
pool = mp.Pool()
try:
async_result = pool.map_async(_upload_individual_benchmark,
invocations)
results = async_result.get(timeout=2000)
except mp.TimeoutError:
logging.error(
'Failed uploading benchmarks to perf dashboard in parallel')
pool.terminate()
results = []
for benchmark_name in benchmark_directory_map:
results.append((benchmark_name, False))
# Keep a mapping of benchmarks to their upload results
benchmark_upload_result_map = {}
for r in results:
benchmark_upload_result_map[r[0]] = bool(r[1])
logdog_dict = {}
upload_failures_counter = 0
logdog_stream = None
logdog_label = 'Results Dashboard'
for benchmark_name, output_file in results_dict.iteritems():
failure = benchmark_upload_result_map[benchmark_name]
if failure:
upload_failures_counter += 1
is_reference = '.reference' in benchmark_name
_write_perf_data_to_logfile(benchmark_name, output_file,
configuration_name, build_properties,
logdog_dict, is_reference, failure)
logdog_file_name = _generate_unique_logdog_filename(
'Results_Dashboard_')
logdog_stream = logdog_helper.text(logdog_file_name,
json.dumps(dict(logdog_dict),
sort_keys=True,
indent=4,
separators=(',', ': ')),
content_type=JSON_CONTENT_TYPE)
if upload_failures_counter > 0:
logdog_label += ('Upload Failure (%s benchmark upload failures)' %
upload_failures_counter)
extra_links[logdog_label] = logdog_stream
end_time = time.time()
print_duration('Uploading results to perf dashboard', begin_time,
end_time)
if upload_failures_counter > 0:
return 1
return 0
finally:
shutil.rmtree(tmpfile_dir)
def _Archive(self):
with open(self.name, 'r') as f:
logdog_helper.text(self._stream_name, f.read())
def _RunTest(self, device, test):
# Run the test.
timeout = (self._test_instance.shard_timeout
* self.GetTool(device).GetTimeoutScale())
if self._test_instance.wait_for_java_debugger:
timeout = None
if self._test_instance.store_tombstones:
tombstones.ClearAllTombstones(device)
with device_temp_file.DeviceTempFile(
adb=device.adb,
dir=self._delegate.ResultsDirectory(device),
suffix='.xml') as device_tmp_results_file:
with contextlib_ext.Optional(
device_temp_file.NamedDeviceTemporaryDirectory(
adb=device.adb, dir='/sdcard/'),
self._test_instance.gs_test_artifacts_bucket) as test_artifacts_dir:
with (contextlib_ext.Optional(
device_temp_file.DeviceTempFile(
adb=device.adb, dir=self._delegate.ResultsDirectory(device)),
self._test_instance.isolated_script_test_perf_output)
) as isolated_script_test_perf_output:
flags = list(self._test_instance.flags)
if self._test_instance.enable_xml_result_parsing:
flags.append('--gtest_output=xml:%s' % device_tmp_results_file.name)
if self._test_instance.gs_test_artifacts_bucket:
flags.append('--test_artifacts_dir=%s' % test_artifacts_dir.name)
if self._test_instance.isolated_script_test_perf_output:
flags.append('--isolated_script_test_perf_output=%s'
% isolated_script_test_perf_output.name)
logging.info('flags:')
for f in flags:
logging.info(' %s', f)
stream_name = 'logcat_%s_%s_%s' % (
hash(tuple(test)),
time.strftime('%Y%m%dT%H%M%S-UTC', time.gmtime()),
device.serial)
with self._env.output_manager.ArchivedTempfile(
stream_name, 'logcat') as logcat_file:
with logcat_monitor.LogcatMonitor(
device.adb,
filter_specs=local_device_environment.LOGCAT_FILTERS,
output_file=logcat_file.name) as logmon:
with contextlib_ext.Optional(
trace_event.trace(str(test)),
self._env.trace_output):
output = self._delegate.Run(
test, device, flags=' '.join(flags),
timeout=timeout, retries=0)
logmon.Close()
if logcat_file.Link():
logging.info('Logcat saved to %s', logcat_file.Link())
if self._test_instance.enable_xml_result_parsing:
try:
gtest_xml = device.ReadFile(
device_tmp_results_file.name,
as_root=True)
except device_errors.CommandFailedError as e:
logging.warning(
'Failed to pull gtest results XML file %s: %s',
device_tmp_results_file.name,
str(e))
gtest_xml = None
if self._test_instance.isolated_script_test_perf_output:
try:
device.PullFile(
isolated_script_test_perf_output.name,
self._test_instance.isolated_script_test_perf_output)
except device_errors.CommandFailedError as e:
logging.warning(
'Failed to pull chartjson results %s: %s',
isolated_script_test_perf_output.name, str(e))
test_artifacts_url = self._UploadTestArtifacts(device,
test_artifacts_dir)
for s in self._servers[str(device)]:
s.Reset()
if self._test_instance.app_files:
self._delegate.PullAppFiles(device, self._test_instance.app_files,
self._test_instance.app_file_dir)
if not self._env.skip_clear_data:
self._delegate.Clear(device)
for l in output:
logging.info(l)
# Parse the output.
# TODO(jbudorick): Transition test scripts away from parsing stdout.
if self._test_instance.enable_xml_result_parsing:
results = gtest_test_instance.ParseGTestXML(gtest_xml)
else:
results = gtest_test_instance.ParseGTestOutput(
output, self._test_instance.symbolizer, device.product_cpu_abi)
tombstones_url = None
for r in results:
if logcat_file:
r.SetLink('logcat', logcat_file.Link())
if self._test_instance.gs_test_artifacts_bucket:
r.SetLink('test_artifacts', test_artifacts_url)
if r.GetType() == base_test_result.ResultType.CRASH:
self._crashes.add(r.GetName())
if self._test_instance.store_tombstones:
if not tombstones_url:
resolved_tombstones = tombstones.ResolveTombstones(
device,
resolve_all_tombstones=True,
include_stack_symbols=False,
wipe_tombstones=True)
stream_name = 'tombstones_%s_%s' % (
time.strftime('%Y%m%dT%H%M%S', time.localtime()),
device.serial)
tombstones_url = logdog_helper.text(
stream_name, '\n'.join(resolved_tombstones))
r.SetLink('tombstones', tombstones_url)
tests_stripped_disabled_prefix = set()
for t in test:
tests_stripped_disabled_prefix.add(
gtest_test_instance.TestNameWithoutDisabledPrefix(t))
not_run_tests = tests_stripped_disabled_prefix.difference(
set(r.GetName() for r in results))
return results, list(not_run_tests) if results else None
def _RunTest(self, device, test):
extras = {}
flags_to_add = []
test_timeout_scale = None
if self._test_instance.coverage_directory:
coverage_basename = '%s.ec' % ('%s_group' %
test[0]['method'] if isinstance(
test, list) else test['method'])
extras['coverage'] = 'true'
coverage_directory = os.path.join(device.GetExternalStoragePath(),
'chrome', 'test', 'coverage')
coverage_device_file = os.path.join(coverage_directory,
coverage_basename)
extras['coverageFile'] = coverage_device_file
# Save screenshot if screenshot dir is specified (save locally) or if
# a GS bucket is passed (save in cloud).
screenshot_device_file = None
if (self._test_instance.screenshot_dir
or self._test_instance.gs_results_bucket):
screenshot_device_file = device_temp_file.DeviceTempFile(
device.adb, suffix='.png', dir=device.GetExternalStoragePath())
extras[EXTRA_SCREENSHOT_FILE] = screenshot_device_file.name
extras[EXTRA_UI_CAPTURE_DIR] = self._ui_capture_dir[device]
if isinstance(test, list):
if not self._test_instance.driver_apk:
raise Exception('driver_apk does not exist. '
'Please build it and try again.')
if any(t.get('is_junit4') for t in test):
raise Exception('driver apk does not support JUnit4 tests')
def name_and_timeout(t):
n = instrumentation_test_instance.GetTestName(t)
i = self._GetTimeoutFromAnnotations(t['annotations'], n)
return (n, i)
test_names, timeouts = zip(*(name_and_timeout(t) for t in test))
test_name = ','.join(test_names)
test_display_name = test_name
target = '%s/%s' % (self._test_instance.driver_package,
self._test_instance.driver_name)
extras.update(
self._test_instance.GetDriverEnvironmentVars(
test_list=test_names))
timeout = sum(timeouts)
else:
test_name = instrumentation_test_instance.GetTestName(test)
test_display_name = self._GetUniqueTestName(test)
if test['is_junit4']:
target = '%s/%s' % (self._test_instance.test_package,
self._test_instance.test_runner_junit4)
else:
target = '%s/%s' % (self._test_instance.test_package,
self._test_instance.test_runner)
extras['class'] = test_name
if 'flags' in test and test['flags']:
flags_to_add.extend(test['flags'])
timeout = self._GetTimeoutFromAnnotations(test['annotations'],
test_display_name)
test_timeout_scale = self._GetTimeoutScaleFromAnnotations(
test['annotations'])
if test_timeout_scale and test_timeout_scale != 1:
valgrind_tools.SetChromeTimeoutScale(
device,
test_timeout_scale * self._test_instance.timeout_scale)
logging.info('preparing to run %s: %s', test_display_name, test)
render_tests_device_output_dir = None
if _IsRenderTest(test):
# TODO(mikecase): Add DeviceTempDirectory class and use that instead.
render_tests_device_output_dir = posixpath.join(
device.GetExternalStoragePath(), 'render_test_output_dir')
flags_to_add.append('--render-test-output-dir=%s' %
render_tests_device_output_dir)
if flags_to_add:
self._CreateFlagChangerIfNeeded(device)
self._flag_changers[str(device)].PushFlags(add=flags_to_add)
time_ms = lambda: int(time.time() * 1e3)
start_ms = time_ms()
stream_name = 'logcat_%s_%s_%s' % (test_name.replace(
'#', '.'), time.strftime('%Y%m%dT%H%M%S-UTC',
time.gmtime()), device.serial)
logmon = logdog_logcat_monitor.LogdogLogcatMonitor(
device.adb, stream_name, filter_specs=LOGCAT_FILTERS)
with contextlib_ext.Optional(logmon,
self._test_instance.should_save_logcat):
with _LogTestEndpoints(device, test_name):
with contextlib_ext.Optional(trace_event.trace(test_name),
self._env.trace_output):
output = device.StartInstrumentation(target,
raw=True,
extras=extras,
timeout=timeout,
retries=0)
logcat_url = logmon.GetLogcatURL()
duration_ms = time_ms() - start_ms
# TODO(jbudorick): Make instrumentation tests output a JSON so this
# doesn't have to parse the output.
result_code, result_bundle, statuses = (
self._test_instance.ParseAmInstrumentRawOutput(output))
results = self._test_instance.GenerateTestResults(
result_code, result_bundle, statuses, start_ms, duration_ms)
def restore_flags():
if flags_to_add:
self._flag_changers[str(device)].Restore()
def restore_timeout_scale():
if test_timeout_scale:
valgrind_tools.SetChromeTimeoutScale(
device, self._test_instance.timeout_scale)
def handle_coverage_data():
if self._test_instance.coverage_directory:
device.PullFile(coverage_directory,
self._test_instance.coverage_directory)
device.RunShellCommand('rm -f %s' %
posixpath.join(coverage_directory, '*'),
check_return=True,
shell=True)
def handle_render_test_data():
if _IsRenderTest(test):
# Render tests do not cause test failure by default. So we have to check
# to see if any failure images were generated even if the test does not
# fail.
try:
self._ProcessRenderTestResults(
device, render_tests_device_output_dir, results)
finally:
device.RemovePath(render_tests_device_output_dir,
recursive=True,
force=True)
# While constructing the TestResult objects, we can parallelize several
# steps that involve ADB. These steps should NOT depend on any info in
# the results! Things such as whether the test CRASHED have not yet been
# determined.
post_test_steps = [
restore_flags, restore_timeout_scale, handle_coverage_data,
handle_render_test_data
]
if self._env.concurrent_adb:
post_test_step_thread_group = reraiser_thread.ReraiserThreadGroup(
reraiser_thread.ReraiserThread(f) for f in post_test_steps)
post_test_step_thread_group.StartAll(will_block=True)
else:
for step in post_test_steps:
step()
for result in results:
if logcat_url:
result.SetLink('logcat', logcat_url)
# Update the result name if the test used flags.
if flags_to_add:
for r in results:
if r.GetName() == test_name:
r.SetName(test_display_name)
# Add UNKNOWN results for any missing tests.
iterable_test = test if isinstance(test, list) else [test]
test_names = set(self._GetUniqueTestName(t) for t in iterable_test)
results_names = set(r.GetName() for r in results)
results.extend(
base_test_result.BaseTestResult(
u, base_test_result.ResultType.UNKNOWN)
for u in test_names.difference(results_names))
# Update the result type if we detect a crash.
if DidPackageCrashOnDevice(self._test_instance.test_package, device):
for r in results:
if r.GetType() == base_test_result.ResultType.UNKNOWN:
r.SetType(base_test_result.ResultType.CRASH)
# Handle failures by:
# - optionally taking a screenshot
# - logging the raw output at INFO level
# - clearing the application state while persisting permissions
if any(r.GetType() not in (base_test_result.ResultType.PASS,
base_test_result.ResultType.SKIP)
for r in results):
with contextlib_ext.Optional(
tempfile_ext.NamedTemporaryDirectory(),
self._test_instance.screenshot_dir is None
and self._test_instance.gs_results_bucket
) as screenshot_host_dir:
screenshot_host_dir = (self._test_instance.screenshot_dir
or screenshot_host_dir)
self._SaveScreenshot(device, screenshot_host_dir,
screenshot_device_file, test_display_name,
results)
logging.info('detected failure in %s. raw output:',
test_display_name)
for l in output:
logging.info(' %s', l)
if (not self._env.skip_clear_data
and self._test_instance.package_info):
permissions = (
self._test_instance.apk_under_test.GetPermissions()
if self._test_instance.apk_under_test else None)
device.ClearApplicationState(
self._test_instance.package_info.package,
permissions=permissions)
else:
logging.debug('raw output from %s:', test_display_name)
for l in output:
logging.debug(' %s', l)
if self._test_instance.store_tombstones:
tombstones_url = None
for result in results:
if result.GetType() == base_test_result.ResultType.CRASH:
if not tombstones_url:
resolved_tombstones = tombstones.ResolveTombstones(
device,
resolve_all_tombstones=True,
include_stack_symbols=False,
wipe_tombstones=True)
stream_name = 'tombstones_%s_%s' % (time.strftime(
'%Y%m%dT%H%M%S-UTC', time.gmtime()), device.serial)
tombstones_url = logdog_helper.text(
stream_name, '\n'.join(resolved_tombstones))
result.SetLink('tombstones', tombstones_url)
if self._env.concurrent_adb:
post_test_step_thread_group.JoinAll()
return results, None
def _RunTest(self, device, test):
extras = {}
flags_to_add = []
test_timeout_scale = None
if self._test_instance.coverage_directory:
coverage_basename = '%s.ec' % ('%s_group' % test[0]['method']
if isinstance(test, list) else test['method'])
extras['coverage'] = 'true'
coverage_directory = os.path.join(
device.GetExternalStoragePath(), 'chrome', 'test', 'coverage')
coverage_device_file = os.path.join(
coverage_directory, coverage_basename)
extras['coverageFile'] = coverage_device_file
# Save screenshot if screenshot dir is specified (save locally) or if
# a GS bucket is passed (save in cloud).
screenshot_device_file = None
if (self._test_instance.screenshot_dir or
self._test_instance.gs_results_bucket):
screenshot_device_file = device_temp_file.DeviceTempFile(
device.adb, suffix='.png', dir=device.GetExternalStoragePath())
extras[EXTRA_SCREENSHOT_FILE] = screenshot_device_file.name
extras[EXTRA_UI_CAPTURE_DIR] = self._ui_capture_dir[device]
if self._env.trace_output:
trace_device_file = device_temp_file.DeviceTempFile(
device.adb, suffix='.json', dir=device.GetExternalStoragePath())
extras[EXTRA_TRACE_FILE] = trace_device_file.name
if isinstance(test, list):
if not self._test_instance.driver_apk:
raise Exception('driver_apk does not exist. '
'Please build it and try again.')
if any(t.get('is_junit4') for t in test):
raise Exception('driver apk does not support JUnit4 tests')
def name_and_timeout(t):
n = instrumentation_test_instance.GetTestName(t)
i = self._GetTimeoutFromAnnotations(t['annotations'], n)
return (n, i)
test_names, timeouts = zip(*(name_and_timeout(t) for t in test))
test_name = ','.join(test_names)
test_display_name = test_name
target = '%s/%s' % (
self._test_instance.driver_package,
self._test_instance.driver_name)
extras.update(
self._test_instance.GetDriverEnvironmentVars(
test_list=test_names))
timeout = sum(timeouts)
else:
test_name = instrumentation_test_instance.GetTestName(test)
test_display_name = self._GetUniqueTestName(test)
if test['is_junit4']:
target = '%s/%s' % (
self._test_instance.test_package,
self._test_instance.junit4_runner_class)
else:
target = '%s/%s' % (
self._test_instance.test_package,
self._test_instance.junit3_runner_class)
extras['class'] = test_name
if 'flags' in test and test['flags']:
flags_to_add.extend(test['flags'])
timeout = self._GetTimeoutFromAnnotations(
test['annotations'], test_display_name)
test_timeout_scale = self._GetTimeoutScaleFromAnnotations(
test['annotations'])
if test_timeout_scale and test_timeout_scale != 1:
valgrind_tools.SetChromeTimeoutScale(
device, test_timeout_scale * self._test_instance.timeout_scale)
logging.info('preparing to run %s: %s', test_display_name, test)
render_tests_device_output_dir = None
if _IsRenderTest(test):
# TODO(mikecase): Add DeviceTempDirectory class and use that instead.
render_tests_device_output_dir = posixpath.join(
device.GetExternalStoragePath(),
'render_test_output_dir')
flags_to_add.append('--render-test-output-dir=%s' %
render_tests_device_output_dir)
if flags_to_add:
self._CreateFlagChangerIfNeeded(device)
self._flag_changers[str(device)].PushFlags(add=flags_to_add)
time_ms = lambda: int(time.time() * 1e3)
start_ms = time_ms()
stream_name = 'logcat_%s_%s_%s' % (
test_name.replace('#', '.'),
time.strftime('%Y%m%dT%H%M%S-UTC', time.gmtime()),
device.serial)
logmon = logdog_logcat_monitor.LogdogLogcatMonitor(
device.adb, stream_name, filter_specs=LOGCAT_FILTERS)
with contextlib_ext.Optional(
logmon, self._test_instance.should_save_logcat):
with _LogTestEndpoints(device, test_name):
with contextlib_ext.Optional(
trace_event.trace(test_name),
self._env.trace_output):
output = device.StartInstrumentation(
target, raw=True, extras=extras, timeout=timeout, retries=0)
logcat_url = logmon.GetLogcatURL()
duration_ms = time_ms() - start_ms
if self._env.trace_output:
self._SaveTraceData(trace_device_file, device, test['class'])
# TODO(jbudorick): Make instrumentation tests output a JSON so this
# doesn't have to parse the output.
result_code, result_bundle, statuses = (
self._test_instance.ParseAmInstrumentRawOutput(output))
results = self._test_instance.GenerateTestResults(
result_code, result_bundle, statuses, start_ms, duration_ms)
def restore_flags():
if flags_to_add:
self._flag_changers[str(device)].Restore()
def restore_timeout_scale():
if test_timeout_scale:
valgrind_tools.SetChromeTimeoutScale(
device, self._test_instance.timeout_scale)
def handle_coverage_data():
if self._test_instance.coverage_directory:
device.PullFile(coverage_directory,
self._test_instance.coverage_directory)
device.RunShellCommand(
'rm -f %s' % posixpath.join(coverage_directory, '*'),
check_return=True, shell=True)
def handle_render_test_data():
if _IsRenderTest(test):
# Render tests do not cause test failure by default. So we have to check
# to see if any failure images were generated even if the test does not
# fail.
try:
self._ProcessRenderTestResults(
device, render_tests_device_output_dir, results)
finally:
device.RemovePath(render_tests_device_output_dir,
recursive=True, force=True)
# While constructing the TestResult objects, we can parallelize several
# steps that involve ADB. These steps should NOT depend on any info in
# the results! Things such as whether the test CRASHED have not yet been
# determined.
post_test_steps = [restore_flags, restore_timeout_scale,
handle_coverage_data, handle_render_test_data]
if self._env.concurrent_adb:
post_test_step_thread_group = reraiser_thread.ReraiserThreadGroup(
reraiser_thread.ReraiserThread(f) for f in post_test_steps)
post_test_step_thread_group.StartAll(will_block=True)
else:
for step in post_test_steps:
step()
for result in results:
if logcat_url:
result.SetLink('logcat', logcat_url)
# Update the result name if the test used flags.
if flags_to_add:
for r in results:
if r.GetName() == test_name:
r.SetName(test_display_name)
# Add UNKNOWN results for any missing tests.
iterable_test = test if isinstance(test, list) else [test]
test_names = set(self._GetUniqueTestName(t) for t in iterable_test)
results_names = set(r.GetName() for r in results)
results.extend(
base_test_result.BaseTestResult(u, base_test_result.ResultType.UNKNOWN)
for u in test_names.difference(results_names))
# Update the result type if we detect a crash.
if DidPackageCrashOnDevice(self._test_instance.test_package, device):
for r in results:
if r.GetType() == base_test_result.ResultType.UNKNOWN:
r.SetType(base_test_result.ResultType.CRASH)
# Handle failures by:
# - optionally taking a screenshot
# - logging the raw output at INFO level
# - clearing the application state while persisting permissions
if any(r.GetType() not in (base_test_result.ResultType.PASS,
base_test_result.ResultType.SKIP)
for r in results):
with contextlib_ext.Optional(
tempfile_ext.NamedTemporaryDirectory(),
self._test_instance.screenshot_dir is None and
self._test_instance.gs_results_bucket) as screenshot_host_dir:
screenshot_host_dir = (
self._test_instance.screenshot_dir or screenshot_host_dir)
self._SaveScreenshot(device, screenshot_host_dir,
screenshot_device_file, test_display_name,
results)
logging.info('detected failure in %s. raw output:', test_display_name)
for l in output:
logging.info(' %s', l)
if (not self._env.skip_clear_data
and self._test_instance.package_info):
permissions = (
self._test_instance.apk_under_test.GetPermissions()
if self._test_instance.apk_under_test
else None)
device.ClearApplicationState(self._test_instance.package_info.package,
permissions=permissions)
else:
logging.debug('raw output from %s:', test_display_name)
for l in output:
logging.debug(' %s', l)
if self._test_instance.store_tombstones:
tombstones_url = None
for result in results:
if result.GetType() == base_test_result.ResultType.CRASH:
if not tombstones_url:
resolved_tombstones = tombstones.ResolveTombstones(
device,
resolve_all_tombstones=True,
include_stack_symbols=False,
wipe_tombstones=True,
tombstone_symbolizer=self._test_instance.symbolizer)
stream_name = 'tombstones_%s_%s' % (
time.strftime('%Y%m%dT%H%M%S-UTC', time.gmtime()),
device.serial)
tombstones_url = logdog_helper.text(
stream_name, '\n'.join(resolved_tombstones))
result.SetLink('tombstones', tombstones_url)
if self._env.concurrent_adb:
post_test_step_thread_group.JoinAll()
return results, None
def _process_perf_results(output_json, configuration_name,
service_account_file, build_properties,
task_output_dir):
"""Process one or more perf JSON results.
Consists of merging the json-test-format output and uploading the perf test
output (chartjson and histogram).
Each directory in the task_output_dir represents one benchmark
that was run. Within this directory, there is a subdirectory with the name
of the benchmark that was run. In that subdirectory, there is a
perftest-output.json file containing the performance results in histogram
or dashboard json format and an output.json file containing the json test
results for the benchmark.
"""
directory_list = [
f for f in listdir(task_output_dir)
if not isfile(join(task_output_dir, f))
]
benchmark_directory_list = []
for directory in directory_list:
benchmark_directory_list += [
join(task_output_dir, directory, f)
for f in listdir(join(task_output_dir, directory))
]
# We need to keep track of disabled benchmarks so we don't try to
# upload the results.
test_results_list = []
tmpfile_dir = tempfile.mkdtemp('resultscache')
upload_failure = False
build_properties = json.loads(build_properties)
if not configuration_name:
# we are deprecating perf-id crbug.com/817823
configuration_name = build_properties['buildername']
try:
logdog_dict = {}
with oauth_api.with_access_token(service_account_file) as oauth_file:
for directory in benchmark_directory_list:
# Obtain the test name we are running
benchmark_name = basename(directory).replace(" benchmark", "")
is_ref = '.reference' in benchmark_name
disabled = False
with open(join(directory, 'test_results.json')) as json_data:
json_results = json.load(json_data)
if not json_results:
# Output is null meaning the test didn't produce any results.
# Want to output an error and continue loading the rest of the
# test results.
print 'No results produced for %s, skipping upload' % directory
continue
if json_results.get('version') == 3:
# Non-telemetry tests don't have written json results but
# if they are executing then they are enabled and will generate
# chartjson results.
if not bool(json_results.get('tests')):
disabled = True
if not is_ref:
# We don't need to upload reference build data to the
# flakiness dashboard since we don't monitor the ref build
test_results_list.append(json_results)
if disabled:
# We don't upload disabled benchmarks
print 'Benchmark %s disabled' % benchmark_name
continue
print 'Uploading perf results from %s benchmark' % benchmark_name
upload_fail = _upload_and_write_perf_data_to_logfile(
benchmark_name, directory, configuration_name,
build_properties, oauth_file, tmpfile_dir, logdog_dict,
is_ref)
upload_failure = upload_failure or upload_fail
logdog_label = 'Results Dashboard'
logdog_file_name = 'Results_Dashboard_' + str(uuid.uuid4())
if upload_failure:
logdog_label += ' Upload Failure'
_merge_json_output(
output_json, test_results_list,
logdog_helper.text(
logdog_file_name,
json.dumps(logdog_dict,
sort_keys=True,
indent=4,
separators=(',', ':'))), logdog_label)
finally:
shutil.rmtree(tmpfile_dir)
return upload_failure
def _RunTest(self, device, test):
extras = {}
flags = None
test_timeout_scale = None
if self._test_instance.coverage_directory:
coverage_basename = '%s.ec' % ('%s_group' % test[0]['method']
if isinstance(test, list) else test['method'])
extras['coverage'] = 'true'
coverage_directory = os.path.join(
device.GetExternalStoragePath(), 'chrome', 'test', 'coverage')
coverage_device_file = os.path.join(
coverage_directory, coverage_basename)
extras['coverageFile'] = coverage_device_file
if isinstance(test, list):
if not self._test_instance.driver_apk:
raise Exception('driver_apk does not exist. '
'Please build it and try again.')
if any(t.get('is_junit4') for t in test):
raise Exception('driver apk does not support JUnit4 tests')
def name_and_timeout(t):
n = instrumentation_test_instance.GetTestName(t)
i = self._GetTimeoutFromAnnotations(t['annotations'], n)
return (n, i)
test_names, timeouts = zip(*(name_and_timeout(t) for t in test))
test_name = ','.join(test_names)
test_display_name = test_name
target = '%s/%s' % (
self._test_instance.driver_package,
self._test_instance.driver_name)
extras.update(
self._test_instance.GetDriverEnvironmentVars(
test_list=test_names))
timeout = sum(timeouts)
else:
test_name = instrumentation_test_instance.GetTestName(test)
test_display_name = self._GetUniqueTestName(test)
if test['is_junit4']:
target = '%s/%s' % (
self._test_instance.test_package,
self._test_instance.test_runner_junit4)
else:
target = '%s/%s' % (
self._test_instance.test_package, self._test_instance.test_runner)
extras['class'] = test_name
if 'flags' in test:
flags = test['flags']
timeout = self._GetTimeoutFromAnnotations(
test['annotations'], test_display_name)
test_timeout_scale = self._GetTimeoutScaleFromAnnotations(
test['annotations'])
if test_timeout_scale and test_timeout_scale != 1:
valgrind_tools.SetChromeTimeoutScale(
device, test_timeout_scale * self._test_instance.timeout_scale)
logging.info('preparing to run %s: %s', test_display_name, test)
if flags:
self._CreateFlagChangerIfNeeded(device)
self._flag_changers[str(device)].PushFlags(
add=flags.add, remove=flags.remove)
try:
device.RunShellCommand(
['log', '-p', 'i', '-t', _TAG, 'START %s' % test_name],
check_return=True)
time_ms = lambda: int(time.time() * 1e3)
start_ms = time_ms()
stream_name = 'logcat_%s_%s_%s' % (
test_name.replace('#', '.'),
time.strftime('%Y%m%dT%H%M%S', time.localtime()),
device.serial)
logmon = logdog_logcat_monitor.LogdogLogcatMonitor(
device.adb, stream_name, filter_specs=LOGCAT_FILTERS)
with contextlib_ext.Optional(
logmon, self._test_instance.should_save_logcat):
with contextlib_ext.Optional(
trace_event.trace(test_name),
self._env.trace_output):
output = device.StartInstrumentation(
target, raw=True, extras=extras, timeout=timeout, retries=0)
logcat_url = logmon.GetLogcatURL()
finally:
device.RunShellCommand(
['log', '-p', 'i', '-t', _TAG, 'END %s' % test_name],
check_return=True)
duration_ms = time_ms() - start_ms
if flags:
self._flag_changers[str(device)].Restore()
if test_timeout_scale:
valgrind_tools.SetChromeTimeoutScale(
device, self._test_instance.timeout_scale)
# TODO(jbudorick): Make instrumentation tests output a JSON so this
# doesn't have to parse the output.
result_code, result_bundle, statuses = (
self._test_instance.ParseAmInstrumentRawOutput(output))
results = self._test_instance.GenerateTestResults(
result_code, result_bundle, statuses, start_ms, duration_ms)
for result in results:
if logcat_url:
result.SetLink('logcat', logcat_url)
# Update the result name if the test used flags.
if flags:
for r in results:
if r.GetName() == test_name:
r.SetName(test_display_name)
# Add UNKNOWN results for any missing tests.
iterable_test = test if isinstance(test, list) else [test]
test_names = set(self._GetUniqueTestName(t) for t in iterable_test)
results_names = set(r.GetName() for r in results)
results.extend(
base_test_result.BaseTestResult(u, base_test_result.ResultType.UNKNOWN)
for u in test_names.difference(results_names))
# Update the result type if we detect a crash.
if DidPackageCrashOnDevice(self._test_instance.test_package, device):
for r in results:
if r.GetType() == base_test_result.ResultType.UNKNOWN:
r.SetType(base_test_result.ResultType.CRASH)
# Handle failures by:
# - optionally taking a screenshot
# - logging the raw output at INFO level
# - clearing the application state while persisting permissions
if any(r.GetType() not in (base_test_result.ResultType.PASS,
base_test_result.ResultType.SKIP)
for r in results):
if self._test_instance.screenshot_dir:
file_name = '%s-%s.png' % (
test_display_name,
time.strftime('%Y%m%dT%H%M%S', time.localtime()))
saved_dir = device.TakeScreenshot(
os.path.join(self._test_instance.screenshot_dir, file_name))
logging.info(
'Saved screenshot for %s to %s.',
test_display_name, saved_dir)
logging.info('detected failure in %s. raw output:', test_display_name)
for l in output:
logging.info(' %s', l)
if (not self._env.skip_clear_data
and self._test_instance.package_info):
permissions = (
self._test_instance.apk_under_test.GetPermissions()
if self._test_instance.apk_under_test
else None)
device.ClearApplicationState(self._test_instance.package_info.package,
permissions=permissions)
else:
logging.debug('raw output from %s:', test_display_name)
for l in output:
logging.debug(' %s', l)
if self._test_instance.coverage_directory:
device.PullFile(coverage_directory,
self._test_instance.coverage_directory)
device.RunShellCommand(
'rm -f %s' % posixpath.join(coverage_directory, '*'),
check_return=True, shell=True)
if self._test_instance.store_tombstones:
tombstones_url = None
for result in results:
if result.GetType() == base_test_result.ResultType.CRASH:
if not tombstones_url:
resolved_tombstones = tombstones.ResolveTombstones(
device,
resolve_all_tombstones=True,
include_stack_symbols=False,
wipe_tombstones=True)
stream_name = 'tombstones_%s_%s' % (
time.strftime('%Y%m%dT%H%M%S', time.localtime()),
device.serial)
tombstones_url = logdog_helper.text(
stream_name, resolved_tombstones)
result.SetLink('tombstones', tombstones_url)
return results, None
def _process_perf_results(output_json, configuration_name,
service_account_file,
build_properties, task_output_dir,
smoke_test_mode):
"""Process one or more perf JSON results.
Consists of merging the json-test-format output and uploading the perf test
output (chartjson and histogram).
Each directory in the task_output_dir represents one benchmark
that was run. Within this directory, there is a subdirectory with the name
of the benchmark that was run. In that subdirectory, there is a
perftest-output.json file containing the performance results in histogram
or dashboard json format and an output.json file containing the json test
results for the benchmark.
"""
directory_list = [
f for f in listdir(task_output_dir)
if not isfile(join(task_output_dir, f))
]
benchmark_directory_list = []
for directory in directory_list:
benchmark_directory_list += [
join(task_output_dir, directory, f)
for f in listdir(join(task_output_dir, directory))
]
test_results_list = []
tmpfile_dir = tempfile.mkdtemp('resultscache')
upload_failure = False
build_properties = json.loads(build_properties)
if not configuration_name:
# we are deprecating perf-id crbug.com/817823
configuration_name = build_properties['buildername']
try:
# First obtain the list of json test results to merge
# and determine the status of each benchmark
benchmark_enabled_map = _handle_perf_json_test_results(
benchmark_directory_list, test_results_list)
# Upload all eligible benchmarks to the perf dashboard
logdog_dict = {}
logdog_stream = None
logdog_label = 'Results Dashboard'
if not smoke_test_mode:
with oauth_api.with_access_token(service_account_file) as oauth_file:
for directory in benchmark_directory_list:
benchmark_name = _get_benchmark_name(directory)
if not benchmark_enabled_map[benchmark_name]:
continue
print 'Uploading perf results from %s benchmark' % benchmark_name
upload_fail = _upload_and_write_perf_data_to_logfile(
benchmark_name, directory, configuration_name, build_properties,
oauth_file, tmpfile_dir, logdog_dict,
('.reference' in benchmark_name))
upload_failure = upload_failure or upload_fail
logdog_file_name = 'Results_Dashboard_' + str(uuid.uuid4())
logdog_stream = logdog_helper.text(logdog_file_name,
json.dumps(logdog_dict, sort_keys=True,
indent=4, separators=(',', ':')))
if upload_failure:
logdog_label += ' Upload Failure'
# Finally, merge all test results json and write out to output location
_merge_json_output(output_json, test_results_list,
logdog_stream, logdog_label)
finally:
shutil.rmtree(tmpfile_dir)
return upload_failure
