def initialize(self, context):
super(Speedometer, self).initialize(context)
self.archive_server = ArchiveServer()
if not self.target.is_rooted:
raise WorkloadError(
"Device must be rooted for the speedometer workload currently"
)
if self.target.adb_server is not None:
raise WorkloadError(
"Workload does not support the adb_server parameter, due to the webpage "
"hosting mechanism."
)
# Temporary directory used for storing the Speedometer files, uiautomator
# dumps, and modified XML chrome config files.
self.temp_dir = tempfile.TemporaryDirectory()
self.document_root = os.path.join(self.temp_dir.name, "document_root")
# Host a copy of Speedometer locally
tarball = context.get_resource(File(self, "speedometer_archive.tgz"))
with tarfile.open(name=tarball) as handle:
handle.extractall(self.temp_dir.name)
self.archive_server.start(self.document_root, self.target)
self.webserver_port = self.archive_server.get_port()
self.speedometer_url = "http://localhost:{}/Speedometer2.0/index.html".format(
self.webserver_port
)
def resolve_package(self, context):
if not self.owner.package_names and not self.package_name:
msg = 'Cannot Resolve package; No package name(s) specified'
raise WorkloadError(msg)
self.error_msg = None
if self.prefer_host_package:
self.resolve_package_from_host(context)
if not self.apk_file:
self.resolve_package_from_target()
else:
self.resolve_package_from_target()
if not self.apk_file:
self.resolve_package_from_host(context)
if self.apk_file:
self.apk_info = get_cacheable_apk_info(self.apk_file)
else:
if self.error_msg:
raise WorkloadError(self.error_msg)
else:
if self.package_name:
message = 'Package "{package}" not found for workload {name} '\
'on host or target.'
elif self.version:
message = 'No matching package found for workload {name} '\
'(version {version}) on host or target.'
else:
message = 'No matching package found for workload {name} on host or target'
raise WorkloadError(
message.format(name=self.owner.name,
version=self.version,
package=self.package_name))
def wait_for_benchmark_to_complete(self, report_end_id):
local_storage = "/data/data/{}/app_chrome/Default/Local Storage/leveldb".format(
self.chrome_package
)
sleep_period_s = 5
find_period_s = 30
timeout_period_m = 15
iterations = 0
local_storage_seen = False
benchmark_complete = False
while not benchmark_complete:
if self.target_file_was_created(local_storage):
if (
iterations % (find_period_s // sleep_period_s) == 0 or
not local_storage_seen
):
# There's a chance we don't see the localstorage file immediately, and there's a
# chance more of them could be created later, so check for those files every ~30
# seconds.
find_cmd = '{} find "{}" -iname "*.log"'.format(
self.target.busybox, local_storage
)
candidate_files = self.target.execute(find_cmd, as_root=True).split(
"\n"
)
local_storage_seen = True
for ls_file in candidate_files:
# Each local storage file is in a binary format. The busybox grep seems to
# print out the line '[KEY][VALUE]' for a match, rather than just reporting
# that 'binary file X matches', so just check the output for our generated ID.
grep_cmd = '{} grep {} "{}"'.format(
self.target.busybox, report_end_id, ls_file
)
output = self.target.execute(
grep_cmd, as_root=True, check_exit_code=False
)
if report_end_id in output:
benchmark_complete = True
break
iterations += 1
if iterations > ((timeout_period_m * 60) // sleep_period_s):
# We've been waiting 15 minutes for Speedometer to finish running - give up.
if not local_storage_seen:
raise WorkloadError(
"Speedometer did not complete within 15m - Local Storage wasn't found"
)
raise WorkloadError("Speedometer did not complete within 15 minutes.")
time.sleep(sleep_period_s)
def initialize(self, context):
super(PcMark, self).initialize(context)
# Need root to get results
if not self.target.is_rooted:
raise WorkloadError('PCMark workload requires device to be rooted')
if not self.target.is_installed(self.package):
raise WorkloadError('Package not installed. ' + INSTALL_INSTRUCTIONS)
path = ('/storage/emulated/0/Android/data/{}/files/dlc/pcma-workv2-data'
.format(self.package))
if not self.target.file_exists(path):
raise WorkloadError('"Work v2" benchmark not installed through app. '
+ INSTALL_INSTRUCTIONS)
def update_output(self, context):
super(Motionmark, self).update_output(context)
num_unprocessed_results = len(self.regex)
logcat_file = context.get_artifact_path('logcat')
with open(logcat_file) as fh:
for line in fh:
for regex in self.regex:
match = regex.search(line)
# Check if we have matched the score string in logcat
if match:
score_match = self.score_regex.search(match.group(1))
# Check if there is valid number found for the score.
if score_match:
result = float(score_match.group(1))
else:
result = float('NaN')
entry = regex.pattern.rsplit(None, 1)[0]
context.add_metric(entry,
result,
'Score',
lower_is_better=False)
num_unprocessed_results -= 1
if num_unprocessed_results > 0:
msg = "The Motionmark workload has failed. Expected {} scores, Missing {} scores."
raise WorkloadError(
msg.format(len(self.regex), num_unprocessed_results))
def update_output(self, context):
super(Speedometer, self).update_output(context)
self.ui_dump_loc = os.path.join(self.target.working_directory,
"ui_dump.xml")
score_read = False
iterations = 0
while not score_read:
score = self.read_score()
if score is not None:
context.add_metric("Speedometer Score",
score,
"Runs per minute",
lower_is_better=False)
score_read = True
else:
if iterations >= 10:
raise WorkloadError(
"The Speedometer workload has failed. No score was obtainable."
)
else:
# Sleep and retry...
time.sleep(2)
iterations += 1
def initialize(self, context):
if not self.target.is_rooted:
raise WorkloadError('stress-ng requires root premissions to run')
resource = Executable(self, self.target.abi, 'stress-ng')
host_exe = context.get_resource(resource)
StressNg.binary = self.target.install(host_exe)
def initialize(self, context):
super(Geekbench, self).initialize(context)
if not self.disable_update_result and not self.target.is_rooted:
raise WorkloadError(
'Geekbench workload requires root to collect results. '
'You can set disable_update_result=True in the workload params '
'to run without collecting results.')
def _execute(self, stage, timeout):
result = self.target.execute(self.commands[stage], timeout)
if 'FAILURE' in result:
raise WorkloadError(result)
else:
self.logger.debug(result)
time.sleep(2)
def get_scores(self):
"""
Returns a tuple (single-thraded score, multi-threaded score) for this workload.
Some workloads only have a single-threaded score, in which case multi-threaded
score will be ``None``.
Geekbench will perform four iterations of each workload in single-threaded and,
for some workloads, multi-threaded configurations. Thus there should always be
either four or eight scores collected for each workload. Single-threaded iterations
are always done before multi-threaded, so the ordering of the scores can be used
to determine which configuration they belong to.
This method should not be called before score collection has finished.
"""
no_of_results = len(self.collected_results)
if no_of_results == 4:
return (self._calculate(self.collected_results[:4],
self.pmac_g5_st_score), None)
if no_of_results == 8:
return (self._calculate(self.collected_results[:4],
self.pmac_g5_st_score),
self._calculate(self.collected_results[4:],
self.pmac_g5_mt_score))
else:
msg = 'Collected {} results for Geekbench {} workload;'.format(
no_of_results, self.name)
msg += ' expecting either 4 or 8.'
raise WorkloadError(msg)
def update_output(self, context):
super(Aitutu, self).update_output(context)
expected_results = len(self.regex_matches)
logcat_file = context.get_artifact_path('logcat')
with open(logcat_file) as fh:
for line in fh:
for regex in self.regex_matches:
match = regex.search(line)
if match:
classifiers = {}
result = match.group(1)
if (len(match.groups())) > 1:
entry = regex.pattern.rsplit(None, 3)[0]
classifiers = {'model': match.group(3)}
else:
entry = regex.pattern.rsplit(None, 1)[0]
context.add_metric(entry,
result,
'',
lower_is_better=False,
classifiers=classifiers)
expected_results -= 1
if expected_results > 0:
msg = "The Aitutu workload has failed. Expected {} scores, Detected {} scores."
raise WorkloadError(
msg.format(len(self.regex_matches), expected_results))
def initialize(self, context): # pylint: disable=no-self-use
if self.target.get_sdk_version() < 23:
raise WorkloadError("This workload relies on ``dumpsys gfxinfo`` \
only present in Android M and onwards")
defs_host = context.get_resource(File(self, "defs.sh"))
Recentfling.defs_target = self.target.install(defs_host)
recentfling_host = context.get_resource(File(self, "recentfling.sh"))
Recentfling.recentfling_target = self.target.install(recentfling_host)
def pull_apk(self, package):
if not self.target.package_is_installed(package):
message = 'Cannot retrieve "{}" as not installed on Target'
raise WorkloadError(message.format(package))
package_info = self.target.get_package_info(package)
self.target.pull(package_info.apk_path, self.owner.dependencies_directory,
timeout=self.install_timeout)
apk_name = self.target.path.basename(package_info.apk_path)
return os.path.join(self.owner.dependencies_directory, apk_name)
def initialize(self, context):
super(Vellamo, self).initialize(context)
if self.version == '2.0.3' or not self.benchmarks: # pylint: disable=access-member-before-definition
self.benchmarks = self.benchmark_types[self.version] # pylint: disable=attribute-defined-outside-init
else:
for benchmark in self.benchmarks:
if benchmark not in self.benchmark_types[self.version]:
raise WorkloadError(
'Version {} does not support {} benchmarks'.format(
self.version, benchmark))
def initialize(self, context):
if self.as_root and not self.target.is_rooted:
raise WorkloadError('Cannot run script as root -- target appears to be unrooted.')
self.script_file = os.path.expanduser(self.script_file)
if not os.path.isfile(self.script_file):
raise ConfigError('Can\'t access file (is the path correct?): {}'.format(self.script_file))
self.output = None
self.command = None
self.on_target_script_file = None
def __init__(self, target, **kwargs):
super(Workload, self).__init__(target, **kwargs)
self.asset_files = []
self.deployed_assets = []
supported_platforms = getattr(self, 'supported_platforms', [])
if supported_platforms and self.target.os not in supported_platforms:
msg = 'Supported platforms for "{}" are "{}", attempting to run on "{}"'
raise WorkloadError(msg.format(self.name, ' '.join(self.supported_platforms),
self.target.os))
def initialize(self, context):
super(Geekbench, self).initialize(context)
self.gui.uiauto_params['version'] = self.version
self.gui.uiauto_params['loops'] = self.loops
self.gui.uiauto_params['is_corporate'] = self.is_corporate
self.gui.timeout = self.timeout
if not self.disable_update_result and not self.target.is_rooted:
raise WorkloadError(
'Geekbench workload requires root to collect results. '
'You can set disable_update_result=True in the workload params '
'to run without collecting results.')
def pull_apk(self, package):
if not self.target.package_is_installed(package):
message = 'Cannot retrieve "{}" as not installed on Target'
raise WorkloadError(message.format(package))
package_info = self.target.get_package_info(package)
apk_name = self._get_package_name(package_info.apk_path)
host_path = os.path.join(self.owner.dependencies_directory, apk_name)
with atomic_write_path(host_path) as at_path:
self.target.pull(package_info.apk_path,
at_path,
timeout=self.install_timeout)
return host_path
def _generate_workgen_config(self, user_file, output_directory):
output_file = os.path.join(output_directory, 'unkind.json')
# use workgen dry run option to generate a use case
# file with proper JSON grammar on host first
try:
check_output('python3 {} -d -o {} {}'.format(
self.workgen_script, output_file, user_file),
shell=True)
except CalledProcessError as e:
message = 'Could not generate config using workgen, got "{}"'
raise WorkloadError(message.format(e))
return output_file
def install_apk(self, context):
# pylint: disable=unused-argument
output = self.target.install_apk(self.apk_file, self.install_timeout,
replace=True, allow_downgrade=True)
if 'Failure' in output:
if 'ALREADY_EXISTS' in output:
msg = 'Using already installed APK (did not uninstall properly?)'
self.logger.warning(msg)
else:
raise WorkloadError(output)
else:
self.logger.debug(output)
def start_activity(self):
if not self.activity:
cmd = 'am start -W {}'.format(self.apk_info.package)
else:
cmd = 'am start -W -n {}/{}'.format(self.apk_info.package,
self.activity)
output = self.target.execute(cmd)
if 'Error:' in output:
# this will dismiss any error dialogs
self.target.execute('am force-stop {}'.format(self.apk_info.package))
raise WorkloadError(output)
self.logger.debug(output)
def setup(self, context):
"""
Perform the setup necessary to run the workload, such as copying the
necessary files to the device, configuring the environments, etc.
This is also the place to perform any on-device checks prior to
attempting to execute the workload.
"""
# pylint: disable=unused-argument
if self.requires_network and not self.target.is_network_connected():
raise WorkloadError(
'Workload "{}" requires internet. Target does not appear '
'to be connected to the internet.'.format(self.name))
def update_output(self, context):
super(Speedometer, self).update_output(context)
result = None
logcat_file = context.get_artifact_path('logcat')
with open(logcat_file) as fh:
for line in fh:
match = self.regex.search(line)
if match:
result = float(match.group(1))
if result is not None:
context.add_metric('Speedometer Score', result, 'Runs per minute', lower_is_better=False)
else:
raise WorkloadError("The Speedometer workload has failed. No score was obtainable.")
def _check_revent_files(self):
if not self.revent_run_file:
# pylint: disable=too-few-format-args
message = '{0}.run.revent file does not exist, ' \
'Please provide one for your target, {0}'
raise WorkloadError(message.format(self.target.model))
self.target.push(self.revent_run_file, self.on_target_run_revent)
if self.revent_setup_file:
self.target.push(self.revent_setup_file, self.on_target_setup_revent)
if self.revent_extract_results_file:
self.target.push(self.revent_extract_results_file, self.on_target_extract_results_revent)
if self.revent_teardown_file:
self.target.push(self.revent_teardown_file, self.on_target_teardown_revent)
def run(self, context):
# All we need to do is
# - start the activity,
# - then use the JbRunMonitor to wait until the benchmark reports on
# logcat that it is finished,
# - pull the result database file.
result = self.target.execute(self.command)
if 'FAILURE' in result:
raise WorkloadError(result)
else:
self.logger.debug(result)
self.monitor.wait_for(REGEXPS['start'], timeout=30)
self.logger.info('Detected Jankbench start')
self.monitor.wait_for(REGEXPS['done'], timeout=300*self.times)
def update_output(self, context):
expected_results = len(self.regex_matches[self.major_version])
zf = zipfile.ZipFile(os.path.join(context.output_directory, self.result_file), 'r').read('Result.xml')
if sys.version_info[0] == 3:
zf = zf.decode(sys.stdout.encoding)
for line in zf.split('\n'):
for regex in self.regex_matches[self.major_version]:
match = regex.search(line)
if match:
scores = float(match.group(1))
entry = regex.pattern
entry = entry[:-9]
context.add_metric(entry, scores, lower_is_better=False)
expected_results -= 1
if expected_results > 0:
msg = "The PCMark workload has failed. Expected {} scores, Detected {} scores."
raise WorkloadError(msg.format(len(self.regex_matches[self.major_version]), expected_results))
def init_resources(self, context):
super(Gmail, self).init_resources(context)
if self.target.get_sdk_version(
) >= 24 and 'com.google.android.apps.photos' not in self.target.list_packages(
):
raise WorkloadError(
'gmail workload requires Google Photos to be installed for Android N onwards'
)
# Allows for getting working directory regardless if path ends with a '/'
work_dir = self.target.working_directory
work_dir = work_dir if work_dir[-1] != os.sep else work_dir[:-1]
self.gui.uiauto_params['workdir_name'] = self.target.path.basename(
work_dir)
self.gui.uiauto_params['recipient'] = self.recipient
self.gui.uiauto_params['offline_mode'] = self.offline_mode
# Only accept certain image formats
if os.path.splitext(
self.test_image.lower())[1] not in ['.jpg', '.jpeg', '.png']:
raise ValidationError('{} must be a JPEG or PNG file'.format(
self.test_image))
def update_output(self, context):
super(Androbench, self).update_output(context)
expected_results = len(self.regex_matches)
logcat_file = context.get_artifact_path('logcat')
with open(logcat_file, errors='replace') as fh:
for line in fh:
for regex in self.regex_matches:
match = regex.search(line)
if match:
result = float(match.group(1))
entry = regex.pattern.rsplit(None, 1)[0]
context.add_metric(entry,
result,
'MB/s',
lower_is_better=False)
expected_results -= 1
if expected_results > 0:
msg = "The Androbench workload has failed. Expected {} scores, Detected {} scores."
raise WorkloadError(
msg.format(len(self.regex_matches), expected_results))
def non_root_update_output(self, context):
failed = []
logcat_file = context.get_artifact_path('logcat')
with open(logcat_file, errors='replace') as fh:
iteration_result_regex = re.compile(
"VELLAMO RESULT: (Browser|Metal|Multicore) (\d+)")
for line in fh:
if 'VELLAMO ERROR:' in line:
msg = "Browser crashed during benchmark, results may not be accurate"
self.logger.warning(msg)
result = iteration_result_regex.findall(line)
if result:
for (metric, score) in result:
if not score:
failed.append(metric)
else:
context.add_metric(metric, score)
if failed:
raise WorkloadError(
"The following benchmark groups failed: {}".format(
", ".join(failed)))
def extract_results(self, context):
remote_zip_path = re.match(self.regexps['result'], self.output).group('path')
local_zip_path = os.path.join(context.output_directory,
self.target.path.basename(remote_zip_path))
self.logger.info('pulling {} -> {}'.format(remote_zip_path, local_zip_path))
self.target.pull(remote_zip_path, local_zip_path, as_root=True)
with ZipFile(local_zip_path, 'r') as archive:
archive.extractall(context.output_directory)
xml_path = os.path.join(context.output_directory, 'Result.xml')
if not os.path.exists(xml_path):
raise WorkloadError("PCMark results .zip didn't contain Result.xml")
context.add_artifact('pcmark_result_xml', xml_path, 'data')
# Fetch workloads names and scores
score_regex = re.compile('\s*<result_Pcma(?P<name>.*)Score>(?P<score>[0-9]*)<')
with open(xml_path) as f:
for line in f:
match = score_regex.match(line)
if match:
metric_name = 'pcmark_{}'.format(match.group('name'))
context.add_metric(metric_name, match.group('score'))