def initialize(self, context): super(Speedometer, self).initialize(context) self.archive_server = ArchiveServer() if not self.target.is_rooted: raise WorkloadError( "Device must be rooted for the speedometer workload currently" ) if self.target.adb_server is not None: raise WorkloadError( "Workload does not support the adb_server parameter, due to the webpage " "hosting mechanism." ) # Temporary directory used for storing the Speedometer files, uiautomator # dumps, and modified XML chrome config files. self.temp_dir = tempfile.TemporaryDirectory() self.document_root = os.path.join(self.temp_dir.name, "document_root") # Host a copy of Speedometer locally tarball = context.get_resource(File(self, "speedometer_archive.tgz")) with tarfile.open(name=tarball) as handle: handle.extractall(self.temp_dir.name) self.archive_server.start(self.document_root, self.target) self.webserver_port = self.archive_server.get_port() self.speedometer_url = "http://localhost:{}/Speedometer2.0/index.html".format( self.webserver_port )
def resolve_package(self, context): if not self.owner.package_names and not self.package_name: msg = 'Cannot Resolve package; No package name(s) specified' raise WorkloadError(msg) self.error_msg = None if self.prefer_host_package: self.resolve_package_from_host(context) if not self.apk_file: self.resolve_package_from_target() else: self.resolve_package_from_target() if not self.apk_file: self.resolve_package_from_host(context) if self.apk_file: self.apk_info = get_cacheable_apk_info(self.apk_file) else: if self.error_msg: raise WorkloadError(self.error_msg) else: if self.package_name: message = 'Package "{package}" not found for workload {name} '\ 'on host or target.' elif self.version: message = 'No matching package found for workload {name} '\ '(version {version}) on host or target.' else: message = 'No matching package found for workload {name} on host or target' raise WorkloadError( message.format(name=self.owner.name, version=self.version, package=self.package_name))
def wait_for_benchmark_to_complete(self, report_end_id): local_storage = "/data/data/{}/app_chrome/Default/Local Storage/leveldb".format( self.chrome_package ) sleep_period_s = 5 find_period_s = 30 timeout_period_m = 15 iterations = 0 local_storage_seen = False benchmark_complete = False while not benchmark_complete: if self.target_file_was_created(local_storage): if ( iterations % (find_period_s // sleep_period_s) == 0 or not local_storage_seen ): # There's a chance we don't see the localstorage file immediately, and there's a # chance more of them could be created later, so check for those files every ~30 # seconds. find_cmd = '{} find "{}" -iname "*.log"'.format( self.target.busybox, local_storage ) candidate_files = self.target.execute(find_cmd, as_root=True).split( "\n" ) local_storage_seen = True for ls_file in candidate_files: # Each local storage file is in a binary format. The busybox grep seems to # print out the line '[KEY][VALUE]' for a match, rather than just reporting # that 'binary file X matches', so just check the output for our generated ID. grep_cmd = '{} grep {} "{}"'.format( self.target.busybox, report_end_id, ls_file ) output = self.target.execute( grep_cmd, as_root=True, check_exit_code=False ) if report_end_id in output: benchmark_complete = True break iterations += 1 if iterations > ((timeout_period_m * 60) // sleep_period_s): # We've been waiting 15 minutes for Speedometer to finish running - give up. if not local_storage_seen: raise WorkloadError( "Speedometer did not complete within 15m - Local Storage wasn't found" ) raise WorkloadError("Speedometer did not complete within 15 minutes.") time.sleep(sleep_period_s)
def initialize(self, context): super(PcMark, self).initialize(context) # Need root to get results if not self.target.is_rooted: raise WorkloadError('PCMark workload requires device to be rooted') if not self.target.is_installed(self.package): raise WorkloadError('Package not installed. ' + INSTALL_INSTRUCTIONS) path = ('/storage/emulated/0/Android/data/{}/files/dlc/pcma-workv2-data' .format(self.package)) if not self.target.file_exists(path): raise WorkloadError('"Work v2" benchmark not installed through app. ' + INSTALL_INSTRUCTIONS)
def update_output(self, context): super(Motionmark, self).update_output(context) num_unprocessed_results = len(self.regex) logcat_file = context.get_artifact_path('logcat') with open(logcat_file) as fh: for line in fh: for regex in self.regex: match = regex.search(line) # Check if we have matched the score string in logcat if match: score_match = self.score_regex.search(match.group(1)) # Check if there is valid number found for the score. if score_match: result = float(score_match.group(1)) else: result = float('NaN') entry = regex.pattern.rsplit(None, 1)[0] context.add_metric(entry, result, 'Score', lower_is_better=False) num_unprocessed_results -= 1 if num_unprocessed_results > 0: msg = "The Motionmark workload has failed. Expected {} scores, Missing {} scores." raise WorkloadError( msg.format(len(self.regex), num_unprocessed_results))
def update_output(self, context): super(Speedometer, self).update_output(context) self.ui_dump_loc = os.path.join(self.target.working_directory, "ui_dump.xml") score_read = False iterations = 0 while not score_read: score = self.read_score() if score is not None: context.add_metric("Speedometer Score", score, "Runs per minute", lower_is_better=False) score_read = True else: if iterations >= 10: raise WorkloadError( "The Speedometer workload has failed. No score was obtainable." ) else: # Sleep and retry... time.sleep(2) iterations += 1
def initialize(self, context): if not self.target.is_rooted: raise WorkloadError('stress-ng requires root premissions to run') resource = Executable(self, self.target.abi, 'stress-ng') host_exe = context.get_resource(resource) StressNg.binary = self.target.install(host_exe)
def initialize(self, context): super(Geekbench, self).initialize(context) if not self.disable_update_result and not self.target.is_rooted: raise WorkloadError( 'Geekbench workload requires root to collect results. ' 'You can set disable_update_result=True in the workload params ' 'to run without collecting results.')
def _execute(self, stage, timeout): result = self.target.execute(self.commands[stage], timeout) if 'FAILURE' in result: raise WorkloadError(result) else: self.logger.debug(result) time.sleep(2)
def get_scores(self): """ Returns a tuple (single-thraded score, multi-threaded score) for this workload. Some workloads only have a single-threaded score, in which case multi-threaded score will be ``None``. Geekbench will perform four iterations of each workload in single-threaded and, for some workloads, multi-threaded configurations. Thus there should always be either four or eight scores collected for each workload. Single-threaded iterations are always done before multi-threaded, so the ordering of the scores can be used to determine which configuration they belong to. This method should not be called before score collection has finished. """ no_of_results = len(self.collected_results) if no_of_results == 4: return (self._calculate(self.collected_results[:4], self.pmac_g5_st_score), None) if no_of_results == 8: return (self._calculate(self.collected_results[:4], self.pmac_g5_st_score), self._calculate(self.collected_results[4:], self.pmac_g5_mt_score)) else: msg = 'Collected {} results for Geekbench {} workload;'.format( no_of_results, self.name) msg += ' expecting either 4 or 8.' raise WorkloadError(msg)
def update_output(self, context): super(Aitutu, self).update_output(context) expected_results = len(self.regex_matches) logcat_file = context.get_artifact_path('logcat') with open(logcat_file) as fh: for line in fh: for regex in self.regex_matches: match = regex.search(line) if match: classifiers = {} result = match.group(1) if (len(match.groups())) > 1: entry = regex.pattern.rsplit(None, 3)[0] classifiers = {'model': match.group(3)} else: entry = regex.pattern.rsplit(None, 1)[0] context.add_metric(entry, result, '', lower_is_better=False, classifiers=classifiers) expected_results -= 1 if expected_results > 0: msg = "The Aitutu workload has failed. Expected {} scores, Detected {} scores." raise WorkloadError( msg.format(len(self.regex_matches), expected_results))
def initialize(self, context): # pylint: disable=no-self-use if self.target.get_sdk_version() < 23: raise WorkloadError("This workload relies on ``dumpsys gfxinfo`` \ only present in Android M and onwards") defs_host = context.get_resource(File(self, "defs.sh")) Recentfling.defs_target = self.target.install(defs_host) recentfling_host = context.get_resource(File(self, "recentfling.sh")) Recentfling.recentfling_target = self.target.install(recentfling_host)
def pull_apk(self, package): if not self.target.package_is_installed(package): message = 'Cannot retrieve "{}" as not installed on Target' raise WorkloadError(message.format(package)) package_info = self.target.get_package_info(package) self.target.pull(package_info.apk_path, self.owner.dependencies_directory, timeout=self.install_timeout) apk_name = self.target.path.basename(package_info.apk_path) return os.path.join(self.owner.dependencies_directory, apk_name)
def initialize(self, context): super(Vellamo, self).initialize(context) if self.version == '2.0.3' or not self.benchmarks: # pylint: disable=access-member-before-definition self.benchmarks = self.benchmark_types[self.version] # pylint: disable=attribute-defined-outside-init else: for benchmark in self.benchmarks: if benchmark not in self.benchmark_types[self.version]: raise WorkloadError( 'Version {} does not support {} benchmarks'.format( self.version, benchmark))
def initialize(self, context): if self.as_root and not self.target.is_rooted: raise WorkloadError('Cannot run script as root -- target appears to be unrooted.') self.script_file = os.path.expanduser(self.script_file) if not os.path.isfile(self.script_file): raise ConfigError('Can\'t access file (is the path correct?): {}'.format(self.script_file)) self.output = None self.command = None self.on_target_script_file = None
def __init__(self, target, **kwargs): super(Workload, self).__init__(target, **kwargs) self.asset_files = [] self.deployed_assets = [] supported_platforms = getattr(self, 'supported_platforms', []) if supported_platforms and self.target.os not in supported_platforms: msg = 'Supported platforms for "{}" are "{}", attempting to run on "{}"' raise WorkloadError(msg.format(self.name, ' '.join(self.supported_platforms), self.target.os))
def initialize(self, context): super(Geekbench, self).initialize(context) self.gui.uiauto_params['version'] = self.version self.gui.uiauto_params['loops'] = self.loops self.gui.uiauto_params['is_corporate'] = self.is_corporate self.gui.timeout = self.timeout if not self.disable_update_result and not self.target.is_rooted: raise WorkloadError( 'Geekbench workload requires root to collect results. ' 'You can set disable_update_result=True in the workload params ' 'to run without collecting results.')
def pull_apk(self, package): if not self.target.package_is_installed(package): message = 'Cannot retrieve "{}" as not installed on Target' raise WorkloadError(message.format(package)) package_info = self.target.get_package_info(package) apk_name = self._get_package_name(package_info.apk_path) host_path = os.path.join(self.owner.dependencies_directory, apk_name) with atomic_write_path(host_path) as at_path: self.target.pull(package_info.apk_path, at_path, timeout=self.install_timeout) return host_path
def _generate_workgen_config(self, user_file, output_directory): output_file = os.path.join(output_directory, 'unkind.json') # use workgen dry run option to generate a use case # file with proper JSON grammar on host first try: check_output('python3 {} -d -o {} {}'.format( self.workgen_script, output_file, user_file), shell=True) except CalledProcessError as e: message = 'Could not generate config using workgen, got "{}"' raise WorkloadError(message.format(e)) return output_file
def install_apk(self, context): # pylint: disable=unused-argument output = self.target.install_apk(self.apk_file, self.install_timeout, replace=True, allow_downgrade=True) if 'Failure' in output: if 'ALREADY_EXISTS' in output: msg = 'Using already installed APK (did not uninstall properly?)' self.logger.warning(msg) else: raise WorkloadError(output) else: self.logger.debug(output)
def start_activity(self): if not self.activity: cmd = 'am start -W {}'.format(self.apk_info.package) else: cmd = 'am start -W -n {}/{}'.format(self.apk_info.package, self.activity) output = self.target.execute(cmd) if 'Error:' in output: # this will dismiss any error dialogs self.target.execute('am force-stop {}'.format(self.apk_info.package)) raise WorkloadError(output) self.logger.debug(output)
def setup(self, context): """ Perform the setup necessary to run the workload, such as copying the necessary files to the device, configuring the environments, etc. This is also the place to perform any on-device checks prior to attempting to execute the workload. """ # pylint: disable=unused-argument if self.requires_network and not self.target.is_network_connected(): raise WorkloadError( 'Workload "{}" requires internet. Target does not appear ' 'to be connected to the internet.'.format(self.name))
def update_output(self, context): super(Speedometer, self).update_output(context) result = None logcat_file = context.get_artifact_path('logcat') with open(logcat_file) as fh: for line in fh: match = self.regex.search(line) if match: result = float(match.group(1)) if result is not None: context.add_metric('Speedometer Score', result, 'Runs per minute', lower_is_better=False) else: raise WorkloadError("The Speedometer workload has failed. No score was obtainable.")
def _check_revent_files(self): if not self.revent_run_file: # pylint: disable=too-few-format-args message = '{0}.run.revent file does not exist, ' \ 'Please provide one for your target, {0}' raise WorkloadError(message.format(self.target.model)) self.target.push(self.revent_run_file, self.on_target_run_revent) if self.revent_setup_file: self.target.push(self.revent_setup_file, self.on_target_setup_revent) if self.revent_extract_results_file: self.target.push(self.revent_extract_results_file, self.on_target_extract_results_revent) if self.revent_teardown_file: self.target.push(self.revent_teardown_file, self.on_target_teardown_revent)
def run(self, context): # All we need to do is # - start the activity, # - then use the JbRunMonitor to wait until the benchmark reports on # logcat that it is finished, # - pull the result database file. result = self.target.execute(self.command) if 'FAILURE' in result: raise WorkloadError(result) else: self.logger.debug(result) self.monitor.wait_for(REGEXPS['start'], timeout=30) self.logger.info('Detected Jankbench start') self.monitor.wait_for(REGEXPS['done'], timeout=300*self.times)
def update_output(self, context): expected_results = len(self.regex_matches[self.major_version]) zf = zipfile.ZipFile(os.path.join(context.output_directory, self.result_file), 'r').read('Result.xml') if sys.version_info[0] == 3: zf = zf.decode(sys.stdout.encoding) for line in zf.split('\n'): for regex in self.regex_matches[self.major_version]: match = regex.search(line) if match: scores = float(match.group(1)) entry = regex.pattern entry = entry[:-9] context.add_metric(entry, scores, lower_is_better=False) expected_results -= 1 if expected_results > 0: msg = "The PCMark workload has failed. Expected {} scores, Detected {} scores." raise WorkloadError(msg.format(len(self.regex_matches[self.major_version]), expected_results))
def init_resources(self, context): super(Gmail, self).init_resources(context) if self.target.get_sdk_version( ) >= 24 and 'com.google.android.apps.photos' not in self.target.list_packages( ): raise WorkloadError( 'gmail workload requires Google Photos to be installed for Android N onwards' ) # Allows for getting working directory regardless if path ends with a '/' work_dir = self.target.working_directory work_dir = work_dir if work_dir[-1] != os.sep else work_dir[:-1] self.gui.uiauto_params['workdir_name'] = self.target.path.basename( work_dir) self.gui.uiauto_params['recipient'] = self.recipient self.gui.uiauto_params['offline_mode'] = self.offline_mode # Only accept certain image formats if os.path.splitext( self.test_image.lower())[1] not in ['.jpg', '.jpeg', '.png']: raise ValidationError('{} must be a JPEG or PNG file'.format( self.test_image))
def update_output(self, context): super(Androbench, self).update_output(context) expected_results = len(self.regex_matches) logcat_file = context.get_artifact_path('logcat') with open(logcat_file, errors='replace') as fh: for line in fh: for regex in self.regex_matches: match = regex.search(line) if match: result = float(match.group(1)) entry = regex.pattern.rsplit(None, 1)[0] context.add_metric(entry, result, 'MB/s', lower_is_better=False) expected_results -= 1 if expected_results > 0: msg = "The Androbench workload has failed. Expected {} scores, Detected {} scores." raise WorkloadError( msg.format(len(self.regex_matches), expected_results))
def non_root_update_output(self, context): failed = [] logcat_file = context.get_artifact_path('logcat') with open(logcat_file, errors='replace') as fh: iteration_result_regex = re.compile( "VELLAMO RESULT: (Browser|Metal|Multicore) (\d+)") for line in fh: if 'VELLAMO ERROR:' in line: msg = "Browser crashed during benchmark, results may not be accurate" self.logger.warning(msg) result = iteration_result_regex.findall(line) if result: for (metric, score) in result: if not score: failed.append(metric) else: context.add_metric(metric, score) if failed: raise WorkloadError( "The following benchmark groups failed: {}".format( ", ".join(failed)))
def extract_results(self, context): remote_zip_path = re.match(self.regexps['result'], self.output).group('path') local_zip_path = os.path.join(context.output_directory, self.target.path.basename(remote_zip_path)) self.logger.info('pulling {} -> {}'.format(remote_zip_path, local_zip_path)) self.target.pull(remote_zip_path, local_zip_path, as_root=True) with ZipFile(local_zip_path, 'r') as archive: archive.extractall(context.output_directory) xml_path = os.path.join(context.output_directory, 'Result.xml') if not os.path.exists(xml_path): raise WorkloadError("PCMark results .zip didn't contain Result.xml") context.add_artifact('pcmark_result_xml', xml_path, 'data') # Fetch workloads names and scores score_regex = re.compile('\s*<result_Pcma(?P<name>.*)Score>(?P<score>[0-9]*)<') with open(xml_path) as f: for line in f: match = score_regex.match(line) if match: metric_name = 'pcmark_{}'.format(match.group('name')) context.add_metric(metric_name, match.group('score'))