class MonsoonBackend(EnergyInstrumentBackend): name = 'monsoon' description = """ Monsoon Solutions power monitor To use this instrument, you need to install the monsoon.py script available from the Android Open Source Project. As of May 2017 this is under the CTS repository: https://android.googlesource.com/platform/cts/+/master/tools/utils/monsoon.py Collects power measurements only, from a selection of two channels, the USB passthrough channel and the main output channel. """ parameters = [ Parameter('monsoon_bin', default=which('monsoon.py'), description=""" Path to monsoon.py executable. If not provided, ``PATH`` is searched. """), Parameter('tty_device', default='/dev/ttyACM0', description=""" TTY device to use to communicate with the Power Monitor. If not provided, /dev/ttyACM0 is used. """) ] instrument = MonsoonInstrument
class YoutubePlayback(ApkWorkload): """ Simple Youtube video playback This triggers a video streaming playback on Youtube. Unlike the more featureful "youtube" workload, this performs no other action that starting the video via an intent and then waiting for a certain amount of playback time. This is therefore only useful when you are confident that the content on the end of the provided URL is stable - that means the video should have no advertisements attached. """ name = 'youtube_playback' package_names = ['com.google.android.youtube'] action = 'android.intent.action.VIEW' parameters = [ Parameter('video_url', default='https://www.youtube.com/watch?v=YE7VzlLtp-4', description='URL of video to play'), Parameter('duration', kind=int, default=20, description='Number of seconds of video to play'), ] def setup(self, context): super(YoutubePlayback, self).setup(context) self.command = 'am start -a {} {}'.format(self.action, self.video_url) def run(self, context): self.target.execute(self.command) time.sleep(self.duration)
class TargzProcessor(OutputProcessor): name = 'targz' description = ''' Create a tarball of the output directory. This will create a gzip-compressed tarball of the output directory. By default, it will be created at the same level and will have the same name as the output directory but with a .tar.gz extensions. ''' parameters = [ Parameter('outfile', description=''' The name of the output file to be used. If this is not an absolute path, the file will be created realtive to the directory in which WA was invoked. If this contains subdirectories, they must already exist. The name may contain named format specifiers. Any of the ``RunInfo`` fields can be named, resulting in the value of that filed (e.g. ``'start_time'``) being formatted into the tarball name. By default, the output file will be created at the same level, share the name of the WA output directory (but with .tar.gz extension). '''), Parameter('delete-output', kind=bool, default=False, description=''' if set to ``True``, WA output directory will be deleted after the tarball is created. '''), ] def initialize(self, context): if self.delete_output: self.logger.debug('Registering RUN_FINALIZED handler.') signal.connect(self.delete_output_directory, signal.RUN_FINALIZED, priority=-100) def export_run_output(self, run_output, target_info): # pylint: disable=unused-argument if self.outfile: outfile_path = self.outfile.format(**run_output.info.to_pod()) else: outfile_path = run_output.basepath.rstrip('/') + '.tar.gz' self.logger.debug('Creating {}'.format(outfile_path)) with tarfile.open(outfile_path, 'w:gz') as tar: tar.add(run_output.basepath) def delete_output_directory(self, context): self.logger.debug('Deleting output directory') shutil.rmtree(context.run_output.basepath)
class EnergyProbeBackend(EnergyInstrumentBackend): name = 'energy_probe' description = """ Arm Energy Probe caiman version This backend relies on caiman utility: https://github.com/ARM-software/caiman For more information about Arm Energy Probe please see https://developer.arm.com/products/software-development-tools/ds-5-development-studio/streamline/arm-energy-probe """ parameters = [ Parameter('resistor_values', kind=list_of_ints, description=""" The values of resistors (in Ohms) across which the voltages are measured on. """), Parameter('labels', kind=list_of_strings, description=""" 'List of port labels. If specified, the length of the list must match the length of ``resistor_values``. """), Parameter('device_entry', kind=str, default='/dev/ttyACM0', description=""" Path to /dev entry for the energy probe (it should be /dev/ttyACMx) """), Parameter('keep_raw', kind=bool, default=False, description=""" If set to ``True``, this will prevent the raw files obtained from the device before processing from being deleted (this is maily used for debugging). """), ] instrument = EnergyProbeInstrument def validate_parameters(self, params): if not params.get('resistor_values'): raise ConfigError( 'Mandatory parameter "resistor_values" is not set.') if params.get('labels'): if len(params.get('labels')) != len(params.get('resistor_values')): msg = 'Number of Energy Probe port labels does not match the number of resistor values.' raise ConfigError(msg)
class ArmEnergyProbeBackend(EnergyInstrumentBackend): name = 'arm_energy_probe' description = """ Arm Energy Probe arm-probe version An alternative Arm Energy Probe backend that relies on arm-probe utility: https://git.linaro.org/tools/arm-probe.git For more information about Arm Energy Probe please see https://developer.arm.com/products/software-development-tools/ds-5-development-studio/streamline/arm-energy-probe """ parameters = [ Parameter('config_file', kind=str, description=""" Path to config file of the AEP """), Parameter('keep_raw', kind=bool, default=False, description=""" If set to ``True``, this will prevent the raw files obtained from the device before processing from being deleted (this is maily used for debugging). """), ] instrument = ArmEnergyProbeInstrument def get_instruments(self, target, metadir, **kwargs): """ Get a dict mapping device keys to an Instruments Typically there is just a single device/instrument, in which case the device key is arbitrary. """ shutil.copy(self.config_file, metadir) return {None: self.instrument(target, **kwargs)} def validate_parameters(self, params): if not params.get('config_file'): raise ConfigError('Mandatory parameter "config_file" is not set.') self.config_file = params.get('config_file') if not os.path.exists(self.config_file): raise ConfigError('"config_file" does not exist.')
class ShellScript(Workload): name = 'shellscript' description = """ Runs an arbitrary shellscript on the target. """ parameters = [ Parameter('script_file', mandatory=True, description=''' The path (on the host) to the shell script file. This must be an absolute path (though it may contain ~). '''), Parameter('argstring', default='', description='A string that should contain arguments passed to the script.'), Parameter('as_root', kind=bool, default=False, description='Specify whether the script should be run as root.'), Parameter('timeout', kind=int, default=60, description='Timeout, in seconds, for the script run time.'), ] def initialize(self, context): if self.as_root and not self.target.is_rooted: raise WorkloadError('Cannot run script as root -- target appears to be unrooted.') self.script_file = os.path.expanduser(self.script_file) if not os.path.isfile(self.script_file): raise ConfigError('Can\'t access file (is the path correct?): {}'.format(self.script_file)) self.output = None self.command = None self.on_target_script_file = None def setup(self, context): self.on_target_script_file = self.target.get_workpath(os.path.basename(self.script_file)) self.target.push(self.script_file, self.on_target_script_file) self.command = 'sh {} {}'.format(self.on_target_script_file, self.argstring) def run(self, context): self.output = self.target.execute(self.command, timeout=self.timeout, as_root=self.as_root) def extract_results(self, context): with open(os.path.join(context.output_directory, 'output.txt'), 'w') as wfh: wfh.write(self.output) def teardown(self, context): if self.cleanup_assets: self.target.remove(self.on_target_script_file)
class Gfxbench(ApkUiautoWorkload): name = 'gfxbench-corporate' package_names = ['net.kishonti.gfxbench.gl.v50000.corporate'] clear_data_on_reset = False regex_matches = [ re.compile(r'Car Chase score (.+)'), re.compile(r'Car Chase Offscreen score (.+)'), re.compile(r'Manhattan 3.1 score (.+)'), re.compile(r'1080p Manhattan 3.1 Offscreen score (.+)'), re.compile(r'1440p Manhattan 3.1 Offscreen score (.+)'), re.compile(r'Tessellation score (.+)'), re.compile(r'Tessellation Offscreen score (.+)') ] score_regex = re.compile(r'.*?([\d.]+).*') description = ''' Execute a subset of graphical performance benchmarks Test description: 1. Open the gfxbench application 2. Execute Car Chase, Manhattan and Tessellation benchmarks ''' parameters = [ Parameter('timeout', kind=int, default=3600, description=('Timeout for an iteration of the benchmark.')), ] def __init__(self, target, **kwargs): super(Gfxbench, self).__init__(target, **kwargs) self.gui.timeout = self.timeout def update_output(self, context): super(Gfxbench, self).update_output(context) expected_results = len(self.regex_matches) logcat_file = context.get_artifact_path('logcat') with open(logcat_file) as fh: for line in fh: for regex in self.regex_matches: match = regex.search(line) # Check if we have matched the score string in logcat if match: score_match = self.score_regex.search(match.group(1)) # Check if there is valid number found for the score. if score_match: result = float(score_match.group(1)) else: result = 'NaN' entry = regex.pattern.rsplit(None, 1)[0] context.add_metric(entry, result, 'FPS', lower_is_better=False) expected_results -= 1 if expected_results > 0: msg = "The GFXBench workload has failed. Expected {} scores, Detected {} scores." raise WorkloadError( msg.format(len(self.regex_matches), expected_results))
class TheChase(ApkWorkload): name = 'thechase' description = """ The Chase demo showcasing the capabilities of Unity game engine. This demo, is a static video-like game demo, that demonstrates advanced features of the unity game engine. It loops continuously until terminated. """ package_names = ['com.unity3d.TheChase'] install_timeout = 200 view = 'SurfaceView - com.unity3d.TheChase/com.unity3d.player.UnityPlayerNativeActivity' parameters = [ Parameter( 'duration', kind=int, default=70, description= ('Duration, in seconds, note that the demo loops the same (roughly) 60 ' 'second sceene until stopped.')), ] def run(self, context): self.target.sleep(self.duration)
class HomeScreen(Workload): name = 'homescreen' description = """ A workload that goes to the home screen and idles for the the specified duration. """ supported_platforms = ['android'] parameters = [ Parameter( 'duration', kind=int, default=20, description='Specifies the duration, in seconds, of this workload.' ), ] def setup(self, context): self.target.clear_logcat() self.target.execute('input keyevent 3') # press the home key def run(self, context): self.target.sleep(self.duration)
class DynamicFrequencyInstrument(SysfsExtractor): name = 'cpufreq' description = """ Collects dynamic frequency (DVFS) settings before and after workload execution. """ tarname = 'cpufreq.tar.gz' parameters = [ Parameter('paths', mandatory=False, override=True), ] def setup(self, context): self.paths = ['/sys/devices/system/cpu'] if self.use_tmpfs: self.paths.append('/sys/class/devfreq/*' ) # the '*' would cause problems for adb pull. super(DynamicFrequencyInstrument, self).setup(context) def validate(self): super(DynamicFrequencyInstrument, self).validate() if not self.tmpfs_mount_point.endswith('-cpufreq'): # pylint: disable=access-member-before-definition self.tmpfs_mount_point += '-cpufreq'
class Speedometer(UiautoWorkload): name = 'speedometer' regex = re.compile(r'Speedometer Score ([\d.]+)') versions = ['1.0', '2.0'] description = ''' A workload to execute the speedometer web based benchmark Test description: 1. Open browser application 2. Navigate to the speedometer website - http://browserbench.org/Speedometer/ 3. Execute the benchmark ''' parameters = [ Parameter('version', allowed_values=versions, kind=str, default='2.0', description=''' The speedometer version to be used. ''') ] def __init__(self, target, **kwargs): super(Speedometer, self).__init__(target, **kwargs) self.gui.timeout = 1500 self.gui.uiauto_params['version'] = self.version def setup(self, context): super(Speedometer, self).setup(context) url = 'am start -a android.intent.action.VIEW -d http://browserbench.org/Speedometer' + self.version if self.version == '1.0': url = 'am start -a android.intent.action.VIEW -d http://browserbench.org/Speedometer' self.target.execute(url) def update_output(self, context): super(Speedometer, self).update_output(context) result = None logcat_file = context.get_artifact_path('logcat') with open(logcat_file) as fh: for line in fh: match = self.regex.search(line) if match: result = float(match.group(1)) if result is not None: context.add_metric('Speedometer Score', result, 'Runs per minute', lower_is_better=False) else: raise WorkloadError( "The Speedometer workload has failed. No score was obtainable." )
class GeekbenchCorproate(Geekbench): # pylint: disable=too-many-ancestors name = "geekbench-corporate" is_corporate = True requires_network = False supported_versions = ['4.1.0', '5.0.0'] package_names = ['com.primatelabs.geekbench4.corporate'] activity = 'com.primatelabs.geekbench.HomeActivity' parameters = [ Parameter('version', allowed_values=supported_versions, override=True) ]
class Speedometer(ApkUiautoWorkload): name = 'speedometer' package_names = ['com.android.chrome'] regex = re.compile(r'Speedometer Score ([\d.]+)') versions = ['1.0', '2.0'] description = ''' A workload to execute the speedometer web based benchmark Test description: 1. Open chrome 2. Navigate to the speedometer website - http://browserbench.org/Speedometer/ 3. Execute the benchmark known working chrome version 80.0.3987.149 ''' parameters = [ Parameter('speedometer_version', allowed_values=versions, kind=str, default='2.0', description=''' The speedometer version to be used. ''') ] requires_network = True def __init__(self, target, **kwargs): super(Speedometer, self).__init__(target, **kwargs) self.gui.timeout = 1500 self.gui.uiauto_params['version'] = self.speedometer_version def update_output(self, context): super(Speedometer, self).update_output(context) result = None logcat_file = context.get_artifact_path('logcat') with open(logcat_file) as fh: for line in fh: match = self.regex.search(line) if match: result = float(match.group(1)) if result is not None: context.add_metric('Speedometer Score', result, 'Runs per minute', lower_is_better=False) else: raise WorkloadError( "The Speedometer workload has failed. No score was obtainable." )
class DmesgInstrument(Instrument): # pylint: disable=no-member,attribute-defined-outside-init """ Collected dmesg output before and during the run. """ name = 'dmesg' parameters = [ Parameter('loglevel', kind=int, allowed_values=list(range(8)), description='Set loglevel for console output.') ] loglevel_file = '/proc/sys/kernel/printk' def initialize(self, context): # pylint: disable=unused-argument self.need_root = self.target.os == 'android' if self.need_root and not self.target.is_rooted: raise InstrumentError('Need root to collect dmesg on Android') def setup(self, context): if self.loglevel: self.old_loglevel = self.target.read_int(self.loglevel_file) self.target.write_value(self.loglevel_file, self.loglevel, verify=False) self.before_file = _f( os.path.join(context.output_directory, 'dmesg', 'before')) self.after_file = _f( os.path.join(context.output_directory, 'dmesg', 'after')) @slow def start(self, context): with open(self.before_file, 'w') as wfh: wfh.write(self.target.execute('dmesg', as_root=self.need_root)) context.add_artifact('dmesg_before', self.before_file, kind='data') if self.target.is_rooted: self.target.execute('dmesg -c', as_root=True) @slow def stop(self, context): with open(self.after_file, 'w') as wfh: wfh.write(self.target.execute('dmesg', as_root=self.need_root)) context.add_artifact('dmesg_after', self.after_file, kind='data') def teardown(self, context): # pylint: disable=unused-argument if self.loglevel: self.target.write_value(self.loglevel_file, self.old_loglevel, verify=False)
class Uibench(ApkWorkload): name = 'uibench' description = """ Runs a particular activity of the UIBench_ workload suite. The suite is provided by Google as a testbench for the Android UI. .. _UIBench: https://android.googlesource.com/platform/frameworks/base/+/refs/heads/master/tests/UiBench/ """ package_names = ['com.android.test.uibench'] loading_time = 1 parameters = [ Parameter('activity', kind=str, description=""" The UIBench activity to be run. Each activity corresponds to a test. If this parameter is ignored, the application is launched in its main menu. Please note that the available activities vary between versions of UIBench (which follow AOSP versioning) and the availability of the services under test may depend on the version of the target Android. We recommend using the APK of UIBench corresponding to the Android version, enforced through the ``version`` parameter to this workload. """), Parameter('duration', kind=int, default=10, description=""" As activities do not finish, this workload will terminate UIBench after the given duration. """), ] def run(self, context): super(Uibench, self).run(context) self.target.sleep(self.duration)
class ScreenCaptureInstrument(Instrument): name = 'screen_capture' description = """ A simple instrument which captures the screen on the target devices with a user-specified period. Please note that if a too short period is specified, then this instrument will capture the screen as fast as possible, rather than at the specified periodicity. """ parameters = [ Parameter('period', kind=int, default=10, description=""" Period (in seconds) at which to capture the screen on the target. """), ] def __init__(self, target, **kwargs): super(ScreenCaptureInstrument, self).__init__(target, **kwargs) self.collector = None def setup(self, context): # We need to create a directory for the captured screenshots output_path = os.path.join(context.output_directory, "screen-capture") os.mkdir(output_path) self.collector = ScreenCaptureCollector(self.target, self.period) self.collector.set_output(output_path) self.collector.reset() def start(self, context): # pylint: disable=unused-argument self.collector.start() def stop(self, context): # pylint: disable=unused-argument self.collector.stop()
class Gmail(ApkUiautoWorkload): name = 'gmail' package_names = ['com.google.android.gm'] description = ''' A workload to perform standard productivity tasks within Gmail. The workload carries out various tasks, such as creating new emails, attaching images and sending them. Test description: 1. Open Gmail application 2. Click to create New mail 3. Attach an image from the local images folder to the email 4. Enter recipient details in the To field 5. Enter text in the Subject field 6. Enter text in the Compose field 7. Click the Send mail button To run the workload in offline mode, a 'mailstore.tar' file is required. In order to generate such a file, Gmail should first be operated from an Internet-connected environment. After this, the relevant database files can be found in the '/data/data/com.google.android.gm/databases' directory. These files can then be archived to produce a tarball using a command such as ``tar -cvf mailstore.tar -C /path/to/databases .``. The result should then be placed in the '~/.workload_automation/dependencies/gmail/' directory on your local machine, creating this if it does not already exist. Known working APK version: 2019.05.26.252424914.release ''' parameters = [ Parameter('recipient', kind=str, default='*****@*****.**', description=''' The email address of the recipient. Setting a void address will stop any mesage failures clogging up your device inbox '''), Parameter('test_image', kind=str, default='uxperf_1600x1200.jpg', description=''' An image to be copied onto the device that will be attached to the email '''), Parameter('offline_mode', kind=bool, default=False, description=''' If set to ``True``, the workload will execute in offline mode. This mode requires root and makes use of a tarball of email database files 'mailstore.tar' for the email account to be used. This file is extracted directly to the application's 'databases' directory at '/data/data/com.google.android.gm/databases'. '''), ] @property def requires_network(self): return not self.offline_mode @property def requires_rerun(self): # In offline mode we need to restart the application after modifying its data directory return self.offline_mode def __init__(self, target, **kwargs): super(Gmail, self).__init__(target, **kwargs) self.deployable_assets = [self.test_image] if self.offline_mode: self.deployable_assets.append('mailstore.tar') self.cleanup_assets = True def initialize(self, context): super(Gmail, self).initialize(context) if self.offline_mode and not self.target.is_rooted: raise WorkloadError( 'This workload requires root to set up Gmail for offline usage.' ) def init_resources(self, context): super(Gmail, self).init_resources(context) # Allows for getting working directory regardless if path ends with a '/' work_dir = self.target.working_directory work_dir = work_dir if work_dir[-1] != os.sep else work_dir[:-1] self.gui.uiauto_params['workdir_name'] = self.target.path.basename( work_dir) self.gui.uiauto_params['recipient'] = self.recipient self.gui.uiauto_params['offline_mode'] = self.offline_mode self.gui.uiauto_params['test_image'] = self.test_image # Only accept certain image formats if os.path.splitext( self.test_image.lower())[1] not in ['.jpg', '.jpeg', '.png']: raise ValidationError('{} must be a JPEG or PNG file'.format( self.test_image)) def setup_rerun(self): super(Gmail, self).setup_rerun() database_src = self.target.path.join(self.target.working_directory, 'mailstore.tar') database_dst = self.target.path.join( self.target.package_data_directory, self.package, 'databases') existing_mailstores = self.target.path.join(database_dst, 'mailstore.*') owner = self.target.execute("{} stat -c '%u' {}".format( self.target.busybox, database_dst), as_root=True).strip() self.target.execute('{} rm {}'.format(self.target.busybox, existing_mailstores), as_root=True) self.target.execute('{} tar -xvf {} -C {}'.format( self.target.busybox, database_src, database_dst), as_root=True) self.target.execute('{0} chown -R {1}:{1} {2}'.format( self.target.busybox, owner, database_dst), as_root=True)
class Jankbench(ApkWorkload): name = 'jankbench' description = """ Internal Google benchmark for evaluating jank on Android. """ package_names = ['com.android.benchmark'] activity = '.app.RunLocalBenchmarksActivity' results_db_file = 'BenchmarkResults' iteration_regex = re.compile( r'System.out: iteration: (?P<iteration>[0-9]+)') metrics_regex = re.compile( r'System.out: Mean: (?P<mean>[0-9\.]+)\s+JankP: (?P<junk_p>[0-9\.]+)\s+' r'StdDev: (?P<std_dev>[0-9\.]+)\s+Count Bad: (?P<count_bad>[0-9]+)\s+' r'Count Jank: (?P<count_junk>[0-9]+)') valid_test_ids = [ # Order matters -- the index of the id must match what is expected by # the App. 'list_view', 'image_list_view', 'shadow_grid', 'low_hitrate_text', 'high_hitrate_text', 'edit_text', 'overdraw_test', ] parameters = [ Parameter('test_ids', kind=list_or_string, allowed_values=valid_test_ids, description='ID of the jankbench test to be run.'), Parameter('loops', kind=int, default=1, constraint=lambda x: x > 0, aliases=['reps'], description=''' Specifies the number of times the benchmark will be run in a "tight loop", i.e. without performaing setup/teardown inbetween. '''), Parameter('pull_results_db', kind=bool, description=''' Secifies whether an sqlite database with detailed results should be pulled from benchmark app's data. This requires the device to be rooted. This defaults to ``True`` for rooted devices and ``False`` otherwise. '''), Parameter('timeout', kind=int, default=10 * 60, aliases=['run_timeout'], description=""" Time out for workload execution. The workload will be killed if it hasn't completed within this period. """), ] def setup(self, context): super(Jankbench, self).setup(context) if self.pull_results_db is None: self.pull_results_db = self.target.is_rooted elif self.pull_results_db and not self.target.is_rooted: raise ConfigError('pull_results_db set for an unrooted device') if not self.target.is_container: self.target.ensure_screen_is_on() self.command = self._build_command() self.monitor = JankbenchRunMonitor(self.target) self.monitor.start() def run(self, context): result = self.target.execute(self.command, timeout=self.timeout) if 'FAILURE' in result: raise WorkloadError(result) else: self.logger.debug(result) self.target.sleep(DELAY) self.monitor.wait_for_run_end(self.timeout) def extract_results(self, context): self.monitor.stop() if self.pull_results_db: target_file = self.target.path.join( self.target.package_data_directory, self.package, 'databases', self.results_db_file) host_file = os.path.join(context.output_directory, self.results_db_file) self.target.pull(target_file, host_file, as_root=True) context.add_artifact('jankbench-results', host_file, 'data') def update_output(self, context): # NOQA super(Jankbench, self).update_output(context) if self.pull_results_db: self.extract_metrics_from_db(context) else: self.extract_metrics_from_logcat(context) def extract_metrics_from_db(self, context): # pylint: disable=no-self-use dbfile = context.get_artifact_path('jankbench-results') with sqlite3.connect(dbfile) as conn: df = pd.read_sql( 'select name, iteration, total_duration, jank_frame from ui_results', conn) g = df.groupby(['name', 'iteration']) janks = g.jank_frame.sum() janks_pc = janks / g.jank_frame.count() * 100 results = pd.concat([ g.total_duration.mean(), g.total_duration.std(), janks, janks_pc, ], axis=1) results.columns = ['mean', 'std_dev', 'count_jank', 'jank_p'] for test_name, rep in results.index: test_results = results.loc[test_name, rep] for metric, value in test_results.items(): context.add_metric(metric, value, units=None, lower_is_better=True, classifiers={ 'test_name': test_name, 'rep': rep }) def extract_metrics_from_logcat(self, context): metric_names = ['mean', 'junk_p', 'std_dev', 'count_bad', 'count_junk'] logcat_file = context.get_artifact_path('logcat') with open(logcat_file, errors='replace') as fh: run_tests = copy(self.test_ids or self.valid_test_ids) current_iter = None current_test = None for line in fh: match = self.iteration_regex.search(line) if match: if current_iter is not None: msg = 'Did not see results for iteration {} of {}' self.logger.warning( msg.format(current_iter, current_test)) current_iter = int(match.group('iteration')) if current_iter == 0: try: current_test = run_tests.pop(0) except IndexError: self.logger.warning( 'Encountered an iteration for an unknown test.' ) current_test = 'unknown' continue match = self.metrics_regex.search(line) if match: if current_iter is None: self.logger.warning( 'Encountered unexpected metrics (no iteration)') continue for name in metric_names: value = numeric(match.group(name)) context.add_metric(name, value, units=None, lower_is_better=True, classifiers={ 'test_id': current_test, 'rep': current_iter }) current_iter = None def _build_command(self): command_params = [] if self.test_ids: test_idxs = [ str(self.valid_test_ids.index(i)) for i in self.test_ids ] command_params.append( '--eia com.android.benchmark.EXTRA_ENABLED_BENCHMARK_IDS {}'. format(','.join(test_idxs))) if self.loops: command_params.append( '--ei com.android.benchmark.EXTRA_RUN_COUNT {}'.format( self.loops)) return 'am start -W -S -n {}/{} {}'.format(self.package, self.activity, ' '.join(command_params))
class PostgresqlResultProcessor(OutputProcessor): name = 'postgres' description = """ Stores results in a Postgresql database. The structure of this database can easily be understood by examining the postgres_schema.sql file (the schema used to generate it): {} """.format(os.path.join(POSTGRES_SCHEMA_DIR, 'postgres_schema.sql')) parameters = [ Parameter('username', default='postgres', description=""" This is the username that will be used to connect to the Postgresql database. Note that depending on whether the user has privileges to modify the database (normally only possible on localhost), the user may only be able to append entries. """), Parameter('password', default=None, description=""" The password to be used to connect to the specified database with the specified username. """), Parameter('dbname', default='wa', description=""" Name of the database that will be created or added to. Note, to override this, you can specify a value in your user wa configuration file. """), Parameter('host', kind=str, default='localhost', description=""" The host where the Postgresql server is running. The default is localhost (i.e. the machine that wa is running on). This is useful for complex systems where multiple machines may be executing workloads and uploading their results to a remote, centralised database. """), Parameter('port', kind=str, default='5432', description=""" The port the Postgresql server is running on, on the host. The default is Postgresql's default, so do not change this unless you have modified the default port for Postgresql. """), ] # Commands sql_command = { "create_run": "INSERT INTO Runs (oid, event_summary, basepath, status, timestamp, run_name, project, project_stage, retry_on_status, max_retries, bail_on_init_failure, allow_phone_home, run_uuid, start_time, metadata, state, _pod_version, _pod_serialization_version) " "VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)", "update_run": "UPDATE Runs SET event_summary=%s, status=%s, timestamp=%s, end_time=%s, duration=%s, state=%s WHERE oid=%s;", "create_job": "INSERT INTO Jobs (oid, run_oid, status, retry, label, job_id, iterations, workload_name, metadata, _pod_version, _pod_serialization_version) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s);", "create_target": "INSERT INTO Targets (oid, run_oid, target, cpus, os, os_version, hostid, hostname, abi, is_rooted, kernel_version, kernel_release, kernel_sha1, kernel_config, sched_features, page_size_kb, screen_resolution, prop, android_id, _pod_version, _pod_serialization_version) " "VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)", "create_event": "INSERT INTO Events (oid, run_oid, job_oid, timestamp, message, _pod_version, _pod_serialization_version) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s", "create_artifact": "INSERT INTO Artifacts (oid, run_oid, job_oid, name, large_object_uuid, description, kind, _pod_version, _pod_serialization_version) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)", "create_metric": "INSERT INTO Metrics (oid, run_oid, job_oid, name, value, units, lower_is_better, _pod_version, _pod_serialization_version) VALUES (%s, %s, %s, %s, %s, %s , %s, %s, %s)", "create_augmentation": "INSERT INTO Augmentations (oid, run_oid, name) VALUES (%s, %s, %s)", "create_classifier": "INSERT INTO Classifiers (oid, artifact_oid, metric_oid, job_oid, run_oid, key, value) VALUES (%s, %s, %s, %s, %s, %s, %s)", "create_parameter": "INSERT INTO Parameters (oid, run_oid, job_oid, augmentation_oid, resource_getter_oid, name, value, value_type, type) " "VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)", "create_resource_getter": "INSERT INTO Resource_Getters (oid, run_oid, name) VALUES (%s, %s, %s)", "create_job_aug": "INSERT INTO Jobs_Augs (oid, job_oid, augmentation_oid) VALUES (%s, %s, %s)", "create_large_object": "INSERT INTO LargeObjects (oid, lo_oid) VALUES (%s, %s)" } # Lists to track which run-related items have already been added metrics_already_added = [] # Dicts needed so that jobs can look up ids artifacts_already_added = {} augmentations_already_added = {} # Status bits (flags) first_job_run = True def __init__(self, *args, **kwargs): super(PostgresqlResultProcessor, self).__init__(*args, **kwargs) self.conn = None self.cursor = None self.run_uuid = None self.target_uuid = None def initialize(self, context): if not psycopg2: raise ImportError( 'The psycopg2 module is required for the ' + 'Postgresql Output Processor: {}'.format(import_error_msg)) # N.B. Typecasters are for postgres->python and adapters the opposite self.connect_to_database() self.cursor = self.conn.cursor() self.verify_schema_versions() # Register the adapters and typecasters for enum types self.cursor.execute("SELECT NULL::status_enum") status_oid = self.cursor.description[0][1] self.cursor.execute("SELECT NULL::param_enum") param_oid = self.cursor.description[0][1] LEVEL = psycopg2.extensions.new_type((status_oid, ), "LEVEL", cast_level) psycopg2.extensions.register_type(LEVEL) PARAM = psycopg2.extensions.new_type((param_oid, ), "PARAM", cast_vanilla) psycopg2.extensions.register_type(PARAM) psycopg2.extensions.register_adapter(level, return_as_is(adapt_level)) psycopg2.extensions.register_adapter(ListOfLevel, adapt_ListOfX(adapt_level)) psycopg2.extensions.register_adapter(KernelVersion, adapt_vanilla) psycopg2.extensions.register_adapter(CpuInfo, adapt_vanilla) psycopg2.extensions.register_adapter(collections.OrderedDict, extras.Json) psycopg2.extensions.register_adapter(dict, extras.Json) psycopg2.extensions.register_adapter( KernelConfig, create_iterable_adapter(2, explicit_iterate=True)) # Register ready-made UUID type adapter extras.register_uuid() # Insert a run_uuid which will be globally accessible during the run self.run_uuid = uuid.UUID(str(uuid.uuid4())) run_output = context.run_output retry_on_status = ListOfLevel(run_output.run_config.retry_on_status) self.cursor.execute( self.sql_command['create_run'], ( self.run_uuid, run_output.event_summary, run_output.basepath, run_output.status, run_output.state.timestamp, run_output.info.run_name, run_output.info.project, run_output.info.project_stage, retry_on_status, run_output.run_config.max_retries, run_output.run_config.bail_on_init_failure, run_output.run_config.allow_phone_home, run_output.info.uuid, run_output.info.start_time, run_output.metadata, json.dumps(run_output.state.to_pod()), run_output.result._pod_version, # pylint: disable=protected-access run_output.result._pod_serialization_version, # pylint: disable=protected-access )) self.target_uuid = uuid.uuid4() target_info = context.target_info target_pod = target_info.to_pod() self.cursor.execute( self.sql_command['create_target'], ( self.target_uuid, self.run_uuid, target_pod['target'], target_pod['cpus'], target_pod['os'], target_pod['os_version'], target_pod['hostid'], target_pod['hostname'], target_pod['abi'], target_pod['is_rooted'], # Important caveat: kernel_version is the name of the column in the Targets table # However, this refers to kernel_version.version, not to kernel_version as a whole target_pod['kernel_version'], target_pod['kernel_release'], target_info.kernel_version.sha1, target_info.kernel_config, target_pod['sched_features'], target_pod['page_size_kb'], # Android Specific list(target_pod.get('screen_resolution', [])), target_pod.get('prop'), target_pod.get('android_id'), target_pod.get('pod_version'), target_pod.get('pod_serialization_version'), )) # Commit cursor commands self.conn.commit() def export_job_output(self, job_output, target_info, run_output): # pylint: disable=too-many-branches, too-many-statements, too-many-locals, unused-argument ''' Run once for each job to upload information that is updated on a job by job basis. ''' job_uuid = uuid.uuid4() # Create a new job self.cursor.execute( self.sql_command['create_job'], ( job_uuid, self.run_uuid, job_output.status, job_output.retry, job_output.label, job_output.id, job_output.iteration, job_output.spec.workload_name, job_output.metadata, job_output.spec._pod_version, # pylint: disable=protected-access job_output.spec._pod_serialization_version, # pylint: disable=protected-access )) for classifier in job_output.classifiers: classifier_uuid = uuid.uuid4() self.cursor.execute( self.sql_command['create_classifier'], (classifier_uuid, None, None, job_uuid, None, classifier, job_output.classifiers[classifier])) # Update the run table and run-level parameters self.cursor.execute( self.sql_command['update_run'], (run_output.event_summary, run_output.status, run_output.state.timestamp, run_output.info.end_time, None, json.dumps(run_output.state.to_pod()), self.run_uuid)) for classifier in run_output.classifiers: classifier_uuid = uuid.uuid4() self.cursor.execute( self.sql_command['create_classifier'], (classifier_uuid, None, None, None, None, self.run_uuid, classifier, run_output.classifiers[classifier])) self.sql_upload_artifacts(run_output, record_in_added=True) self.sql_upload_metrics(run_output, record_in_added=True) self.sql_upload_augmentations(run_output) self.sql_upload_resource_getters(run_output) self.sql_upload_events(job_output, job_uuid=job_uuid) self.sql_upload_artifacts(job_output, job_uuid=job_uuid) self.sql_upload_metrics(job_output, job_uuid=job_uuid) self.sql_upload_job_augmentations(job_output, job_uuid=job_uuid) self.sql_upload_parameters("workload", job_output.spec.workload_parameters, job_uuid=job_uuid) self.sql_upload_parameters("runtime", job_output.spec.runtime_parameters, job_uuid=job_uuid) self.conn.commit() def export_run_output(self, run_output, target_info): # pylint: disable=unused-argument, too-many-locals ''' A final export of the RunOutput that updates existing parameters and uploads ones which are only generated after jobs have run. ''' if not self.cursor: # Database did not connect correctly. return # Update the job statuses following completion of the run for job in run_output.jobs: job_id = job.id job_status = job.status self.cursor.execute( "UPDATE Jobs SET status=%s WHERE job_id=%s and run_oid=%s", (job_status, job_id, self.run_uuid)) run_uuid = self.run_uuid # Update the run entry after jobs have completed run_info_pod = run_output.info.to_pod() run_state_pod = run_output.state.to_pod() sql_command_update_run = self.sql_command['update_run'] self.cursor.execute(sql_command_update_run, ( run_output.event_summary, run_output.status, run_info_pod['start_time'], run_info_pod['end_time'], run_info_pod['duration'], json.dumps(run_state_pod), run_uuid, )) self.sql_upload_events(run_output) self.sql_upload_artifacts(run_output, check_uniqueness=True) self.sql_upload_metrics(run_output, check_uniqueness=True) self.sql_upload_augmentations(run_output) self.conn.commit() # Upload functions for use with both jobs and runs def sql_upload_resource_getters(self, output_object): for resource_getter in output_object.run_config.resource_getters: resource_getter_uuid = uuid.uuid4() self.cursor.execute(self.sql_command['create_resource_getter'], ( resource_getter_uuid, self.run_uuid, resource_getter, )) self.sql_upload_parameters( 'resource_getter', output_object.run_config.resource_getters[resource_getter], owner_id=resource_getter_uuid, ) def sql_upload_events(self, output_object, job_uuid=None): for event in output_object.events: event_uuid = uuid.uuid4() self.cursor.execute( self.sql_command['create_event'], ( event_uuid, self.run_uuid, job_uuid, event.timestamp, event.message, event._pod_version, # pylint: disable=protected-access event._pod_serialization_version, # pylint: disable=protected-access )) def sql_upload_job_augmentations(self, output_object, job_uuid=None): ''' This is a table which links the uuids of augmentations to jobs. Note that the augmentations table is prepopulated, leading to the necessity of an augmentaitions_already_added dictionary, which gives us the corresponding uuids. Augmentations which are prefixed by ~ are toggled off and not part of the job, therefore not added. ''' for augmentation in output_object.spec.augmentations: if augmentation.startswith('~'): continue augmentation_uuid = self.augmentations_already_added[augmentation] job_aug_uuid = uuid.uuid4() self.cursor.execute(self.sql_command['create_job_aug'], ( job_aug_uuid, job_uuid, augmentation_uuid, )) def sql_upload_augmentations(self, output_object): for augmentation in output_object.augmentations: if augmentation.startswith( '~') or augmentation in self.augmentations_already_added: continue augmentation_uuid = uuid.uuid4() self.cursor.execute(self.sql_command['create_augmentation'], ( augmentation_uuid, self.run_uuid, augmentation, )) self.sql_upload_parameters( 'augmentation', output_object.run_config.augmentations[augmentation], owner_id=augmentation_uuid, ) self.augmentations_already_added[augmentation] = augmentation_uuid def sql_upload_metrics(self, output_object, record_in_added=False, check_uniqueness=False, job_uuid=None): for metric in output_object.metrics: if metric in self.metrics_already_added and check_uniqueness: continue metric_uuid = uuid.uuid4() self.cursor.execute( self.sql_command['create_metric'], ( metric_uuid, self.run_uuid, job_uuid, metric.name, metric.value, metric.units, metric.lower_is_better, metric._pod_version, # pylint: disable=protected-access metric._pod_serialization_version, # pylint: disable=protected-access )) for classifier in metric.classifiers: classifier_uuid = uuid.uuid4() self.cursor.execute(self.sql_command['create_classifier'], ( classifier_uuid, None, metric_uuid, None, None, classifier, metric.classifiers[classifier], )) if record_in_added: self.metrics_already_added.append(metric) def sql_upload_artifacts(self, output_object, record_in_added=False, check_uniqueness=False, job_uuid=None): ''' Uploads artifacts to the database. record_in_added will record the artifacts added in artifacts_aleady_added check_uniqueness will ensure artifacts in artifacts_already_added do not get added again ''' for artifact in output_object.artifacts: if artifact in self.artifacts_already_added and check_uniqueness: self.logger.debug( 'Skipping uploading {} as already added'.format(artifact)) continue if artifact in self.artifacts_already_added: self._sql_update_artifact(artifact, output_object) else: self._sql_create_artifact(artifact, output_object, record_in_added, job_uuid) def sql_upload_parameters(self, parameter_type, parameter_dict, owner_id=None, job_uuid=None): # Note, currently no augmentation parameters are workload specific, but in the future # this may change augmentation_id = None resource_getter_id = None if parameter_type not in [ 'workload', 'resource_getter', 'augmentation', 'runtime' ]: # boot parameters are not yet implemented # device parameters are redundant due to the targets table raise NotImplementedError( "{} is not a valid parameter type.".format(parameter_type)) if parameter_type == "resource_getter": resource_getter_id = owner_id elif parameter_type == "augmentation": augmentation_id = owner_id for parameter in parameter_dict: parameter_uuid = uuid.uuid4() self.cursor.execute(self.sql_command['create_parameter'], ( parameter_uuid, self.run_uuid, job_uuid, augmentation_id, resource_getter_id, parameter, json.dumps(parameter_dict[parameter]), str(type(parameter_dict[parameter])), parameter_type, )) def connect_to_database(self): dsn = "dbname={} user={} password={} host={} port={}".format( self.dbname, self.username, self.password, self.host, self.port) try: self.conn = connect(dsn=dsn) except Psycopg2Error as e: raise OutputProcessorError( "Database error, if the database doesn't exist, " + "please use 'wa create database' to create the database: {}". format(e)) def execute_sql_line_by_line(self, sql): cursor = self.conn.cursor() for line in sql.replace('\n', "").replace(";", ";\n").split("\n"): if line and not line.startswith('--'): cursor.execute(line) cursor.close() self.conn.commit() self.conn.reset() def verify_schema_versions(self): local_schema_version, db_schema_version = get_schema_versions( self.conn) if local_schema_version != db_schema_version: self.cursor.close() self.cursor = None self.conn.commit() self.conn.reset() msg = 'The current database schema is v{} however the local ' \ 'schema version is v{}. Please update your database ' \ 'with the create command' raise OutputProcessorError( msg.format(db_schema_version, local_schema_version)) def _sql_write_lobject(self, source, lobject): with open(source) as lobj_file: lobj_data = lobj_file.read() if len(lobj_data) > 50000000: # Notify if LO inserts larger than 50MB self.logger.debug("Inserting large object of size {}".format( len(lobj_data))) lobject.write(lobj_data) self.conn.commit() def _sql_update_artifact(self, artifact, output_object): self.logger.debug('Updating artifact: {}'.format(artifact)) lobj = self.conn.lobject(oid=self.artifacts_already_added[artifact], mode='w') self._sql_write_lobject( os.path.join(output_object.basepath, artifact.path), lobj) def _sql_create_artifact(self, artifact, output_object, record_in_added=False, job_uuid=None): self.logger.debug('Uploading artifact: {}'.format(artifact)) artifact_uuid = uuid.uuid4() lobj = self.conn.lobject() loid = lobj.oid large_object_uuid = uuid.uuid4() self._sql_write_lobject( os.path.join(output_object.basepath, artifact.path), lobj) self.cursor.execute(self.sql_command['create_large_object'], ( large_object_uuid, loid, )) self.cursor.execute( self.sql_command['create_artifact'], ( artifact_uuid, self.run_uuid, job_uuid, artifact.name, large_object_uuid, artifact.description, str(artifact.kind), artifact._pod_version, # pylint: disable=protected-access artifact._pod_serialization_version, # pylint: disable=protected-access )) for classifier in artifact.classifiers: classifier_uuid = uuid.uuid4() self.cursor.execute(self.sql_command['create_classifier'], ( classifier_uuid, artifact_uuid, None, None, None, classifier, artifact.classifiers[classifier], )) if record_in_added: self.artifacts_already_added[artifact] = loid
class GoogleMaps(ApkUiautoWorkload): name = 'googlemaps' description = ''' A workload to perform standard navigation tasks with Google Maps. This workload searches for known locations, pans and zooms around the map, and follows driving directions along a route. To run the workload in offline mode, ``databases.tar`` and ``files.tar`` archives are required. In order to generate these files, Google Maps should first be operated from an Internet-connected environment, and a region around Cambridge, England should be downloaded for offline use. This region must include the landmarks used in the UIAutomator program, which include Cambridge train station and Corpus Christi college. Following this, the files of interest can be found in the ``databases`` and ``files`` subdirectories of the ``/data/data/com.google.android.apps.maps/`` directory. The contents of these subdirectories can be archived into tarballs using commands such as ``tar -cvf databases.tar -C /path/to/databases .``. These ``databases.tar`` and ``files.tar`` archives should then be placed in the ``~/.workload_automation/dependencies/googlemaps`` directory on your local machine, creating this if it does not already exist. Known working APK version: 10.19.1 ''' package_names = ['com.google.android.apps.maps'] parameters = [ Parameter('offline_mode', kind=bool, default=False, description=''' If set to ``True``, the workload will execute in offline mode. This mode requires root and makes use of a tarball of database files ``databases.tar`` and a tarball of auxiliary files ``files.tar``. These tarballs are extracted directly to the application's ``databases`` and ``files`` directories respectively in ``/data/data/com.google.android.apps.maps/``. '''), ] @property def requires_network(self): return not self.offline_mode @property def requires_rerun(self): # In offline mode we need to restart the application after modifying its data directory return self.offline_mode def __init__(self, target, **kwargs): super(GoogleMaps, self).__init__(target, **kwargs) if self.offline_mode: self.deployable_assets = ['databases.tar', 'files.tar'] self.cleanup_assets = True def initialize(self, context): super(GoogleMaps, self).initialize(context) if self.offline_mode and not self.target.is_rooted: raise WorkloadError( 'This workload requires root to set up Google Maps for offline usage.' ) def init_resources(self, context): super(GoogleMaps, self).init_resources(context) self.gui.uiauto_params['offline_mode'] = self.offline_mode def setup_rerun(self): super(GoogleMaps, self).setup_rerun() package_data_dir = self.target.path.join( self.target.package_data_directory, self.package) databases_src = self.target.path.join(self.target.working_directory, 'databases.tar') databases_dst = self.target.path.join(package_data_dir, 'databases') files_src = self.target.path.join(self.target.working_directory, 'files.tar') files_dst = self.target.path.join(package_data_dir, 'files') owner = self.target.execute("{} stat -c '%u' {}".format( self.target.busybox, package_data_dir), as_root=True).strip() self.target.execute('{} tar -xvf {} -C {}'.format( self.target.busybox, databases_src, databases_dst), as_root=True) self.target.execute('{} tar -xvf {} -C {}'.format( self.target.busybox, files_src, files_dst), as_root=True) self.target.execute('{} chown -R {}:{} {}'.format( self.target.busybox, owner, owner, package_data_dir), as_root=True)
class ExoPlayer(ApkWorkload): """ Android ExoPlayer ExoPlayer is the basic video player library that is used by the YouTube android app. The aim of this workload is to test a proxy for YouTube performance on targets where running the real YouTube app is not possible due its dependencies. ExoPlayer sources: https://github.com/google/ExoPlayer The 'demo' application is used by this workload. It can easily be built by loading the ExoPlayer sources into Android Studio. Version r2.4.0 built from commit d979469 is known to work Produces a metric 'exoplayer_dropped_frames' - this is the count of frames that Exoplayer itself reports as dropped. This is not the same thing as the dropped frames reported by gfxinfo. """ name = 'exoplayer' video_directory = os.path.join(settings.dependencies_directory, name) package_names = ['com.google.android.exoplayer2.demo'] supported_versions = ['2.4', '2.5', '2.6'] action = 'com.google.android.exoplayer.demo.action.VIEW' default_format = 'mov_720p' view = 'SurfaceView - com.google.android.exoplayer2.demo/com.google.android.exoplayer2.demo.PlayerActivity' parameters = [ Parameter('version', allowed_values=supported_versions, override=True), Parameter('duration', kind=int, default=20, description=""" Playback duration of the video file. This becomes the duration of the workload. If provided must be shorter than the length of the media. """), Parameter('format', allowed_values=list(DOWNLOAD_URLS.keys()), description=""" Specifies which format video file to play. Default is {} """.format(default_format)), Parameter('filename', description=""" The name of the video file to play. This can be either a path to the file anywhere on your file system, or it could be just a name, in which case, the workload will look for it in ``{}`` *Note*: either format or filename should be specified, but not both! """.format(video_directory)), Parameter('force_dependency_push', kind=boolean, default=False, description=""" If true, video will always be pushed to device, regardless of whether the file is already on the device. Default is ``False``. """), Parameter('landscape', kind=boolean, default=False, description=""" Configure the screen in landscape mode, otherwise ensure portrait orientation by default. Default is ``False``. """), ] # pylint: disable=access-member-before-definition def validate(self): if self.format and self.filename: raise ConfigError( 'Either format *or* filename must be specified; but not both.') if not self.format and not self.filename: self.format = self.default_format def _find_host_video_file(self): """Pick the video file we're going to use, download it if necessary""" if self.filename: if self.filename[0] in './' or len( self.filename) > 1 and self.filename[1] == ':': filepath = os.path.abspath(self.filename) else: filepath = os.path.join(self.video_directory, self.filename) if not os.path.isfile(filepath): raise WorkloadError('{} does not exist.'.format(filepath)) return filepath else: # Search for files we've already downloaded files = [] format_ext, format_resolution = self.format.split('_') for filename in os.listdir(self.video_directory): _, file_ext = os.path.splitext(filename) if file_ext == '.' + format_ext and format_resolution in filename: files.append(os.path.join(self.video_directory, filename)) if not files: # Download a file with the requested format url = DOWNLOAD_URLS[self.format] filename = '{}_{}'.format(format_resolution, os.path.basename(url)) filepath = os.path.join(self.video_directory, filename) self.logger.info('Downloading {} to {}...'.format( url, filepath)) urllib.request.urlretrieve(url, filepath) return filepath else: if len(files) > 1: self.logger.warning( 'Multiple files found for {} format. Using {}.'.format( self.format, files[0])) self.logger.warning( 'Use "filename"parameter instead of ' '"format" to specify a different file.') return files[0] def init_resources(self, context): # pylint: disable=unused-argument # Needs to happen first, as it sets self.format, which is required by # _find_host_video_file self.validate() ensure_directory_exists(self.video_directory) self.host_video_file = self._find_host_video_file() def setup(self, context): super(ExoPlayer, self).setup(context) grant_app_permissions(self.target, self.package) self.device_video_file = self.target.path.join( self.target.working_directory, os.path.basename(self.host_video_file)) if self.force_dependency_push or not self.target.file_exists( self.device_video_file): self.logger.info('Copying {} to device.'.format( self.host_video_file)) self.target.push(self.host_video_file, self.device_video_file) self._original_orientation = self.target.get_rotation() self.target.set_rotation(1 if self.landscape else 0) self.play_cmd = 'am start -a {} -d "file://{}"'.format( self.action, self.device_video_file) self.monitor = self.target.get_logcat_monitor(list(REGEXPS.values())) self.monitor.start() def run(self, context): self.target.execute(self.play_cmd) self.monitor.wait_for(REGEXPS['start']) self.logger.info('Playing media file') line = self.monitor.wait_for(REGEXPS['duration'])[0] media_duration_s = int( round(float( re.search(REGEXPS['duration'], line).group('duration')))) self.logger.info( 'Media duration is {} seconds'.format(media_duration_s)) if self.duration > media_duration_s: raise ConfigError( "'duration' param ({}) longer than media duration ({})".format( self.duration, media_duration_s)) if self.duration: self.logger.info( 'Waiting {} seconds before ending playback'.format( self.duration)) time.sleep(self.duration) else: self.logger.info( 'Waiting for playback completion ({} seconds)'.format( media_duration_s)) self.monitor.wait_for(REGEXPS['end'], timeout=media_duration_s + 30) def update_output(self, context): regex = re.compile(REGEXPS['dropped_frames']) dropped_frames = 0 for line in self.monitor.get_log(): match = regex.match(line) if match: dropped_frames += int(match.group('count')) context.add_metric('exoplayer_dropped_frames', dropped_frames, lower_is_better=True) def teardown(self, context): super(ExoPlayer, self).teardown(context) self.monitor.stop() if self._original_orientation is not None: self.target.set_rotation(self._original_orientation)
class RtApp(Workload): # pylint: disable=no-member,attribute-defined-outside-init name = 'rt-app' description = """ A test application that simulates configurable real-time periodic load. rt-app is a test application that starts multiple periodic threads in order to simulate a real-time periodic load. It supports SCHED_OTHER, SCHED_FIFO, SCHED_RR as well as the AQuoSA framework and SCHED_DEADLINE. The load is described using JSON-like config files. Below are a couple of simple examples. Simple use case which creates a thread that run 1ms then sleep 9ms until the use case is stopped with Ctrl+C: .. code-block:: json { "tasks" : { "thread0" : { "loop" : -1, "run" : 20000, "sleep" : 80000 } }, "global" : { "duration" : 2, "calibration" : "CPU0", "default_policy" : "SCHED_OTHER", "pi_enabled" : false, "lock_pages" : false, "logdir" : "./", "log_basename" : "rt-app1", "ftrace" : false, "gnuplot" : true, } } Simple use case with 2 threads that runs for 10 ms and wake up each other until the use case is stopped with Ctrl+C .. code-block:: json { "tasks" : { "thread0" : { "loop" : -1, "run" : 10000, "resume" : "thread1", "suspend" : "thread0" }, "thread1" : { "loop" : -1, "run" : 10000, "resume" : "thread0", "suspend" : "thread1" } } } Please refer to the existing configs in ``$WA_ROOT/wa/workloads/rt_app/use_case`` for more examples. The upstream version of rt-app is hosted here: https://github.com/scheduler-tools/rt-app """ parameters = [ Parameter('config', kind=str, default='taskset', description=''' Use case configuration file to run with rt-app. This may be either the name of one of the "standard" configurations included with the workload. or a path to a custom JSON file provided by the user. Either way, the ".json" extension is implied and will be added automatically if not specified in the argument. The following is the list of standard configurations currently included with the workload: {} '''.format(', '.join( os.listdir(PACKAGED_USE_CASE_DIRECTORY)))), Parameter('duration', kind=int, description=''' Duration of the workload execution in Seconds. If specified, this will override the corresponding parameter in the JSON config. '''), Parameter('cpus', kind=cpu_mask, default=0, aliases=['taskset_mask'], description='Constrain execution to specific CPUs.'), Parameter('uninstall', aliases=['uninstall_on_exit'], kind=bool, default=False, override=True, description=""" If set to ``True``, rt-app binary will be uninstalled from the device at the end of the run. """), Parameter('force_install', kind=bool, default=False, description=""" If set to ``True``, rt-app binary will always be deployed to the target device at the beginning of the run, regardless of whether it was already installed there. """), ] @once def initialize(self, context): # initialize() runs once per run. setting a class variable to make it # available to other instances of the workload RtApp.target_working_directory = self.target.path.join( self.target.working_directory, 'rt-app-working') RtApp.host_binary = context.get_resource(Executable( self, self.target.abi, BINARY_NAME), strict=False) RtApp.workgen_script = context.get_resource(File(self, 'workgen')) self.target.execute('mkdir -p {}'.format( self.target_working_directory)) self._deploy_rt_app_binary_if_necessary() def setup(self, context): self.output = None self.log_basename = context.current_job.label self.host_json_config = self._load_json_config(context) self.config_file_on_target = self.target.path.join( self.target_working_directory, os.path.basename(self.host_json_config)) self.target.push(self.host_json_config, self.config_file_on_target) self.command = '{} {}'.format(self.target_binary, self.config_file_on_target) time_buffer = 30 self.timeout = self.duration + time_buffer def run(self, context): self.output = self.target.invoke( self.command, in_directory=self.target_working_directory, on_cpus=self.cpus and self.cpus.list() or None, redirect_stderr=True, timeout=self.timeout, as_root=self.target.is_rooted) def update_output(self, context): self._pull_rt_app_logs(context) context.output.classifiers.update( dict( duration=self.duration, task_count=self.task_count, )) if not self.output: return outfile = os.path.join(context.output_directory, RAW_OUTPUT_FILENAME) with open(outfile, 'w') as wfh: wfh.write(self.output) error_count = 0 crit_count = 0 for line in self.output.split('\n'): match = PLOAD_REGEX.search(line) if match: pload_value = match.group(1) pload_unit = match.group(2) calib_cpu_value = match.group(3) context.add_metric('pLoad', float(pload_value), pload_unit) context.add_metric('calib_cpu', float(calib_cpu_value)) error_match = ERROR_REGEX.search(line) if error_match: error_count += 1 crit_match = CRIT_REGEX.search(line) if crit_match: crit_count += 1 context.add_metric('error_count', error_count, 'count') context.add_metric('crit_count', crit_count, 'count') @once def finalize(self, context): if self.uninstall: self.target.uninstall(self.target_binary) if self.cleanup_assets: self.target.execute('rm -rf {}'.format( self.target_working_directory)) def _deploy_rt_app_binary_if_necessary(self): # called from initialize() so gets invoked once per run RtApp.target_binary = self.target.get_installed("rt-app") if self.force_install or not RtApp.target_binary: if not self.host_binary: message = '''rt-app is not installed on the target and could not be found in workload resources''' raise ResourceError(message) RtApp.target_binary = self.target.install(self.host_binary) def _load_json_config(self, context): user_config_file = self._get_raw_json_config(context) config_file = self._generate_workgen_config(user_config_file, context.output_directory) with open(config_file) as fh: try: config_data = json.load(fh, object_pairs_hook=OrderedDict) except ValueError: # We were not able to parse the JSON file. Raise an informative error. msg = "Failed to parse {}. Please make sure it is valid JSON." raise ConfigError(msg.format(user_config_file)) self._update_rt_app_config(config_data) self.duration = config_data['global'].get('duration', 0) self.task_count = len(config_data.get('tasks', [])) with open(config_file, 'w') as wfh: json.dump(config_data, wfh, indent=4) return config_file def _get_raw_json_config(self, resolver): if os.path.splitext(self.config)[1] != '.json': self.config += '.json' if os.path.isfile(self.config): return os.path.abspath(self.config) partial_path = os.path.join('use_cases', self.config) return resolver.get(File(self, partial_path)) def _generate_workgen_config(self, user_file, output_directory): output_file = os.path.join(output_directory, 'unkind.json') # use workgen dry run option to generate a use case # file with proper JSON grammar on host first try: check_output('python3 {} -d -o {} {}'.format( self.workgen_script, output_file, user_file), shell=True) except CalledProcessError as e: message = 'Could not generate config using workgen, got "{}"' raise WorkloadError(message.format(e)) return output_file def _update_rt_app_config(self, config_data): config_data['global'] = config_data.get('global', {}) config_data['global']['logdir'] = self.target_working_directory config_data['global']['log_basename'] = self.log_basename if self.duration is not None: config_data['global']['duration'] = self.duration def _pull_rt_app_logs(self, context): tar_command = '{} tar czf {}/{} -C {} .'.format( self.target.busybox, self.target_working_directory, TARBALL_FILENAME, self.target_working_directory) self.target.execute(tar_command, timeout=300) target_path = self.target.path.join(self.target_working_directory, TARBALL_FILENAME) host_path = os.path.join(context.output_directory, TARBALL_FILENAME) self.target.pull(target_path, host_path) with tarfile.open(host_path, 'r:gz') as tf: tf.extractall(context.output_directory) os.remove(host_path) self.target.execute('rm -rf {}/*'.format( self.target_working_directory))
class FpsInstrument(Instrument): name = 'fps' description = """ Measures Frames Per Second (FPS) and associated metrics for a workload. .. note:: This instrument depends on pandas Python library (which is not part of standard WA dependencies), so you will need to install that first, before you can use it. Android L and below use SurfaceFlinger to calculate the FPS data. Android M and above use gfxinfo to calculate the FPS data. SurfaceFlinger: The view is specified by the workload as ``view`` attribute. This defaults to ``'SurfaceView'`` for game workloads, and ``None`` for non-game workloads (as for them FPS mesurement usually doesn't make sense). Individual workloads may override this. gfxinfo: The view is specified by the workload as ``package`` attribute. This is because gfxinfo already processes for all views in a package. """ parameters = [ Parameter('drop_threshold', kind=numeric, default=5, description=""" Data points below this FPS will be dropped as they do not constitute "real" gameplay. The assumption being that while actually running, the FPS in the game will not drop below X frames per second, except on loading screens, menus, etc, which should not contribute to FPS calculation. """), Parameter('keep_raw', kind=bool, default=False, description=""" If set to ``True``, this will keep the raw dumpsys output in the results directory (this is maily used for debugging) Note: frames.csv with collected frames data will always be generated regardless of this setting. """), Parameter('crash_threshold', kind=float, default=0.7, description=""" Specifies the threshold used to decided whether a measured/expected frames ration indicates a content crash. E.g. a value of ``0.75`` means the number of actual frames counted is a quarter lower than expected, it will treated as a content crash. If set to zero, no crash check will be performed. """), Parameter('period', kind=float, default=2, constraint=lambda x: x > 0, description=""" Specifies the time period between polling frame data in seconds when collecting frame data. Using a lower value improves the granularity of timings when recording actions that take a short time to complete. Note, this will produce duplicate frame data in the raw dumpsys output, however, this is filtered out in frames.csv. It may also affect the overall load on the system. The default value of 2 seconds corresponds with the NUM_FRAME_RECORDS in android/services/surfaceflinger/FrameTracker.h (as of the time of writing currently 128) and a frame rate of 60 fps that is applicable to most devices. """), Parameter('force_surfaceflinger', kind=bool, default=False, description=""" By default, the method to capture fps data is based on Android version. If this is set to true, force the instrument to use the SurfaceFlinger method regardless of its Android version. """), ] def __init__(self, target, **kwargs): super(FpsInstrument, self).__init__(target, **kwargs) self.collector = None self.processor = None self._is_enabled = None def setup(self, context): use_gfxinfo = self.target.get_sdk_version( ) >= 23 and not self.force_surfaceflinger if use_gfxinfo: collector_target_attr = 'package' else: collector_target_attr = 'view' collector_target = getattr(context.workload, collector_target_attr, None) if not collector_target: self._is_enabled = False msg = 'Workload {} does not define a {}; disabling frame collection and FPS evaluation.' self.logger.info( msg.format(context.workload.name, collector_target_attr)) return self._is_enabled = True if use_gfxinfo: self.collector = GfxInfoFramesInstrument(self.target, collector_target, self.period) self.processor = DerivedGfxInfoStats(self.drop_threshold, filename='fps.csv') else: self.collector = SurfaceFlingerFramesInstrument( self.target, collector_target, self.period) self.processor = DerivedSurfaceFlingerStats(self.drop_threshold, filename='fps.csv') self.collector.reset() def start(self, context): # pylint: disable=unused-argument if not self._is_enabled: return self.collector.start() def stop(self, context): # pylint: disable=unused-argument if not self._is_enabled: return self.collector.stop() def update_output(self, context): if not self._is_enabled: return outpath = os.path.join(context.output_directory, 'frames.csv') frames_csv = self.collector.get_data(outpath) raw_output = self.collector.get_raw() processed = self.processor.process(frames_csv) processed.extend(self.processor.process_raw(*raw_output)) fps, frame_count, fps_csv = processed[:3] rest = processed[3:] context.add_metric(fps.name, fps.value, fps.units) context.add_metric(frame_count.name, frame_count.value, frame_count.units) context.add_artifact('frames', frames_csv.path, kind='raw') context.add_artifact('fps', fps_csv.path, kind='data') for metric in rest: context.add_metric(metric.name, metric.value, metric.units, lower_is_better=True) if not self.keep_raw: for entry in raw_output: if os.path.isdir(entry): shutil.rmtree(entry) elif os.path.isfile(entry): os.remove(entry) if not frame_count.value: context.add_event('Could not frind frames data in gfxinfo output') context.set_status('PARTIAL') self.check_for_crash(context, fps.value, frame_count.value, context.current_job.run_time.total_seconds()) def check_for_crash(self, context, fps, frames, exec_time): if not self.crash_threshold: return self.logger.debug('Checking for crashed content.') if all([exec_time, fps, frames]): expected_frames = fps * exec_time ratio = frames / expected_frames self.logger.debug('actual/expected frames: {:.2}'.format(ratio)) if ratio < self.crash_threshold: msg = 'Content for {} appears to have crashed.\n'.format( context.current_job.spec.label) msg += 'Content crash detected (actual/expected frames: {:.2}).'.format( ratio) raise WorkloadError(msg)
class Filer(ResourceGetter): name = 'filer' description = """ Finds resources on a (locally mounted) remote filer and caches them locally. This assumes that the filer is mounted on the local machine (e.g. as a samba share). """ parameters = [ Parameter('remote_path', global_alias='remote_assets_path', default='', description=""" Path, on the local system, where the assets are located. """), Parameter('always_fetch', kind=boolean, default=False, global_alias='always_fetch_remote_assets', description=""" If ``True``, will always attempt to fetch assets from the remote, even if a local cached copy is available. """), ] def register(self, resolver): resolver.register(self.get, SourcePriority.lan) def get(self, resource): if resource.owner: remote_path = os.path.join(self.remote_path, resource.owner.name) local_path = os.path.join(settings.dependencies_directory, '__filer', resource.owner.dependencies_directory) return self.try_get_resource(resource, remote_path, local_path) else: # No owner result = None for entry in os.listdir(remote_path): remote_path = os.path.join(self.remote_path, entry) local_path = os.path.join(settings.dependencies_directory, '__filer', settings.dependencies_directory, entry) result = self.try_get_resource(resource, remote_path, local_path) if result: break return result def try_get_resource(self, resource, remote_path, local_path): if not self.always_fetch: result = get_from_location(local_path, resource) if result: return result if not os.path.exists(local_path): return None if os.path.exists(remote_path): # Didn't find it cached locally; now check the remoted result = get_from_location(remote_path, resource) if not result: return result else: # remote path is not set return None # Found it remotely, cache locally, then return it local_full_path = os.path.join(_d(local_path), os.path.basename(result)) self.logger.debug('cp {} {}'.format(result, local_full_path)) shutil.copy(result, local_full_path) return result
class SysfsExtractor(Instrument): name = 'sysfs_extractor' description = """ Collects the contest of a set of directories, before and after workload execution and diffs the result. """ mount_command = 'mount -t tmpfs -o size={} tmpfs {}' extract_timeout = 30 tarname = 'sysfs.tar.gz' DEVICE_PATH = 0 BEFORE_PATH = 1 AFTER_PATH = 2 DIFF_PATH = 3 parameters = [ Parameter( 'paths', kind=list_of_strings, mandatory=True, description= """A list of paths to be pulled from the device. These could be directories as well as files.""", global_alias='sysfs_extract_dirs'), Parameter('use_tmpfs', kind=bool, default=None, description=""" Specifies whether tmpfs should be used to cache sysfile trees and then pull them down as a tarball. This is significantly faster then just copying the directory trees from the device directly, but requires root and may not work on all devices. Defaults to ``True`` if the device is rooted and ``False`` if it is not. """), Parameter( 'tmpfs_mount_point', default=None, description= """Mount point for tmpfs partition used to store snapshots of paths.""" ), Parameter('tmpfs_size', default='32m', description="""Size of the tempfs partition."""), ] def initialize(self, context): if not self.target.is_rooted and self.use_tmpfs: # pylint: disable=access-member-before-definition raise ConfigError( 'use_tempfs must be False for an unrooted device.') elif self.use_tmpfs is None: # pylint: disable=access-member-before-definition self.use_tmpfs = self.target.is_rooted if self.use_tmpfs: self.on_device_before = self.target.path.join( self.tmpfs_mount_point, 'before') self.on_device_after = self.target.path.join( self.tmpfs_mount_point, 'after') if not self.target.file_exists(self.tmpfs_mount_point): self.target.execute('mkdir -p {}'.format( self.tmpfs_mount_point), as_root=True) self.target.execute(self.mount_command.format( self.tmpfs_size, self.tmpfs_mount_point), as_root=True) def setup(self, context): before_dirs = [ _d( os.path.join(context.output_directory, 'before', self._local_dir(d))) for d in self.paths ] after_dirs = [ _d( os.path.join(context.output_directory, 'after', self._local_dir(d))) for d in self.paths ] diff_dirs = [ _d( os.path.join(context.output_directory, 'diff', self._local_dir(d))) for d in self.paths ] self.device_and_host_paths = list( zip(self.paths, before_dirs, after_dirs, diff_dirs)) if self.use_tmpfs: for d in self.paths: before_dir = self.target.path.join( self.on_device_before, self.target.path.dirname(as_relative(d))) after_dir = self.target.path.join( self.on_device_after, self.target.path.dirname(as_relative(d))) if self.target.file_exists(before_dir): self.target.execute('rm -rf {}'.format(before_dir), as_root=True) self.target.execute('mkdir -p {}'.format(before_dir), as_root=True) if self.target.file_exists(after_dir): self.target.execute('rm -rf {}'.format(after_dir), as_root=True) self.target.execute('mkdir -p {}'.format(after_dir), as_root=True) @slow def start(self, context): if self.use_tmpfs: for d in self.paths: dest_dir = self.target.path.join(self.on_device_before, as_relative(d)) if '*' in dest_dir: dest_dir = self.target.path.dirname(dest_dir) self.target.execute('{} cp -Hr {} {}'.format( self.target.busybox, d, dest_dir), as_root=True, check_exit_code=False) else: # not rooted for dev_dir, before_dir, _, _ in self.device_and_host_paths: self.target.pull(dev_dir, before_dir) @slow def stop(self, context): if self.use_tmpfs: for d in self.paths: dest_dir = self.target.path.join(self.on_device_after, as_relative(d)) if '*' in dest_dir: dest_dir = self.target.path.dirname(dest_dir) self.target.execute('{} cp -Hr {} {}'.format( self.target.busybox, d, dest_dir), as_root=True, check_exit_code=False) else: # not using tmpfs for dev_dir, _, after_dir, _ in self.device_and_host_paths: self.target.pull(dev_dir, after_dir) def update_output(self, context): if self.use_tmpfs: on_device_tarball = self.target.path.join( self.target.working_directory, self.tarname) on_host_tarball = self.target.path.join(context.output_directory, self.tarname) self.target.execute('{} tar czf {} -C {} .'.format( self.target.busybox, on_device_tarball, self.tmpfs_mount_point), as_root=True) self.target.execute('chmod 0777 {}'.format(on_device_tarball), as_root=True) self.target.pull(on_device_tarball, on_host_tarball) with tarfile.open(on_host_tarball, 'r:gz') as tf: tf.extractall(context.output_directory) self.target.remove(on_device_tarball) os.remove(on_host_tarball) for paths in self.device_and_host_paths: after_dir = paths[self.AFTER_PATH] dev_dir = paths[self.DEVICE_PATH].strip( '*') # remove potential trailing '*' if (not os.listdir(after_dir) and self.target.file_exists(dev_dir) and self.target.list_directory(dev_dir)): self.logger.error( 'sysfs files were not pulled from the device.') self.device_and_host_paths.remove( paths) # Path is removed to skip diffing it for dev_dir, before_dir, after_dir, diff_dir in self.device_and_host_paths: diff_sysfs_dirs(before_dir, after_dir, diff_dir) context.add_artifact('{} [before]'.format(dev_dir), before_dir, kind='data', classifiers={'stage': 'before'}) context.add_artifact('{} [after]'.format(dev_dir), after_dir, kind='data', classifiers={'stage': 'after'}) context.add_artifact('{} [diff]'.format(dev_dir), diff_dir, kind='data', classifiers={'stage': 'diff'}) def teardown(self, context): self._one_time_setup_done = [] def finalize(self, context): if self.use_tmpfs: try: self.target.execute('umount {}'.format(self.tmpfs_mount_point), as_root=True) except (TargetError, CalledProcessError): # assume a directory but not mount point pass self.target.execute('rm -rf {}'.format(self.tmpfs_mount_point), as_root=True, check_exit_code=False) def validate(self): if not self.tmpfs_mount_point: # pylint: disable=access-member-before-definition self.tmpfs_mount_point = self.target.get_workpath('temp-fs') def _local_dir(self, directory): return os.path.dirname( as_relative(directory).replace(self.target.path.sep, os.sep))
class PerfInstrument(Instrument): name = 'perf' description = """ Perf is a Linux profiling with performance counters. Simpleperf is an Android profiling tool with performance counters. It is highly recomended to use perf_type = simpleperf when using this instrument on android devices since it recognises android symbols in record mode and is much more stable when reporting record .data files. For more information see simpleperf documentation at: https://android.googlesource.com/platform/system/extras/+/master/simpleperf/doc/README.md Performance counters are CPU hardware registers that count hardware events such as instructions executed, cache-misses suffered, or branches mispredicted. They form a basis for profiling applications to trace dynamic control flow and identify hotspots. perf accepts options and events. If no option is given the default '-a' is used. For events, the default events for perf are migrations and cs. The default events for simpleperf are raw-cpu-cycles, raw-l1-dcache, raw-l1-dcache-refill, raw-instructions-retired. They both can be specified in the config file. Events must be provided as a list that contains them and they will look like this :: (for perf_type = perf ) perf_events = ['migrations', 'cs'] (for perf_type = simpleperf) perf_events = ['raw-cpu-cycles', 'raw-l1-dcache'] Events can be obtained by typing the following in the command line on the device :: perf list simpleperf list Whereas options, they can be provided as a single string as following :: perf_options = '-a -i' perf_options = '--app com.adobe.reader' Options can be obtained by running the following in the command line :: man perf-stat """ parameters = [ Parameter('perf_type', kind=str, allowed_values=['perf', 'simpleperf'], default='perf', global_alias='perf_type', description="""Specifies which type of perf binaries to install. Use simpleperf for collecting perf data on android systems.""" ), Parameter( 'command', kind=str, default='stat', allowed_values=['stat', 'record'], global_alias='perf_command', description="""Specifies which perf command to use. If in record mode report command will also be executed and results pulled from target along with raw data file"""), Parameter('events', kind=list_of_strs, global_alias='perf_events', description="""Specifies the events to be counted."""), Parameter('optionstring', kind=list_or_string, default='-a', global_alias='perf_options', description= """Specifies options to be used for the perf command. This may be a list of option strings, in which case, multiple instances of perf will be kicked off -- one for each option string. This may be used to e.g. collected different events from different big.LITTLE clusters. In order to profile a particular application process for android with simpleperf use the --app option e.g. --app com.adobe.reader """), Parameter( 'report_option_string', kind=str, global_alias='perf_report_options', default=None, description= """Specifies options to be used to gather report when record command is used. It's highly recommended to use perf_type simpleperf when running on android devices as reporting options are unstable with perf""" ), Parameter( 'labels', kind=list_of_strs, default=None, global_alias='perf_labels', description= """Provides labels for perf/simpleperf output for each optionstring. If specified, the number of labels must match the number of ``optionstring``\ s. """), Parameter('force_install', kind=bool, default=False, description=""" always install perf binary even if perf is already present on the device. """), ] def __init__(self, target, **kwargs): super(PerfInstrument, self).__init__(target, **kwargs) self.collector = None self.outdir = None def initialize(self, context): self.collector = PerfCollector(self.target, self.perf_type, self.command, self.events, self.optionstring, self.report_option_string, self.labels, self.force_install) def setup(self, context): self.outdir = os.path.join(context.output_directory, self.perf_type) self.collector.set_output(self.outdir) self.collector.reset() def start(self, context): self.collector.start() def stop(self, context): self.collector.stop() def update_output(self, context): self.logger.info('Extracting reports from target...') self.collector.get_data() if self.perf_type == 'perf': self._process_perf_output(context) else: self._process_simpleperf_output(context) def teardown(self, context): self.collector.reset() def _process_perf_output(self, context): if self.command == 'stat': self._process_perf_stat_output(context) elif self.command == 'record': self._process_perf_record_output(context) def _process_simpleperf_output(self, context): if self.command == 'stat': self._process_simpleperf_stat_output(context) elif self.command == 'record': self._process_simpleperf_record_output(context) def _process_perf_stat_output(self, context): for host_file in os.listdir(self.outdir): label = host_file.split('.out')[0] host_file_path = os.path.join(self.outdir, host_file) context.add_artifact(label, host_file_path, 'raw') with open(host_file_path) as fh: in_results_section = False for line in fh: if 'Performance counter stats' in line: in_results_section = True next(fh) # skip the following blank line if not in_results_section: continue if not line.strip(): # blank line in_results_section = False break else: self._add_perf_stat_metric(line, label, context) @staticmethod def _add_perf_stat_metric(line, label, context): line = line.split('#')[0] # comment match = PERF_COUNT_REGEX.search(line) if not match: return classifiers = {} cpu = match.group(1) if cpu is not None: classifiers['cpu'] = int(cpu.replace('CPU', '')) count = int(match.group(2)) metric = '{}_{}'.format(label, match.group(3)) context.add_metric(metric, count, classifiers=classifiers) def _process_perf_record_output(self, context): for host_file in os.listdir(self.outdir): label, ext = os.path.splitext(host_file) context.add_artifact(label, os.path.join(self.outdir, host_file), 'raw') column_headers = [] column_header_indeces = [] event_type = '' if ext == '.rpt': with open(os.path.join(self.outdir, host_file)) as fh: for line in fh: words = line.split() if not words: continue event_type = self._get_report_event_type( words, event_type) column_headers = self._get_report_column_headers( column_headers, words, 'perf') for column_header in column_headers: column_header_indeces.append( line.find(column_header)) self._add_report_metric(column_headers, column_header_indeces, line, words, context, event_type, label) @staticmethod def _get_report_event_type(words, event_type): if words[0] != '#': return event_type if len(words) == 6 and words[4] == 'event': event_type = words[5] event_type = event_type.strip("'") return event_type def _process_simpleperf_stat_output(self, context): labels = [] for host_file in os.listdir(self.outdir): labels.append(host_file.split('.out')[0]) for opts, label in zip(self.optionstring, labels): stat_file = os.path.join(self.outdir, '{}{}'.format(label, '.out')) if '--csv' in opts: self._process_simpleperf_stat_from_csv(stat_file, context, label) else: self._process_simpleperf_stat_from_raw(stat_file, context, label) @staticmethod def _process_simpleperf_stat_from_csv(stat_file, context, label): with open(stat_file) as csv_file: readCSV = csv.reader(csv_file, delimiter=',') line_num = 0 for row in readCSV: if line_num > 0 and 'Total test time' not in row: classifiers = { 'scaled from(%)': row[len(row) - 2].replace('(', '').replace(')', '').replace( '%', '') } context.add_metric('{}_{}'.format(label, row[1]), row[0], 'count', classifiers=classifiers) line_num += 1 @staticmethod def _process_simpleperf_stat_from_raw(stat_file, context, label): with open(stat_file) as fh: for line in fh: if '#' in line: tmp_line = line.split('#')[0] tmp_line = line.strip() count, metric = tmp_line.split(' ')[0], tmp_line.split( ' ')[2] count = int(count.replace(',', '')) scaled_percentage = line.split('(')[1].strip().replace( ')', '').replace('%', '') scaled_percentage = int(scaled_percentage) metric = '{}_{}'.format(label, metric) context.add_metric( metric, count, 'count', classifiers={'scaled from(%)': scaled_percentage}) def _process_simpleperf_record_output(self, context): for host_file in os.listdir(self.outdir): label, ext = os.path.splitext(host_file) context.add_artifact(label, os.path.join(self.outdir, host_file), 'raw') if ext != '.rpt': continue column_headers = [] column_header_indeces = [] event_type = '' with open(os.path.join(self.outdir, host_file)) as fh: for line in fh: words = line.split() if not words: continue if words[0] == 'Event:': event_type = words[1] column_headers = self._get_report_column_headers( column_headers, words, 'simpleperf') for column_header in column_headers: column_header_indeces.append(line.find(column_header)) self._add_report_metric(column_headers, column_header_indeces, line, words, context, event_type, label) @staticmethod def _get_report_column_headers(column_headers, words, perf_type): if 'Overhead' not in words: return column_headers if perf_type == 'perf': words.remove('#') column_headers = words # Concatonate Shared Objects header if 'Shared' in column_headers: shared_index = column_headers.index('Shared') column_headers[shared_index:shared_index + 2] = [ '{} {}'.format(column_headers[shared_index], column_headers[shared_index + 1]) ] return column_headers @staticmethod def _add_report_metric(column_headers, column_header_indeces, line, words, context, event_type, label): if '%' not in words[0]: return classifiers = {} for i in range(1, len(column_headers)): classifiers[column_headers[i]] = line[ column_header_indeces[i]:column_header_indeces[i + 1]].strip() context.add_metric('{}_{}_Overhead'.format(label, event_type), numeric(words[0].strip('%')), 'percent', classifiers=classifiers)
class Http(ResourceGetter): name = 'http' description = """ Downloads resources from a server based on an index fetched from the specified URL. Given a URL, this will try to fetch ``<URL>/index.json``. The index file maps extension names to a list of corresponing asset descriptons. Each asset description continas a path (relative to the base URL) of the resource and a SHA256 hash, so that this Getter can verify whether the resource on the remote has changed. For example, let's assume we want to get the APK file for workload "foo", and that assets are hosted at ``http://example.com/assets``. This Getter will first try to donwload ``http://example.com/assests/index.json``. The index file may contian something like :: { "foo": [ { "path": "foo-app.apk", "sha256": "b14530bb47e04ed655ac5e80e69beaa61c2020450e18638f54384332dffebe86" }, { "path": "subdir/some-other-asset.file", "sha256": "48d9050e9802246d820625717b72f1c2ba431904b8484ca39befd68d1dbedfff" } ] } This Getter will look through the list of assets for "foo" (in this case, two) check the paths until it finds one matching the resource (in this case, "foo-app.apk"). Finally, it will try to dowload that file relative to the base URL and extension name (in this case, "http://example.com/assets/foo/foo-app.apk"). The downloaded version will be cached locally, so that in the future, the getter will check the SHA256 hash of the local file against the one advertised inside index.json, and provided that hasn't changed, it won't try to download the file again. """ parameters = [ Parameter('url', global_alias='remote_assets_url', description=""" URL of the index file for assets on an HTTP server. """), Parameter('username', description=""" User name for authenticating with assets URL """), Parameter('password', description=""" Password for authenticationg with assets URL """), Parameter('always_fetch', kind=boolean, default=False, global_alias='always_fetch_remote_assets', description=""" If ``True``, will always attempt to fetch assets from the remote, even if a local cached copy is available. """), Parameter('chunk_size', kind=int, default=1024, description=""" Chunk size for streaming large assets. """), ] def __init__(self, **kwargs): super(Http, self).__init__(**kwargs) self.logger = logger self.index = {} def register(self, resolver): resolver.register(self.get, SourcePriority.remote) def get(self, resource): if not resource.owner: return # TODO: add support for unowned resources if not self.index: try: self.index = self.fetch_index() except requests.exceptions.RequestException as e: msg = 'Skipping HTTP getter due to connection error: {}' self.logger.debug(msg.format(e.message)) return if resource.kind == 'apk': # APKs must always be downloaded to run ApkInfo for version # information. return self.resolve_apk(resource) else: asset = self.resolve_resource(resource) if not asset: return return self.download_asset(asset, resource.owner.name) def fetch_index(self): if not self.url: return {} index_url = urljoin(self.url, 'index.json') response = self.geturl(index_url) if response.status_code != http.client.OK: message = 'Could not fetch "{}"; recieved "{} {}"' self.logger.error( message.format(index_url, response.status_code, response.reason)) return {} if sys.version_info[0] == 3: content = response.content.decode('utf-8') else: content = response.content return json.loads(content) def download_asset(self, asset, owner_name): url = urljoin(self.url, owner_name, asset['path']) local_path = _f( os.path.join(settings.dependencies_directory, '__remote', owner_name, asset['path'].replace('/', os.sep))) if os.path.exists(local_path) and not self.always_fetch: local_sha = sha256(local_path) if local_sha == asset['sha256']: self.logger.debug('Local SHA256 matches; not re-downloading') return local_path self.logger.debug('Downloading {}'.format(url)) response = self.geturl(url, stream=True) if response.status_code != http.client.OK: message = 'Could not download asset "{}"; recieved "{} {}"' self.logger.warning( message.format(url, response.status_code, response.reason)) return with open(local_path, 'wb') as wfh: for chunk in response.iter_content(chunk_size=self.chunk_size): wfh.write(chunk) return local_path def geturl(self, url, stream=False): if self.username: auth = (self.username, self.password) else: auth = None return requests.get(url, auth=auth, stream=stream) def resolve_apk(self, resource): assets = self.index.get(resource.owner.name, {}) if not assets: return None asset_map = {a['path']: a for a in assets} paths = get_path_matches(resource, list(asset_map.keys())) local_paths = [] for path in paths: local_paths.append( self.download_asset(asset_map[path], resource.owner.name)) for path in local_paths: if resource.match(path): return path def resolve_resource(self, resource): # pylint: disable=too-many-branches,too-many-locals assets = self.index.get(resource.owner.name, {}) if not assets: return {} asset_map = {a['path']: a for a in assets} if resource.kind in ['jar', 'revent']: path = get_generic_resource(resource, list(asset_map.keys())) if path: return asset_map[path] elif resource.kind == 'executable': path = '/'.join(['bin', resource.abi, resource.filename]) for asset in assets: if asset['path'].lower() == path.lower(): return asset else: # file for asset in assets: if asset['path'].lower() == resource.path.lower(): return asset
class Googlephotos(ApkUiautoWorkload): name = 'googlephotos' package_names = ['com.google.android.apps.photos'] description = ''' A workload to perform standard productivity tasks with Google Photos. The workload carries out various tasks, such as browsing images, performing zooms, and post-processing the image. Test description: 1. Four images are copied to the target 2. The application is started in offline access mode 3. Gestures are performed to pinch zoom in and out of the selected image 4. The colour of a selected image is edited by selecting the colour menu, incrementing the colour, resetting the colour and decrementing the colour using the seek bar. 5. A crop test is performed on a selected image. UiAutomator does not allow the selection of the crop markers so the image is tilted positively, reset and then tilted negatively to get a similar cropping effect. 6. A rotate test is performed on a selected image, rotating anticlockwise 90 degrees, 180 degrees and 270 degrees. Known working APK version: 4.0.0.212659618 ''' default_test_images = [ 'uxperf_1200x1600.png', 'uxperf_1600x1200.jpg', 'uxperf_2448x3264.png', 'uxperf_3264x2448.jpg', ] parameters = [ Parameter('test_images', kind=list_of_strs, default=default_test_images, constraint=lambda x: len(unique(x)) == 4, description=''' A list of four JPEG and/or PNG files to be pushed to the target. Absolute file paths may be used but tilde expansion must be escaped. '''), ] def __init__(self, target, **kwargs): super(Googlephotos, self).__init__(target, **kwargs) self.deployable_assets = self.test_images def init_resources(self, context): super(Googlephotos, self).init_resources(context) # Only accept certain image formats for image in self.test_images: if os.path.splitext( image.lower())[1] not in ['.jpg', '.jpeg', '.png']: raise ValidationError( '{} must be a JPEG or PNG file'.format(image)) def deploy_assets(self, context): super(Googlephotos, self).deploy_assets(context) # Create a subfolder for each test_image named ``wa-[1-4]`` # Move each image into its subfolder # This is to guarantee ordering and allows the workload to select a specific # image by subfolder, as filenames are not shown easily within the app d = self.target.working_directory e = self.target.external_storage file_list = [] for i, f in enumerate(self.test_images): orig_file_path = self.target.path.join(d, f) new_dir = self.target.path.join(e, 'wa', 'wa-{}'.format(i + 1)) new_file_path = self.target.path.join(new_dir, f) self.target.execute('mkdir -p {}'.format(new_dir)) self.target.execute('cp {} {}'.format(orig_file_path, new_file_path)) self.target.execute('rm {}'.format(orig_file_path)) file_list.append(new_file_path) self.deployed_assets = file_list # Force rescan self.target.refresh_files(self.deployed_assets) def remove_assets(self, context): for asset in self.deployed_assets: self.target.remove(os.path.dirname(asset)) self.target.refresh_files(self.deployed_assets)
class SerialMon(Instrument): name = 'serialmon' description = """ Records the traffic on a serial connection The traffic on a serial connection is monitored and logged to a file. In the event that the device is reset, the instrument will stop monitoring during the reset, and will reconnect once the reset has completed. This is to account for devices (i.e., the Juno) which utilise the serial connection to reset the board. """ parameters = [ Parameter('serial_port', kind=str, default="/dev/ttyS0", description=""" The serial device to monitor. """), Parameter('baudrate', kind=int, default=115200, description=""" The baud-rate to use when connecting to the serial connection. """), ] def __init__(self, target, **kwargs): super(SerialMon, self).__init__(target, **kwargs) self._collector = SerialTraceCollector(target, self.serial_port, self.baudrate) def start_logging(self, context, filename="serial.log"): outpath = os.path.join(context.output_directory, filename) self._collector.set_output(outpath) self._collector.reset() self.logger.debug("Acquiring serial port ({})".format( self.serial_port)) if self._collector.collecting: self.stop_logging(context) self._collector.start() def stop_logging(self, context, identifier="job"): self.logger.debug("Releasing serial port ({})".format( self.serial_port)) if self._collector.collecting: self._collector.stop() data = self._collector.get_data() for l in data: # noqa: E741 context.add_artifact("{}_serial_log".format(identifier), l.path, kind="log") def on_run_start(self, context): self.start_logging(context, "preamble_serial.log") def before_job_queue_execution(self, context): self.stop_logging(context, "preamble") def after_job_queue_execution(self, context): self.start_logging(context, "postamble_serial.log") def on_run_end(self, context): self.stop_logging(context, "postamble") def on_job_start(self, context): self.start_logging(context) def on_job_end(self, context): self.stop_logging(context) @hostside def before_reboot(self, context): self.stop_logging(context)
class ManualWorkload(Workload): name = 'manual' description = """ Yields control to the user, either for a fixed period or based on user input, to perform custom operations on the device, which workload automation does not know of. """ default_duration = 30 parameters = [ Parameter('duration', kind=int, default=None, description=""" Control of the devices is yielded for the duration (in seconds) specified. If not specified, ``user_triggered`` is assumed. """), Parameter('user_triggered', kind=bool, default=None, description=""" If ``True``, WA will wait for user input after starting the workload; otherwise fixed duration is expected. Defaults to ``True`` if ``duration`` is not specified, and ``False`` otherwise. """), Parameter('view', default='SurfaceView', description=""" Specifies the View of the workload. This enables instruments that require a View to be specified, such as the ``fps`` instrument. This is required for using "SurfaceFlinger" to collect FPS statistics and is primarily used on devices pre API level 23. """), Parameter('package', description=""" Specifies the package name of the workload. This enables instruments that require a Package to be specified, such as the ``fps`` instrument. This allows for "gfxinfo" to be used and is the preferred method of collection for FPS statistics on devices API level 23+. """), ] def validate(self): if self.duration is None: if self.user_triggered is None: self.user_triggered = True elif self.user_triggered is False: self.duration = self.default_duration if self.user_triggered and self.duration: message = 'Manual Workload can either specify duration or be user triggered, but not both' raise ConfigError(message) if not self.user_triggered and not self.duration: raise ConfigError('Either user_triggered must be ``True`` or duration must be > 0.') def setup(self, context): self.logger.info('Any setup required by your workload should be done now.') self.logger.info('As soon as you are done hit any key and wait for the message') self.logger.info('"START NOW!" to begin your manual workload.') self.logger.info('') self.logger.info('hit any key to finalize your setup...') getch() def run(self, context): self.logger.info('START NOW!') if self.duration: self.target.sleep(self.duration) elif self.user_triggered: self.logger.info('') self.logger.info('hit any key to end your workload execution...') getch() else: raise ConfigError('Illegal parameters for manual workload') self.logger.info('DONE!')