class ConfigTest(TestCase): def setUp(self): self.config = RunConfiguration(MockExtensionLoader()) self.config.load_config({'device': 'MockDevice'}) def test_case(self): devparams = { 'sysfile_values': { '/sys/test/MyFile': 1, '/sys/test/other file': 2, } } ws = AgendaWorkloadEntry(id='a', iterations=1, name='linpack', runtime_parameters=devparams) self.config.set_agenda(MockAgenda(ws)) spec = self.config.workload_specs[0] assert_in('/sys/test/MyFile', spec.runtime_parameters['sysfile_values']) assert_in('/sys/test/other file', spec.runtime_parameters['sysfile_values']) def test_list_defaults_params(self): ws = AgendaWorkloadEntry(id='a', iterations=1, name='defaults_workload', workload_parameters={'param':[3]}) self.config.set_agenda(MockAgenda(ws)) spec = self.config.workload_specs[0] assert_equal(spec.workload_parameters, {'param': [3]}) def test_exetension_params_lists(self): a = Agenda(LIST_PARAMS_AGENDA_TEXT) self.config.set_agenda(a) self.config.finalize() assert_equal(self.config.instrumentation['list_params']['param'], [0.1, 0.1, 0.1]) def test_instrumentation_specification(self): a = Agenda(INSTRUMENTATION_AGENDA_TEXT) self.config.set_agenda(a) self.config.finalize() assert_equal(self.config.workload_specs[0].instrumentation, ['execution_time']) assert_equal(self.config.workload_specs[1].instrumentation, ['fsp', 'execution_time']) def test_remove_instrument(self): self.config.load_config({'instrumentation': ['list_params']}) a = Agenda('{config: {instrumentation: [~list_params] }}') self.config.set_agenda(a) self.config.finalize() assert_equal(self.config.instrumentation, {}) def test_global_instrumentation(self): self.config.load_config({'instrumentation': ['global_instrument']}) ws = AgendaWorkloadEntry(id='a', iterations=1, name='linpack', instrumentation=['local_instrument']) self.config.set_agenda(MockAgenda(ws)) self.config.finalize() assert_equal(self.config.workload_specs[0].instrumentation, ['local_instrument', 'global_instrument'])
class ReventCommand(Command): # Validate command options def validate_args(self, args): if args.clear and not args.package: print "Package must be specified if you want to clear cache\n" self.parser.print_help() sys.exit() # pylint: disable=W0201 def execute(self, args): self.validate_args(args) self.logger.info("Connecting to device...") ext_loader = ExtensionLoader(packages=settings.extension_packages, paths=settings.extension_paths) # Setup config self.config = RunConfiguration(ext_loader) for filepath in settings.get_config_paths(): self.config.load_config(filepath) self.config.set_agenda(Agenda()) self.config.finalize() context = LightContext(self.config) # Setup device self.device = ext_loader.get_device(settings.device, **settings.device_config) self.device.validate() self.device.dynamic_modules = [] self.device.connect() self.device.initialize(context) host_binary = context.resolver.get( Executable(NO_ONE, self.device.abi, 'revent')) self.target_binary = self.device.install_executable(host_binary) self.run(args) def run(self, args): raise NotImplementedError()
class ReventCommand(Command): # Validate command options def validate_args(self, args): if args.clear and not args.package: print "Package must be specified if you want to clear cache\n" self.parser.print_help() sys.exit() # pylint: disable=W0201 def execute(self, args): self.validate_args(args) self.logger.info("Connecting to device...") ext_loader = ExtensionLoader(packages=settings.extension_packages, paths=settings.extension_paths) # Setup config self.config = RunConfiguration(ext_loader) for filepath in settings.get_config_paths(): self.config.load_config(filepath) self.config.set_agenda(Agenda()) self.config.finalize() context = LightContext(self.config) # Setup device self.device = ext_loader.get_device(settings.device, **settings.device_config) self.device.validate() self.device.dynamic_modules = [] self.device.connect() self.device.initialize(context) host_binary = context.resolver.get(Executable(NO_ONE, self.device.abi, 'revent')) self.target_binary = self.device.install_executable(host_binary) self.run(args) def run(self, args): raise NotImplementedError()
class Executor(object): """ The ``Executor``'s job is to set up the execution context and pass to a ``Runner`` along with a loaded run specification. Once the ``Runner`` has done its thing, the ``Executor`` performs some final reporint before returning. The initial context set up involves combining configuration from various sources, loading of requided workloads, loading and installation of instruments and result processors, etc. Static validation of the combined configuration is also performed. """ # pylint: disable=R0915 def __init__(self): self.logger = logging.getLogger('Executor') self.error_logged = False self.warning_logged = False self.config = None self.ext_loader = None self.device = None self.context = None def execute(self, agenda, selectors=None): # NOQA """ Execute the run specified by an agenda. Optionally, selectors may be used to only selecute a subset of the specified agenda. Params:: :agenda: an ``Agenda`` instance to be executed. :selectors: A dict mapping selector name to the coresponding values. **Selectors** Currently, the following seectors are supported: ids The value must be a sequence of workload specfication IDs to be executed. Note that if sections are specified inthe agenda, the workload specifacation ID will be a combination of the section and workload IDs. """ signal.connect(self._error_signalled_callback, signal.ERROR_LOGGED) signal.connect(self._warning_signalled_callback, signal.WARNING_LOGGED) self.logger.info('Initializing') self.ext_loader = ExtensionLoader(packages=settings.extension_packages, paths=settings.extension_paths) self.logger.debug('Loading run configuration.') self.config = RunConfiguration(self.ext_loader) for filepath in settings.get_config_paths(): self.config.load_config(filepath) self.config.set_agenda(agenda, selectors) self.config.finalize() config_outfile = os.path.join(settings.meta_directory, 'run_config.json') with open(config_outfile, 'w') as wfh: self.config.serialize(wfh) self.logger.debug('Initialising device configuration.') if not self.config.device: raise ConfigError('Make sure a device is specified in the config.') self.device = self.ext_loader.get_device(self.config.device, **self.config.device_config) self.device.validate() self.context = ExecutionContext(self.device, self.config) self.logger.debug('Loading resource discoverers.') self.context.initialize() self.context.resolver.load() self.context.add_artifact('run_config', config_outfile, 'meta') self.logger.debug('Installing instrumentation') for name, params in self.config.instrumentation.iteritems(): instrument = self.ext_loader.get_instrument( name, self.device, **params) instrumentation.install(instrument) instrumentation.validate() self.logger.debug('Installing result processors') result_manager = ResultManager() for name, params in self.config.result_processors.iteritems(): processor = self.ext_loader.get_result_processor(name, **params) result_manager.install(processor) result_manager.validate() self.logger.debug('Loading workload specs') for workload_spec in self.config.workload_specs: workload_spec.load(self.device, self.ext_loader) workload_spec.workload.init_resources(self.context) workload_spec.workload.validate() if self.config.flashing_config: if not self.device.flasher: msg = 'flashing_config specified for {} device that does not support flashing.' raise ConfigError(msg.format(self.device.name)) self.logger.debug('Flashing the device') self.device.flasher.flash(self.device) self.logger.info('Running workloads') runner = self._get_runner(result_manager) runner.init_queue(self.config.workload_specs) runner.run() self.execute_postamble() def execute_postamble(self): """ This happens after the run has completed. The overall results of the run are summarised to the user. """ result = self.context.run_result counter = Counter() for ir in result.iteration_results: counter[ir.status] += 1 self.logger.info('Done.') self.logger.info('Run duration: {}'.format( format_duration(self.context.run_info.duration))) status_summary = 'Ran a total of {} iterations: '.format( sum(self.context.job_iteration_counts.values())) parts = [] for status in IterationResult.values: if status in counter: parts.append('{} {}'.format(counter[status], status)) self.logger.info(status_summary + ', '.join(parts)) self.logger.info('Results can be found in {}'.format( settings.output_directory)) if self.error_logged: self.logger.warn('There were errors during execution.') self.logger.warn('Please see {}'.format(settings.log_file)) elif self.warning_logged: self.logger.warn('There were warnings during execution.') self.logger.warn('Please see {}'.format(settings.log_file)) def _get_runner(self, result_manager): if not self.config.execution_order or self.config.execution_order == 'by_iteration': if self.config.reboot_policy == 'each_spec': self.logger.info( 'each_spec reboot policy with the default by_iteration execution order is ' 'equivalent to each_iteration policy.') runnercls = ByIterationRunner elif self.config.execution_order in ['classic', 'by_spec']: runnercls = BySpecRunner elif self.config.execution_order == 'by_section': runnercls = BySectionRunner elif self.config.execution_order == 'random': runnercls = RandomRunner else: raise ConfigError('Unexpected execution order: {}'.format( self.config.execution_order)) return runnercls(self.device, self.context, result_manager) def _error_signalled_callback(self): self.error_logged = True signal.disconnect(self._error_signalled_callback, signal.ERROR_LOGGED) def _warning_signalled_callback(self): self.warning_logged = True signal.disconnect(self._warning_signalled_callback, signal.WARNING_LOGGED)
class Executor(object): """ The ``Executor``'s job is to set up the execution context and pass to a ``Runner`` along with a loaded run specification. Once the ``Runner`` has done its thing, the ``Executor`` performs some final reporint before returning. The initial context set up involves combining configuration from various sources, loading of requided workloads, loading and installation of instruments and result processors, etc. Static validation of the combined configuration is also performed. """ # pylint: disable=R0915 def __init__(self): self.logger = logging.getLogger('Executor') self.error_logged = False self.warning_logged = False self.config = None self.ext_loader = None self.device = None self.context = None def execute(self, agenda, selectors=None): # NOQA """ Execute the run specified by an agenda. Optionally, selectors may be used to only selecute a subset of the specified agenda. Params:: :agenda: an ``Agenda`` instance to be executed. :selectors: A dict mapping selector name to the coresponding values. **Selectors** Currently, the following seectors are supported: ids The value must be a sequence of workload specfication IDs to be executed. Note that if sections are specified inthe agenda, the workload specifacation ID will be a combination of the section and workload IDs. """ signal.connect(self._error_signalled_callback, signal.ERROR_LOGGED) signal.connect(self._warning_signalled_callback, signal.WARNING_LOGGED) self.logger.info('Initializing') self.ext_loader = ExtensionLoader(packages=settings.extension_packages, paths=settings.extension_paths) self.logger.debug('Loading run configuration.') self.config = RunConfiguration(self.ext_loader) for filepath in settings.get_config_paths(): self.config.load_config(filepath) self.config.set_agenda(agenda, selectors) self.config.finalize() config_outfile = os.path.join(settings.meta_directory, 'run_config.json') with open(config_outfile, 'w') as wfh: self.config.serialize(wfh) self.logger.debug('Initialising device configuration.') if not self.config.device: raise ConfigError('Make sure a device is specified in the config.') self.device = self.ext_loader.get_device(self.config.device, **self.config.device_config) self.device.validate() self.context = ExecutionContext(self.device, self.config) self.logger.debug('Loading resource discoverers.') self.context.initialize() self.context.resolver.load() self.context.add_artifact('run_config', config_outfile, 'meta') self.logger.debug('Installing instrumentation') for name, params in self.config.instrumentation.iteritems(): instrument = self.ext_loader.get_instrument(name, self.device, **params) instrumentation.install(instrument) instrumentation.validate() self.logger.debug('Installing result processors') result_manager = ResultManager() for name, params in self.config.result_processors.iteritems(): processor = self.ext_loader.get_result_processor(name, **params) result_manager.install(processor) result_manager.validate() self.logger.debug('Loading workload specs') for workload_spec in self.config.workload_specs: workload_spec.load(self.device, self.ext_loader) workload_spec.workload.init_resources(self.context) workload_spec.workload.validate() if self.config.flashing_config: if not self.device.flasher: msg = 'flashing_config specified for {} device that does not support flashing.' raise ConfigError(msg.format(self.device.name)) self.logger.debug('Flashing the device') self.device.flasher.flash(self.device) self.logger.info('Running workloads') runner = self._get_runner(result_manager) runner.init_queue(self.config.workload_specs) runner.run() self.execute_postamble() def execute_postamble(self): """ This happens after the run has completed. The overall results of the run are summarised to the user. """ result = self.context.run_result counter = Counter() for ir in result.iteration_results: counter[ir.status] += 1 self.logger.info('Done.') self.logger.info('Run duration: {}'.format(format_duration(self.context.run_info.duration))) status_summary = 'Ran a total of {} iterations: '.format(sum(self.context.job_iteration_counts.values())) parts = [] for status in IterationResult.values: if status in counter: parts.append('{} {}'.format(counter[status], status)) self.logger.info(status_summary + ', '.join(parts)) self.logger.info('Results can be found in {}'.format(settings.output_directory)) if self.error_logged: self.logger.warn('There were errors during execution.') self.logger.warn('Please see {}'.format(settings.log_file)) elif self.warning_logged: self.logger.warn('There were warnings during execution.') self.logger.warn('Please see {}'.format(settings.log_file)) def _get_runner(self, result_manager): if not self.config.execution_order or self.config.execution_order == 'by_iteration': if self.config.reboot_policy == 'each_spec': self.logger.info('each_spec reboot policy with the default by_iteration execution order is ' 'equivalent to each_iteration policy.') runnercls = ByIterationRunner elif self.config.execution_order in ['classic', 'by_spec']: runnercls = BySpecRunner elif self.config.execution_order == 'by_section': runnercls = BySectionRunner elif self.config.execution_order == 'random': runnercls = RandomRunner else: raise ConfigError('Unexpected execution order: {}'.format(self.config.execution_order)) return runnercls(self.device, self.context, result_manager) def _error_signalled_callback(self): self.error_logged = True signal.disconnect(self._error_signalled_callback, signal.ERROR_LOGGED) def _warning_signalled_callback(self): self.warning_logged = True signal.disconnect(self._warning_signalled_callback, signal.WARNING_LOGGED)
class RecordCommand(Command): name = 'record' description = '''Performs a revent recording This command helps making revent recordings. It will automatically deploy revent and even has the option of automatically opening apps. Revent allows you to record raw inputs such as screen swipes or button presses. This can be useful for recording inputs for workloads such as games that don't have XML UI layouts that can be used with UIAutomator. As a drawback from this, revent recordings are specific to the device type they were recorded on. WA uses two parts to the names of revent recordings in the format, {device_name}.{suffix}.revent. - device_name can either be specified manually with the ``-d`` argument or it can be automatically determined. On Android device it will be obtained from ``build.prop``, on Linux devices it is obtained from ``/proc/device-tree/model``. - suffix is used by WA to determine which part of the app execution the recording is for, currently these are either ``setup`` or ``run``. This should be specified with the ``-s`` argument. ''' def initialize(self, context): self.context = context self.parser.add_argument('-d', '--device', help='The name of the device') self.parser.add_argument('-s', '--suffix', help='The suffix of the revent file, e.g. ``setup``') self.parser.add_argument('-o', '--output', help='Directory to save the recording in') self.parser.add_argument('-p', '--package', help='Package to launch before recording') self.parser.add_argument('-C', '--clear', help='Clear app cache before launching it', action="store_true") # Validate command options def validate_args(self, args): if args.clear and not args.package: print "Package must be specified if you want to clear cache\n" self.parser.print_help() sys.exit() # pylint: disable=W0201 def execute(self, args): self.validate_args(args) self.logger.info("Connecting to device...") ext_loader = ExtensionLoader(packages=settings.extension_packages, paths=settings.extension_paths) # Setup config self.config = RunConfiguration(ext_loader) for filepath in settings.get_config_paths(): self.config.load_config(filepath) self.config.set_agenda(Agenda()) self.config.finalize() context = LightContext(self.config) # Setup device self.device = ext_loader.get_device(settings.device, **settings.device_config) self.device.validate() self.device.dynamic_modules = [] self.device.connect() self.device.initialize(context) host_binary = context.resolver.get(Executable(NO_ONE, self.device.abi, 'revent')) self.target_binary = self.device.install_if_needed(host_binary) self.run(args) def run(self, args): if args.device: self.device_name = args.device else: self.device_name = self.device.get_device_model() if args.suffix: args.suffix += "." revent_file = self.device.path.join(self.device.working_directory, '{}.{}revent'.format(self.device_name, args.suffix or "")) if args.clear: self.device.execute("pm clear {}".format(args.package)) if args.package: self.logger.info("Starting {}".format(args.package)) self.device.execute('monkey -p {} -c android.intent.category.LAUNCHER 1'.format(args.package)) self.logger.info("Press Enter when you are ready to record...") raw_input("") command = "{} record -t 100000 -s {}".format(self.target_binary, revent_file) self.device.kick_off(command) self.logger.info("Press Enter when you have finished recording...") raw_input("") self.device.killall("revent") self.logger.info("Pulling files from device") self.device.pull_file(revent_file, args.output or os.getcwdu())
class ConfigTest(TestCase): def setUp(self): self.config = RunConfiguration(MockExtensionLoader()) self.config.load_config({'device': 'MockDevice'}) def test_case(self): devparams = { 'sysfile_values': { '/sys/test/MyFile': 1, '/sys/test/other file': 2, } } ws = AgendaWorkloadEntry(id='a', iterations=1, name='linpack', runtime_parameters=devparams) self.config.set_agenda(MockAgenda(ws)) spec = self.config.workload_specs[0] assert_in('/sys/test/MyFile', spec.runtime_parameters['sysfile_values']) assert_in('/sys/test/other file', spec.runtime_parameters['sysfile_values']) def test_list_defaults_params(self): ws = AgendaWorkloadEntry(id='a', iterations=1, name='defaults_workload', workload_parameters={'param': [3]}) self.config.set_agenda(MockAgenda(ws)) spec = self.config.workload_specs[0] assert_equal(spec.workload_parameters, {'param': [3]}) def test_exetension_params_lists(self): a = Agenda(LIST_PARAMS_AGENDA_TEXT) self.config.set_agenda(a) self.config.finalize() assert_equal(self.config.instrumentation['list_params']['param'], [0.1, 0.1, 0.1]) def test_instrumentation_specification(self): a = Agenda(INSTRUMENTATION_AGENDA_TEXT) self.config.set_agenda(a) self.config.finalize() assert_equal(self.config.workload_specs[0].instrumentation, ['execution_time']) assert_equal(self.config.workload_specs[1].instrumentation, ['fsp', 'execution_time']) def test_remove_instrument(self): self.config.load_config({'instrumentation': ['list_params']}) a = Agenda('{config: {instrumentation: [~list_params] }}') self.config.set_agenda(a) self.config.finalize() assert_equal(self.config.instrumentation, {}) def test_global_instrumentation(self): self.config.load_config({'instrumentation': ['global_instrument']}) ws = AgendaWorkloadEntry(id='a', iterations=1, name='linpack', instrumentation=['local_instrument']) self.config.set_agenda(MockAgenda(ws)) self.config.finalize() assert_equal(self.config.workload_specs[0].instrumentation, ['local_instrument', 'global_instrument'])