def run(self, out_dir, test_name, collect=''): """ Run single PCMark workload. Returns a collection of results. :param out_dir: Path to experiment directory on the host where to store results. :type out_dir: str :param test_name: Name of the test to run :type test_name: str :param collect: Specifies what to collect. Possible values: - 'energy' - 'systrace' - 'ftrace' - any combination of the above as a single space-separated string. :type collect: list(str) """ self.out_dir = out_dir self.collect = collect # Unlock device screen (assume no password required) System.menu(self._target) # Make sure we exit the app if already open System.back(self._target) # Close and NOT clear application (benchmark tests are downloaded from web) System.force_stop(self._target, self.package, clear=False) # Set airplane mode System.set_airplane_mode(self._target, on=True) # Set min brightness Screen.set_brightness(self._target, auto=False, percent=0) # Force screen in PORTRAIT mode Screen.set_orientation(self._target, portrait=True) monitor = self._target.get_logcat_monitor(REGEXPS.values()) # Start PCMark on the target device System.start_activity(self._target, self.package, self.activity) # Wait a few seconds while application is being loaded time.sleep(5) if test_name.lower() == 'work': # Move to benchmark run page System.tab(self._target) System.enter(self._target) # Wait for page animations to end time.sleep(10) else: raise ValueError('PCMark test [%s] not supported', test_name) # Start benchmark monitor.start() System.enter(self._target) monitor.wait_for(REGEXPS['start']) self.tracingStart() self._log.info('PCmark started!') monitor.wait_for(REGEXPS['end'], timeout=600) self.tracingStop() self._log.info('PCmark ended!') # That should only match one line, but use the most recent one # in case logcat wasn't cleared properly result = monitor.wait_for(REGEXPS['result'])[-1] monitor.stop() remote_archive = re.match(REGEXPS['result'], result).group('path') local_archive = os.path.join(out_dir, self._target.path.basename(remote_archive)) self._target.pull(remote_archive, local_archive, as_root=True) # Several files in the archive # Only "Result.xml" matters to us with ZipFile(local_archive, 'r') as archive: archive.extractall(out_dir) # Fetch workloads names and scores scores = {} input_result_filepath = os.path.join(out_dir, INPUT_RESULT_FILE) with open(input_result_filepath, 'r') as fd: for line in fd: match = re.match(REGEXPS['score'], line) if match: scores[match.group('name')] = match.group('score') # Dump scores to json output_result_filepath = os.path.join(out_dir, OUTPUT_RESULT_FILE) with open(output_result_filepath, 'w') as fd: json.dump(scores, fd) self._log.info('Scores available in {}'.format(output_result_filepath)) System.force_stop(self._target, self.package, clear=False) # Restore default configuration System.home(self._target) # Set orientation back to auto Screen.set_orientation(self._target, auto=True) # Go back to home screen System.home(self._target) # Turn off airplane mode System.set_airplane_mode(self._target, on=False) # Set brightness back to auto Screen.set_brightness(self._target, auto=True)
def run(self, out_dir, test_name, duration_s, collect='', actions='default'): """ Run single UiBench workload. :param out_dir: Path to experiment directory where to store results. :type out_dir: str :param test_name: Name of the test to run :type test_name: str :param duration_s: Run benchmak for this required number of seconds :type duration_s: int :param collect: Specifies what to collect. Possible values: - 'energy' - 'systrace' - 'ftrace' - any combination of the above :type collect: list(str) :param actions: Specifies what actions to perform. Possible values: - None : Perform no action - 'default' : Use the predefined default actions from the `test_actions` dict - 'vswipe' : Perform a vertical swipe - 'tap' : Perform a centre tap - A list with any combination of vswipe and tap, in execution order :type actions: list(str) """ activity = '.' + test_name # If default, get the default actions from test_actions # If this is an undefined test, no action will be the default if actions == 'default': actions = None if test_name in self.test_actions: actions = self.test_actions[test_name] # Format actions as a list if it is just a single string item, # or an empty list if it is None actions = [actions] if isinstance(actions, basestring) else actions if actions else [] # Keep track of mandatory parameters self.out_dir = out_dir self.collect = collect # Unlock device screen (assume no password required) Screen.unlock(self._target) # Close and clear application System.force_stop(self._target, self.package, clear=True) # Set airplane mode System.set_airplane_mode(self._target, on=True) # Set min brightness Screen.set_brightness(self._target, auto=False, percent=0) # Start the main view of the app which must be running # to reset the frame statistics. System.monkey(self._target, self.package) # Force screen in PORTRAIT mode Screen.set_orientation(self._target, portrait=True) # Reset frame statistics System.gfxinfo_reset(self._target, self.package) sleep(1) # Clear logcat os.system(self._adb('logcat -c')); # Regexps for benchmark synchronization start_logline = r'ActivityManager: START.*'\ 'cmp=com.android.test.uibench/{}'.format(activity) UIBENCH_BENCHMARK_START_RE = re.compile(start_logline) self._log.debug("START string [%s]", start_logline) # Parse logcat output lines logcat_cmd = self._adb( 'logcat ActivityManager:* System.out:I *:S BENCH:*'\ .format(self._target.adb_name)) self._log.info("%s", logcat_cmd) # Prepare user-input commands on the device if actions: script = TargetScript(self._te, "uibench.sh") for action in actions: self._perform_action(script, action) script.push() # Start the activity System.start_activity(self._target, self.package, activity) logcat = Popen(logcat_cmd, shell=True, stdout=PIPE) while True: # read next logcat line (up to max 1024 chars) message = logcat.stdout.readline(1024) # Benchmark start trigger match = UIBENCH_BENCHMARK_START_RE.search(message) if match: self.tracingStart() self._log.debug("Benchmark started!") break # Run the workload for the required time self._log.info('Benchmark [%s] started, waiting %d [s]', activity, duration_s) start = time() if actions: script.run() while (time() - start) < duration_s: sleep(1) self._log.debug("Benchmark done!") self.tracingStop() # Get frame stats self.db_file = os.path.join(out_dir, "framestats.txt") System.gfxinfo_get(self._target, self.package, self.db_file) # Close and clear application System.force_stop(self._target, self.package, clear=True) # Go back to home screen System.home(self._target) # Switch back to original settings Screen.set_orientation(self._target, auto=True) System.set_airplane_mode(self._target, on=False) Screen.set_brightness(self._target, auto=True)
def run(self, out_dir, test_name, collect=''): """ Run single Vellamo workload. Returns a collection of results. :param out_dir: Path to experiment directory where to store results. :type out_dir: str :param test_name: Name of the test to run :type test_name: str :param collect: Specifies what to collect. Possible values: - 'energy' - 'systrace' - 'ftrace' - any combination of the above :type collect: list(str) """ self.out_dir = out_dir self.collect = collect # Check if the density of the target device screen is different from # the one used to get the values below density = Screen.get_screen_density(self._target) if DEFAULT_DENSITY not in density: msg = 'Screen density of target device differs from {}.\n'\ 'Please set it to {}' raise RuntimeError(msg.format(DEFAULT_DENSITY, DEFAULT_DENSITY)) if test_name.upper() not in VELLAMO_TESTS: raise ValueError('Vellamo workload [%s] not supported', test_name) # Set parameter depending on test self._log.debug('Start Vellamo Benchmark [%s]', test_name) test_x, test_y = (0, 0) sleep_time = 0 if test_name.upper() == 'BROWSER': test_x, test_y = (91, 33) sleep_time = 3.5 elif test_name.upper() == 'METAL': test_x, test_y = (91, 59) sleep_time = 1 elif test_name.upper() == 'MULTI': test_x, test_y = (91, 71) sleep_time = 3.5 # Unlock device screen (assume no password required) Screen.unlock(self._target) System.force_stop(self._target, self.package, clear=True) # Set min brightness Screen.set_brightness(self._target, auto=False, percent=0) # Clear logcat os.system(self._adb('logcat -c')) # Regexps for benchmark synchronization start_logline = r'ActivityManager: Start.*'\ ':com.quicinc.vellamo:benchmarking' VELLAMO_BENCHMARK_START_RE = re.compile(start_logline) self._log.debug("START string [%s]", start_logline) # End of benchmark is marked by displaying results end_logline = r'ActivityManager: START.*'\ 'act=com.quicinc.vellamo.*_RESULTS' VELLAMO_BENCHMARK_END_RE = re.compile(end_logline) self._log.debug("END string [%s]", end_logline) # Parse logcat output lines logcat_cmd = self._adb( 'logcat ActivityManager:* System.out:I *:S BENCH:*'\ .format(self._target.adb_name)) self._log.info("%s", logcat_cmd) # Start the activity System.start_activity(self._target, self.package, self.activity) logcat = Popen(logcat_cmd, shell=True, stdout=PIPE) sleep(2) # Accept EULA System.tap(self._target, 80, 86) sleep(1) # Click Let's Roll System.tap(self._target, 50, 67) sleep(1) # Skip Arrow System.tap(self._target, 46, 78) # Run Workload System.tap(self._target, test_x, test_y) # Skip instructions System.hswipe(self._target, 10, 80, duration=100, swipe_right=False) System.hswipe(self._target, 10, 80, duration=100, swipe_right=False) System.hswipe(self._target, 10, 80, duration=100, swipe_right=False) self._log.info("Vellamo - {} started!".format(test_name.upper())) while True: # read next logcat line (up to max 1024 chars) message = logcat.stdout.readline(1024) # Benchmark start trigger if VELLAMO_BENCHMARK_START_RE.search(message): # Start tracing self.tracingStart() self._log.debug("Benchmark started!") elif VELLAMO_BENCHMARK_END_RE.search(message): # Stop tracing self.tracingStop() break else: continue # Gather scores file from the device db_file = os.path.join(out_dir, VELLAMO_SCORE_NAME) self._target.pull('{}/{}'.format(VELLAMO_DB_PATH, VELLAMO_SCORE_NAME), db_file) System.force_stop(self._target, self.package, clear=True) # Go back to home screen System.home(self._target) # Set brightness back to auto Screen.set_brightness(self._target, auto=True)
def run(self, out_dir, test_name, duration_s, collect=''): """ Run single UiBench workload. :param out_dir: Path to experiment directory where to store results. :type out_dir: str :param test_name: Name of the test to run :type test_name: str :param duration_s: Run benchmak for this required number of seconds :type duration_s: int :param collect: Specifies what to collect. Possible values: - 'energy' - 'systrace' - 'ftrace' - any combination of the above :type collect: list(str) """ activity = '.' + test_name # Keep track of mandatory parameters self.out_dir = out_dir self.collect = collect # Unlock device screen (assume no password required) Screen.unlock(self._target) # Close and clear application System.force_stop(self._target, self.package, clear=True) # Set airplane mode System.set_airplane_mode(self._target, on=True) # Set min brightness Screen.set_brightness(self._target, auto=False, percent=0) # Start the main view of the app which must be running # to reset the frame statistics. System.monkey(self._target, self.package) # Force screen in PORTRAIT mode Screen.set_orientation(self._target, portrait=True) # Reset frame statistics System.gfxinfo_reset(self._target, self.package) sleep(1) # Clear logcat os.system(self._adb('logcat -c')); # Regexps for benchmark synchronization start_logline = r'ActivityManager: START.*'\ 'cmp=com.android.test.uibench/{}'.format(activity) UIBENCH_BENCHMARK_START_RE = re.compile(start_logline) self._log.debug("START string [%s]", start_logline) # Parse logcat output lines logcat_cmd = self._adb( 'logcat ActivityManager:* System.out:I *:S BENCH:*'\ .format(self._target.adb_name)) self._log.info("%s", logcat_cmd) # Start the activity System.start_activity(self._target, self.package, activity) logcat = Popen(logcat_cmd, shell=True, stdout=PIPE) while True: # read next logcat line (up to max 1024 chars) message = logcat.stdout.readline(1024) # Benchmark start trigger match = UIBENCH_BENCHMARK_START_RE.search(message) if match: self.tracingStart() self._log.debug("Benchmark started!") break # Run the workload for the required time self._log.info('Benchmark [%s] started, waiting %d [s]', activity, duration_s) sleep(duration_s) self._log.debug("Benchmark done!") self.tracingStop() # Get frame stats self.db_file = os.path.join(out_dir, "framestats.txt") System.gfxinfo_get(self._target, self.package, self.db_file) # Close and clear application System.force_stop(self._target, self.package, clear=True) # Go back to home screen System.home(self._target) # Switch back to original settings Screen.set_orientation(self._target, auto=True) System.set_airplane_mode(self._target, on=False) Screen.set_brightness(self._target, auto=True)
def run(self, out_dir, test_name, duration_s, collect=''): """ Run single UiBench workload. :param out_dir: Path to experiment directory where to store results. :type out_dir: str :param test_name: Name of the test to run :type test_name: str :param duration_s: Run benchmak for this required number of seconds :type duration_s: int :param collect: Specifies what to collect. Possible values: - 'energy' - 'systrace' - 'ftrace' - any combination of the above :type collect: list(str) """ activity = '.' + test_name # Keep track of mandatory parameters self.out_dir = out_dir self.collect = collect # Press Back button to be sure we run the video from the start System.menu(self._target) System.back(self._target) # Close and clear application System.force_stop(self._target, self.package, clear=True) # Set airplane mode System.set_airplane_mode(self._target, on=True) # Set min brightness Screen.set_brightness(self._target, auto=False, percent=0) # Start the main view of the app which must be running # to reset the frame statistics. System.monkey(self._target, self.package) # Force screen in PORTRAIT mode Screen.set_orientation(self._target, portrait=True) # Reset frame statistics System.gfxinfo_reset(self._target, self.package) sleep(1) # Clear logcat os.system(self._adb('logcat -c')) # Regexps for benchmark synchronization start_logline = r'ActivityManager: START.*'\ 'cmp=com.android.test.uibench/{}'.format(activity) UIBENCH_BENCHMARK_START_RE = re.compile(start_logline) self._log.debug("START string [%s]", start_logline) # Parse logcat output lines logcat_cmd = self._adb( 'logcat ActivityManager:* System.out:I *:S BENCH:*'\ .format(self._target.adb_name)) self._log.info("%s", logcat_cmd) # Start the activity System.start_activity(self._target, self.package, activity) logcat = Popen(logcat_cmd, shell=True, stdout=PIPE) while True: # read next logcat line (up to max 1024 chars) message = logcat.stdout.readline(1024) # Benchmark start trigger match = UIBENCH_BENCHMARK_START_RE.search(message) if match: self.tracingStart() self._log.debug("Benchmark started!") break # Run the workload for the required time self._log.info('Benchmark [%s] started, waiting %d [s]', activity, duration_s) sleep(duration_s) self._log.debug("Benchmark done!") self.tracingStop() # Get frame stats self.db_file = os.path.join(out_dir, "framestats.txt") System.gfxinfo_get(self._target, self.package, self.db_file) # Close and clear application System.force_stop(self._target, self.package, clear=True) # Go back to home screen System.home(self._target) # Switch back to original settings Screen.set_orientation(self._target, auto=True) System.set_airplane_mode(self._target, on=False) Screen.set_brightness(self._target, auto=True)
def run(self, out_dir, test_name, collect=''): """ Run single Geekbench workload. :param out_dir: Path on host to experiment directory where to store results. :type out_dir: str :param test_name: Name of the test to run :type test_name: str :param collect: Specifies what to collect. Possible values: - 'energy' - 'systrace' - 'ftrace' - any combination of the above in a space-separated string. :type collect: list(str) """ # Initialize energy meter results self.out_dir = out_dir self.collect = collect # Clear the stored data for the application, so we always start with # an EULA to clear System.force_stop(self._target, self.package, clear=True) # Clear logcat from any previous runs # do this on the target as then we don't need to build a string self._target.clear_logcat() # Unlock device screen (assume no password required) System.menu(self._target) # Press Back button to be sure we run the benchmark from the start System.back(self._target) # Force screen in PORTRAIT mode Screen.set_orientation(self._target, portrait=True) # Set min brightness Screen.set_brightness(self._target, auto=False, percent=0) # Start app on the target device System.start_activity(self._target, self.package, self.activity) # Allow the activity to start sleep(2) # Parse logcat output lines to find beginning and end logcat_cmd = self._adb( 'logcat ActivityManager:* System.out:I *:S GEEKBENCH_RESULT:*'\ .format(self._target.adb_name)) self._log.info("%s", logcat_cmd) logcat = Popen(logcat_cmd, shell=True, stdout=PIPE) # Click to accept the EULA System.tap(self._target, 73, 55) sleep(1) # The main window opened will have the CPU benchmark # Swipe to get the COMPUTE one if test_name.upper() == 'COMPUTE': System.hswipe(self._target, 10, 80, duration=100, swipe_right=False) # Press the 'RUN <test_name> BENCHMARK' button System.tap(self._target, 73, 72) while True: # read next logcat line (up to max 1024 chars) message = logcat.stdout.readline(1024) # Benchmark start trigger match = GEEKBENCH_BENCHMARK_START_RE.search(message) if match: # Start tracing self.tracingStart() self._log.debug("Benchmark started!") # Benchmark end trigger match = GEEKBENCH_BENCHMARK_END_RE.search(message) if match: # Stop tracing self.tracingStop() remote_result_file = match.group('results_file') self._log.debug("Benchmark finished! Results are in {}".format(remote_result_file)) break # Get Geekbench Results file target_result_file = self._target.path.basename(remote_result_file) result_file = os.path.join(self.out_dir, target_result_file) self._log.debug("result_file={}".format(result_file)) self._target.pull(remote_result_file, result_file) # Close the app System.force_stop(self._target, self.package, clear=False) # Go back to home screen System.home(self._target) # Switch back to screen auto rotation Screen.set_orientation(self._target, auto=True) # Set brightness back to auto Screen.set_brightness(self._target, auto=True)
def run(self, out_dir, test_name, collect=''): """ Run single Vellamo workload. Returns a collection of results. :param out_dir: Path to experiment directory where to store results. :type out_dir: str :param test_name: Name of the test to run :type test_name: str :param collect: Specifies what to collect. Possible values: - 'energy' - 'systrace' - 'ftrace' - any combination of the above :type collect: list(str) """ self.out_dir = out_dir self.collect = collect # Check if the density of the target device screen is different from # the one used to get the values below density = Screen.get_screen_density(self._target) if DEFAULT_DENSITY not in density: msg = 'Screen density of target device differs from {}.\n'\ 'Please set it to {}' raise RuntimeError(msg.format(DEFAULT_DENSITY, DEFAULT_DENSITY)) if test_name.upper() not in VELLAMO_TESTS: raise ValueError('Vellamo workload [%s] not supported', test_name) # Set parameter depending on test self._log.debug('Start Vellamo Benchmark [%s]', test_name) test_x, test_y = (0, 0) sleep_time = 0 if test_name.upper() == 'BROWSER': test_x, test_y = (91, 33) sleep_time = 3.5 elif test_name.upper() == 'METAL': test_x, test_y = (91, 59) sleep_time = 1 elif test_name.upper() == 'MULTI': test_x, test_y = (91, 71) sleep_time = 3.5 # Unlock device screen (assume no password required) Screen.unlock(self._target) System.force_stop(self._target, self.package, clear=True) # Set min brightness Screen.set_brightness(self._target, auto=False, percent=0) # Clear logcat os.system(self._adb('logcat -c')); # Regexps for benchmark synchronization start_logline = r'ActivityManager: Start.*'\ ':com.quicinc.vellamo:benchmarking' VELLAMO_BENCHMARK_START_RE = re.compile(start_logline) self._log.debug("START string [%s]", start_logline) # End of benchmark is marked by displaying results end_logline = r'ActivityManager: START.*'\ 'act=com.quicinc.vellamo.*_RESULTS' VELLAMO_BENCHMARK_END_RE = re.compile(end_logline) self._log.debug("END string [%s]", end_logline) # Parse logcat output lines logcat_cmd = self._adb( 'logcat ActivityManager:* System.out:I *:S BENCH:*'\ .format(self._target.adb_name)) self._log.info("%s", logcat_cmd) # Start the activity System.start_activity(self._target, self.package, self.activity) logcat = Popen(logcat_cmd, shell=True, stdout=PIPE) sleep(2) # Accept EULA System.tap(self._target, 80, 86) sleep(1) # Click Let's Roll System.tap(self._target, 50, 67) sleep(1) # Skip Arrow System.tap(self._target, 46, 78) # Run Workload System.tap(self._target, test_x, test_y) # Skip instructions System.hswipe(self._target, 10, 80, duration=100, swipe_right=False) System.hswipe(self._target, 10, 80, duration=100, swipe_right=False) System.hswipe(self._target, 10, 80, duration=100, swipe_right=False) self._log.info("Vellamo - {} started!".format(test_name.upper())) while True: # read next logcat line (up to max 1024 chars) message = logcat.stdout.readline(1024) # Benchmark start trigger if VELLAMO_BENCHMARK_START_RE.search(message): # Start tracing self.tracingStart() self._log.debug("Benchmark started!") elif VELLAMO_BENCHMARK_END_RE.search(message): # Stop tracing self.tracingStop() break else: continue # Gather scores file from the device db_file = os.path.join(out_dir, VELLAMO_SCORE_NAME) self._target.pull('{}/{}'.format(VELLAMO_DB_PATH, VELLAMO_SCORE_NAME), db_file) System.force_stop(self._target, self.package, clear=True) # Go back to home screen System.home(self._target) # Set brightness back to auto Screen.set_brightness(self._target, auto=True)
def run(self, out_dir, test_name, collect=''): """ Run single Geekbench workload. :param out_dir: Path on host to experiment directory where to store results. :type out_dir: str :param test_name: Name of the test to run :type test_name: str :param collect: Specifies what to collect. Possible values: - 'energy' - 'systrace' - 'ftrace' - any combination of the above in a space-separated string. :type collect: list(str) """ # Initialize energy meter results self.out_dir = out_dir self.collect = collect # Clear the stored data for the application, so we always start with # an EULA to clear System.force_stop(self._target, self.package, clear=True) # Clear logcat from any previous runs # do this on the target as then we don't need to build a string self._target.clear_logcat() # Unlock device screen (assume no password required) System.menu(self._target) # Press Back button to be sure we run the benchmark from the start System.back(self._target) # Force screen in PORTRAIT mode Screen.set_orientation(self._target, portrait=True) # Set min brightness Screen.set_brightness(self._target, auto=False, percent=0) # Start app on the target device System.start_activity(self._target, self.package, self.activity) # Allow the activity to start sleep(2) # Parse logcat output lines to find beginning and end logcat_cmd = self._adb( 'logcat ActivityManager:* System.out:I *:S GEEKBENCH_RESULT:*'\ .format(self._target.adb_name)) self._log.info("%s", logcat_cmd) logcat = Popen(logcat_cmd, shell=True, stdout=PIPE) # Click to accept the EULA System.tap(self._target, 73, 55) sleep(1) # The main window opened will have the CPU benchmark # Swipe to get the COMPUTE one if test_name.upper() == 'COMPUTE': System.hswipe(self._target, 10, 80, duration=100, swipe_right=False) # Press the 'RUN <test_name> BENCHMARK' button System.tap(self._target, 73, 72) while True: # read next logcat line (up to max 1024 chars) message = logcat.stdout.readline(1024) # Benchmark start trigger match = GEEKBENCH_BENCHMARK_START_RE.search(message) if match: # Start tracing self.tracingStart() self._log.debug("Benchmark started!") # Benchmark end trigger match = GEEKBENCH_BENCHMARK_END_RE.search(message) if match: # Stop tracing self.tracingStop() remote_result_file = match.group('results_file') self._log.debug("Benchmark finished! Results are in {}".format( remote_result_file)) break # Get Geekbench Results file target_result_file = self._target.path.basename(remote_result_file) result_file = os.path.join(self.out_dir, target_result_file) self._log.debug("result_file={}".format(result_file)) self._target.pull(remote_result_file, result_file) # Close the app System.force_stop(self._target, self.package, clear=False) # Go back to home screen System.home(self._target) # Switch back to screen auto rotation Screen.set_orientation(self._target, auto=True) # Set brightness back to auto Screen.set_brightness(self._target, auto=True)