def run(self, out_dir, uri, portrait=True, collect=''):
"""
Run viewer workload
:param out_dir: Path to experiment directory where to store results.
:type out_dir: str
:param uri: The URI of the item to display
:type location_search: str
:param portrait: If True, display mode will be set to 'portrait' prior
to viewing the item. If False, 'landscape' mode will be set.
:param collect: Specifies what to collect. Possible values:
- 'energy'
- 'systrace'
- 'ftrace'
- any combination of the above
:type collect: list(str)
"""
# Keep track of mandatory parameters
self.out_dir = out_dir
self.collect = collect
# Set min brightness
Screen.set_brightness(self._target, auto=False, percent=0)
# Unlock device screen (assume no password required)
Screen.unlock(self._target)
# Force screen in requested orientation
Screen.set_orientation(self._target, portrait=portrait)
System.gfxinfo_reset(self._target, self.package)
# Wait for gfxinfo reset to be completed
sleep(1)
# Open the requested uri
System.view_uri(self._target, uri)
self.pre_interact()
self.tracingStart()
self.interact()
self.tracingStop()
self.post_interact()
# Get frame stats
self.db_file = os.path.join(out_dir, "framestats.txt")
System.gfxinfo_get(self._target, self.package, self.db_file)
# Go back to home screen
System.home(self._target)
# Set brightness back to auto
Screen.set_brightness(self._target, auto=True)
# Switch back to screen auto rotation
Screen.set_orientation(self._target, auto=True)
def run(self, exp_dir, video_url, video_duration_s, collect=''):
# Initialize energy meter results
nrg_report = None
# Unlock device screen (assume no password required)
self.target.execute('input keyevent 82')
# Press Back button to be sure we run the video from the start
self.target.execute('input keyevent KEYCODE_BACK')
# Force screen in LANDSCAPE mode
Screen.set_orientation(self.target, portrait=False)
# Start YouTube video on the target device
youtube_cmd = 'am start -a android.intent.action.VIEW "{}"'\
.format(video_url)
logging.info(youtube_cmd)
self.target.execute(youtube_cmd)
# Allow the activity to start
sleep(3)
# Reset framestats collection
self.target.execute('dumpsys gfxinfo --reset')
# Start energy collection
if 'energy' in collect and self.te.emeter:
self.te.emeter.reset()
# Wait until the end of the video
logging.info("Play video for %d [s]", video_duration_s)
sleep(video_duration_s)
# Stop energy collection
if 'energy' in collect and self.te.emeter:
nrg_report = self.te.emeter.report(exp_dir)
logging.info("Estimated energy: %7.3f",
float(nrg_report.channels['BAT']))
# Get frame stats
db_file = os.path.join(exp_dir, "framestats.txt")
self._adb(YOUTUBE_CMD.format(db_file))
# Close and clear application
self.target.execute('am force-stop com.google.android.youtube')
self.target.execute('pm clear com.google.android.youtube')
# Go back to home screen
self.target.execute('input keyevent KEYCODE_HOME')
# Switch back to screen auto rotation
Screen.set_orientation(self.target, auto=True)
return db_file, nrg_report
def run(self, exp_dir, video_url, video_duration_s, collect=''):
# Initialize energy meter results
nrg_report = None
# Unlock device screen (assume no password required)
System.menu(self.target)
# Press Back button to be sure we run the video from the start
System.back(self.target)
# Use the monkey tool to start YouTube without playing any video.
# This allows to subsequently set the screen orientation to LANDSCAPE
# and to reset the frame statistics.
System.monkey(self.target, self.package)
# Force screen in LANDSCAPE mode
Screen.set_orientation(self.target, portrait=False)
System.gfxinfo_reset(self.target, self.package)
sleep(1)
# Start YouTube video on the target device
System.start_action(self.target, self.action, video_url)
# Allow the activity to start
sleep(1)
# Start energy collection
if 'energy' in collect and self.te.emeter:
self.te.emeter.reset()
# Wait until the end of the video
self.logger.info("Play video for %d [s]", video_duration_s)
sleep(video_duration_s)
# Stop energy collection
if 'energy' in collect and self.te.emeter:
nrg_report = self.te.emeter.report(exp_dir)
# Get frame stats
db_file = os.path.join(exp_dir, "framestats.txt")
System.gfxinfo_get(self.target, self.package, db_file)
# Close the app without clearing the local data to
# avoid the dialog to select the account at next start
System.force_stop(self.target, self.package, clear=False)
# Go back to home screen
System.home(self.target)
# Switch back to screen auto rotation
Screen.set_orientation(self.target, auto=True)
return db_file, nrg_report
def run(self, out_dir, location_search, swipe_count=5, collect=''):
"""
Run single Gmaps workload.
:param out_dir: Path to experiment directory where to store results.
:type out_dir: str
:param location_search: Search string to be used in GMaps
:type location_search: str
:param swipe_count: Number of sets of (left, right, up, down) swipes to do
:type swipe_count: int
:param collect: Specifies what to collect. Possible values:
- 'energy'
- 'systrace'
- 'ftrace'
- any combination of the above
:type collect: list(str)
"""
# Keep track of mandatory parameters
self.out_dir = out_dir
self.collect = collect
# Set min brightness
Screen.set_brightness(self._target, auto=False, percent=0)
# Unlock device screen (assume no password required)
Screen.unlock(self._target)
# Use the monkey tool to start GMaps
# This allows to subsequently set the screen orientation to LANDSCAPE
# and to reset the frame statistics.
System.monkey(self._target, self.package)
# Force screen in PORTRAIT mode
Screen.set_orientation(self._target, portrait=True)
System.gfxinfo_reset(self._target, self.package)
sleep(1)
# Start GMaps on the target device
loc_url = 'geo:0,0?q='
loc_url += '+'.join(location_search.split())
System.start_action(self._target, self.action, loc_url)
# Allow the activity to start
sleep(1)
script = TargetScript(self._te, "gmaps_swiper.sh")
self._log.debug('Accumulating commands')
for i in range(swipe_count):
System.hswipe(script, 20, 80, 100, True)
script.append('sleep 1')
System.hswipe(script, 20, 80, 100, False)
script.append('sleep 1')
System.vswipe(script, 40, 60, 100, True)
script.append('sleep 1')
System.vswipe(script, 40, 60, 100, False)
script.append('sleep 1')
self._log.debug('Accumulation done')
# Push script to the target
script.push()
self._log.info('Opening GMaps to [%s]', loc_url)
# Let GMaps zoom in on the location
sleep(2)
self.tracingStart()
self._log.info('Launching target script')
script.run()
self._log.info('Target script ended')
self.tracingStop()
# Get frame stats
self.db_file = os.path.join(out_dir, "framestats.txt")
System.gfxinfo_get(self._target, self.package, self.db_file)
# Close the app without clearing the local data to
# avoid the dialog to select the account at next start
System.force_stop(self._target, self.package, clear=False)
# Go back to home screen
System.home(self._target)
# Set brightness back to auto
Screen.set_brightness(self._target, auto=True)
# Switch back to screen auto rotation
Screen.set_orientation(self._target, auto=True)
def run(self, out_dir, test_name, collect=''):
"""
Run single PCMark workload. Returns a collection of results.
:param out_dir: Path to experiment directory on the host
where to store results.
:type out_dir: str
:param test_name: Name of the test to run
:type test_name: str
:param collect: Specifies what to collect. Possible values:
- 'energy'
- 'systrace'
- 'ftrace'
- any combination of the above as a single space-separated string.
:type collect: list(str)
"""
self.out_dir = out_dir
self.collect = collect
# Unlock device screen (assume no password required)
System.menu(self._target)
# Make sure we exit the app if already open
System.back(self._target)
# Close and NOT clear application (benchmark tests are downloaded from web)
System.force_stop(self._target, self.package, clear=False)
# Set airplane mode
System.set_airplane_mode(self._target, on=True)
# Set min brightness
Screen.set_brightness(self._target, auto=False, percent=0)
# Force screen in PORTRAIT mode
Screen.set_orientation(self._target, portrait=True)
monitor = self._target.get_logcat_monitor(REGEXPS.values())
# Start PCMark on the target device
System.start_activity(self._target, self.package, self.activity)
# Wait a few seconds while application is being loaded
time.sleep(5)
if test_name.lower() == 'work':
# Move to benchmark run page
System.tab(self._target)
System.enter(self._target)
# Wait for page animations to end
time.sleep(10)
else:
raise ValueError('PCMark test [%s] not supported', test_name)
# Start benchmark
monitor.start()
System.enter(self._target)
monitor.wait_for(REGEXPS['start'])
self.tracingStart()
self._log.info('PCmark started!')
monitor.wait_for(REGEXPS['end'], timeout=600)
self.tracingStop()
self._log.info('PCmark ended!')
# That should only match one line, but use the most recent one
# in case logcat wasn't cleared properly
result = monitor.wait_for(REGEXPS['result'])[-1]
monitor.stop()
remote_archive = re.match(REGEXPS['result'], result).group('path')
local_archive = os.path.join(out_dir, self._target.path.basename(remote_archive))
self._target.pull(remote_archive, local_archive, as_root=True)
# Several files in the archive
# Only "Result.xml" matters to us
with ZipFile(local_archive, 'r') as archive:
archive.extractall(out_dir)
# Fetch workloads names and scores
scores = {}
input_result_filepath = os.path.join(out_dir, INPUT_RESULT_FILE)
with open(input_result_filepath, 'r') as fd:
for line in fd:
match = re.match(REGEXPS['score'], line)
if match:
scores[match.group('name')] = match.group('score')
# Dump scores to json
output_result_filepath = os.path.join(out_dir, OUTPUT_RESULT_FILE)
with open(output_result_filepath, 'w') as fd:
json.dump(scores, fd)
self._log.info('Scores available in {}'.format(output_result_filepath))
System.force_stop(self._target, self.package, clear=False)
# Restore default configuration
System.home(self._target)
# Set orientation back to auto
Screen.set_orientation(self._target, auto=True)
# Go back to home screen
System.home(self._target)
# Turn off airplane mode
System.set_airplane_mode(self._target, on=False)
# Set brightness back to auto
Screen.set_brightness(self._target, auto=True)
def run(self, out_dir, test_name, iterations, collect):
"""
Run Jankbench workload for a number of iterations.
Returns a collection of results.
:param out_dir: Path to experiment directory on the host
where to store results.
:type out_dir: str
:param test_name: Name of the test to run
:type test_name: str
:param iterations: Number of iterations for the named test
:type iterations: int
:param collect: Specifies what to collect. Possible values:
- 'energy'
- 'systrace'
- 'ftrace'
- any combination of the above as a single space-separated string.
:type collect: list(str)
"""
# Keep track of mandatory parameters
self.out_dir = out_dir
self.collect = collect
# Setup test id
try:
test_id = _jankbench[test_name]
except KeyError:
raise ValueError('Jankbench test [%s] not supported', test_name)
# Unlock device screen (assume no password required)
Screen.unlock(self._target)
# Close and clear application
System.force_stop(self._target, self.package, clear=True)
# Set airplane mode
System.set_airplane_mode(self._target, on=True)
# Set min brightness
Screen.set_brightness(self._target, auto=False, percent=0)
# Force screen in PORTRAIT mode
Screen.set_orientation(self._target, portrait=True)
# Clear logcat
self._target.clear_logcat()
self._log.debug('Start Jank Benchmark [%d:%s]', test_id, test_name)
test_cmd = 'am start -n "com.android.benchmark/.app.RunLocalBenchmarksActivity" '\
'--eia "com.android.benchmark.EXTRA_ENABLED_BENCHMARK_IDS" {0} '\
'--ei "com.android.benchmark.EXTRA_RUN_COUNT" {1}'\
.format(test_id, iterations)
self._log.info(test_cmd)
self._target.execute(test_cmd);
# Parse logcat output lines
logcat_cmd = self._adb(
'logcat ActivityManager:* System.out:I *:S BENCH:*'\
.format(self._target.adb_name))
self._log.info(logcat_cmd)
self._log.debug('Iterations:')
logcat = Popen(logcat_cmd, shell=True, stdout=PIPE)
while True:
# read next logcat line (up to max 1024 chars)
message = logcat.stdout.readline(1024)
# Benchmark start trigger
match = JANKBENCH_BENCHMARK_START_RE.search(message)
if match:
self.tracingStart()
self._log.debug('Benchmark started!')
# Benchmark completed trigger
match = JANKBENCH_BENCHMARK_DONE_RE.search(message)
if match:
self._log.debug('Benchmark done!')
self.tracingStop()
break
# Iteration completd
match = JANKBENCH_ITERATION_COUNT_RE.search(message)
if match:
self._log.debug('Iteration %2d:',
int(match.group('iteration'))+1)
# Iteration metrics
match = JANKBENCH_ITERATION_METRICS_RE.search(message)
if match:
self._log.info(' Mean: %7.3f JankP: %7.3f StdDev: %7.3f Count Bad: %4d Count Jank: %4d',
float(match.group('mean')),
float(match.group('junk_p')),
float(match.group('std_dev')),
int(match.group('count_bad')),
int(match.group('count_junk')))
# Get results
self.db_file = os.path.join(out_dir, JANKBENCH_DB_NAME)
self._target.pull(JANKBENCH_DB_PATH + JANKBENCH_DB_NAME, self.db_file)
System.force_stop(self._target, self.package, clear=True)
# Go back to home screen
System.home(self._target)
# Reset initial setup
# Set orientation back to auto
Screen.set_orientation(self._target, auto=True)
# Turn off airplane mode
System.set_airplane_mode(self._target, on=False)
# Set brightness back to auto
Screen.set_brightness(self._target, auto=True)
def run(self, out_dir, duration_s=10, collect='surfaceflinger'):
"""
Run a camera startup workload
:param out_dir: Path to experiment directory where to store results.
:type out_dir: str
:param duration_s: Duration of test
:type duration_s: int
:param collect: Specifies what to collect. Possible values:
- 'energy'
- 'systrace'
- 'ftrace'
- any combination of the above
:type collect: list(str)
"""
self._log.info(
"Running CameraStartup for {}s and collecting {}".format(
duration_s, collect))
# Keep track of mandatory parameters
self.out_dir = out_dir
self.collect = collect
# Unlock device screen (assume no password required)
Screen.unlock(self._target)
# Set airplane mode
System.set_airplane_mode(self._target, on=True)
# Set min brightness
Screen.set_brightness(self._target, auto=False, percent=0)
# Force screen in PORTRAIT mode
Screen.set_orientation(self._target, portrait=True)
sleep(1)
self.tracingStart()
# Wait for a few seconds so that you can clear see start of trace and start of camera app
sleep(3)
# Use the monkey tool to start CameraStartup
System.monkey(self._target, self.package)
sleep(duration_s)
self.tracingStop()
# Close the app without clearing the local data to
# avoid the dialog to select the account at next start
System.force_stop(self._target, self.package, clear=False)
# Go back to home screen
System.home(self._target)
# Set brightness back to auto
Screen.set_brightness(self._target, auto=True)
# Switch back to screen auto rotation
Screen.set_orientation(self._target, auto=True)
# Switch off airplane mode
System.set_airplane_mode(self._target, on=False)
def run(self, out_dir, location_search, swipe_count=5, collect=''):
"""
Run single Gmaps workload.
:param out_dir: Path to experiment directory where to store results.
:type out_dir: str
:param location_search: Search string to be used in GMaps
:type location_search: str
:param swipe_count: Number of sets of (left, right, up, down) swipes to do
:type swipe_count: int
:param collect: Specifies what to collect. Possible values:
- 'energy'
- 'systrace'
- 'ftrace'
- any combination of the above
:type collect: list(str)
"""
# Keep track of mandatory parameters
self.out_dir = out_dir
self.collect = collect
# Set min brightness
Screen.set_brightness(self._target, auto=False, percent=0)
# Unlock device screen (assume no password required)
System.menu(self._target)
# Press Back button to be sure we run the video from the start
System.back(self._target)
# Use the monkey tool to start GMaps
# This allows to subsequently set the screen orientation to LANDSCAPE
# and to reset the frame statistics.
System.monkey(self._target, self.package)
# Force screen in PORTRAIT mode
Screen.set_orientation(self._target, portrait=True)
System.gfxinfo_reset(self._target, self.package)
sleep(1)
# Start GMaps on the target device
loc_url = 'geo:0,0?q='
loc_url += '+'.join(location_search.split())
System.start_action(self._target, self.action, loc_url)
# Allow the activity to start
sleep(1)
self.tracingStart()
self._log.info('Opening GMaps to [%s]', loc_url)
# Let GMaps zoom in on the location
sleep(2)
for i in range(swipe_count):
System.hswipe(self._target, 20, 80, 100, True)
sleep(.5)
System.hswipe(self._target, 20, 80, 100, False)
sleep(.5)
System.vswipe(self._target, 40, 60, 100, True)
sleep(.5)
System.vswipe(self._target, 40, 60, 100, False)
sleep(.5)
self.tracingStop()
# Get frame stats
self.db_file = os.path.join(out_dir, "framestats.txt")
System.gfxinfo_get(self._target, self.package, self.db_file)
# Close the app without clearing the local data to
# avoid the dialog to select the account at next start
System.force_stop(self._target, self.package, clear=False)
# Go back to home screen
System.home(self._target)
# Set brightness back to auto
Screen.set_brightness(self._target, auto=True)
# Switch back to screen auto rotation
Screen.set_orientation(self._target, auto=True)
def run(self, out_dir, test_name, iterations=10, collect=''):
"""
Run single UiBench workload.
:param out_dir: Path to experiment directory where to store results.
:type out_dir: str
:param test_name: Name of the test to run
:type test_name: str
:param iterations: Run benchmak for this required number of iterations
:type iterations: int
:param collect: Specifies what to collect. Possible values:
- 'systrace'
- 'ftrace'
:type collect: list(str)
"""
if 'energy' in collect:
raise ValueError('UiBench workload does not support energy data collection')
activity = '.' + test_name
# Keep track of mandatory parameters
self.out_dir = out_dir
self.collect = collect
# Filter out test overhead
filter_prop = System.get_boolean_property(self._target, 'debug.hwui.filter_test_overhead')
if not filter_prop:
System.set_property(
self._target, 'debug.hwui.filter_test_overhead', 'true', restart=True)
# Unlock device screen (assume no password required)
Screen.unlock(self._target)
# Close and clear application
System.force_stop(self._target, self.package, clear=True)
# Set airplane mode
System.set_airplane_mode(self._target, on=True)
# Set min brightness
Screen.set_brightness(self._target, auto=False, percent=0)
# Force screen in PORTRAIT mode
Screen.set_orientation(self._target, portrait=True)
# Clear logcat
os.system(self._adb('logcat -c'));
# Regexps for benchmark synchronization
start_logline = r'TestRunner: started'
UIBENCH_BENCHMARK_START_RE = re.compile(start_logline)
self._log.debug("START string [%s]", start_logline)
finish_logline = r'TestRunner: finished'
UIBENCH_BENCHMARK_FINISH_RE = re.compile(finish_logline)
self._log.debug("FINISH string [%s]", start_logline)
# Parse logcat output lines
logcat_cmd = self._adb(
'logcat TestRunner:* System.out:I *:S BENCH:*'\
.format(self._target.adb_name))
self._log.info("%s", logcat_cmd)
command = "am instrument -e iterations {} -e class {}{} -w {}".format(
iterations, self.test_package, activity, self.test_package)
test_proc = self._target.background(command)
logcat = Popen(logcat_cmd, shell=True, stdout=PIPE)
while True:
# read next logcat line (up to max 1024 chars)
message = logcat.stdout.readline(1024)
# Benchmark start trigger
match = UIBENCH_BENCHMARK_START_RE.search(message)
if match:
self.tracingStart()
self._log.debug("Benchmark started!")
match = UIBENCH_BENCHMARK_FINISH_RE.search(message)
if match:
self.tracingStop()
self._log.debug("Benchmark finished!")
test_proc.wait()
break
# Get frame stats
self.db_file = os.path.join(out_dir, "framestats.txt")
with open(self.db_file, 'w') as f:
f.writelines(test_proc.stdout.readlines())
self.results = self.get_results(out_dir)
# Close and clear application
System.force_stop(self._target, self.package, clear=True)
# Go back to home screen
System.home(self._target)
# Switch back to original settings
Screen.set_orientation(self._target, auto=True)
System.set_airplane_mode(self._target, on=False)
Screen.set_brightness(self._target, auto=True)
def run(self, exp_dir, test_name, iterations, collect=''):
# Setup test id
try:
test_id = _jankbench[test_name]
except KeyError:
raise ValueError('Jankbench test [%s] not supported', test_name)
# Initialize energy meter results
nrg_report = None
# Make sure we exit the app if already open
System.menu(self.target)
System.back(self.target)
# Close and clear application
System.force_stop(self.target, self.package, clear=True)
# Set airplane mode
System.set_airplane_mode(self.target, on=True)
# Force screen in PORTRAIT mode
Screen.set_orientation(self.target, portrait=True)
# Clear logcat
os.system(self._adb('logcat -c'));
self.logger.debug('Start Jank Benchmark [%d:%s]', test_id, test_name)
test_cmd = 'am start -n "com.android.benchmark/.app.RunLocalBenchmarksActivity" '\
'--eia "com.android.benchmark.EXTRA_ENABLED_BENCHMARK_IDS" {0} '\
'--ei "com.android.benchmark.EXTRA_RUN_COUNT" {1}'\
.format(test_id, iterations)
self.logger.info(test_cmd)
self.target.execute(test_cmd);
# Parse logcat output lines
logcat_cmd = self._adb(
'logcat ActivityManager:* System.out:I *:S BENCH:*'\
.format(self.target.adb_name))
self.logger.info("%s", logcat_cmd)
self.logger.debug("Iterations:")
logcat = Popen(logcat_cmd, shell=True, stdout=PIPE)
while True:
# read next logcat line (up to max 1024 chars)
message = logcat.stdout.readline(1024)
# Benchmark start trigger
match = JANKBENCH_BENCHMARK_START_RE.search(message)
if match:
if 'energy' in collect and self.te.emeter:
self.te.emeter.reset()
self.logger.debug("Benchmark started!")
# Benchmark completed trigger
match = JANKBENCH_BENCHMARK_DONE_RE.search(message)
if match:
if 'energy' in collect and self.te.emeter:
nrg_report = self.te.emeter.report(exp_dir)
self.logger.info("Estimated energy: %7.3f",
float(nrg_report.channels['BAT']))
self.logger.debug("Benchmark done!")
break
# Iteration completd
match = JANKBENCH_ITERATION_COUNT_RE.search(message)
if match:
self.logger.debug("Iteration %2d:",
int(match.group('iteration'))+1)
# Iteration metrics
match = JANKBENCH_ITERATION_METRICS_RE.search(message)
if match:
self.logger.info(" Mean: %7.3f JankP: %7.3f StdDev: %7.3f Count Bad: %4d Count Jank: %4d",
float(match.group('mean')),
float(match.group('junk_p')),
float(match.group('std_dev')),
int(match.group('count_bad')),
int(match.group('count_junk')))
# get results
db_file = os.path.join(exp_dir, JANKBENCH_DB_NAME)
self.target.pull(JANKBENCH_DB_PATH + JANKBENCH_DB_NAME, db_file)
System.force_stop(self.target, self.package, clear=True)
# Go back to home screen
System.home(self.target)
# Switch back to screen auto rotation
Screen.set_orientation(self.target, auto=True)
return db_file, nrg_report
def run(self, out_dir, test_name, collect=''):
"""
Run single Geekbench workload.
:param out_dir: Path on host to experiment directory where
to store results.
:type out_dir: str
:param test_name: Name of the test to run
:type test_name: str
:param collect: Specifies what to collect. Possible values:
- 'energy'
- 'systrace'
- 'ftrace'
- any combination of the above in a space-separated string.
:type collect: list(str)
"""
# Initialize energy meter results
self.out_dir = out_dir
self.collect = collect
# Clear the stored data for the application, so we always start with
# an EULA to clear
System.force_stop(self._target, self.package, clear=True)
# Clear logcat from any previous runs
# do this on the target as then we don't need to build a string
self._target.clear_logcat()
# Unlock device screen (assume no password required)
System.menu(self._target)
# Press Back button to be sure we run the benchmark from the start
System.back(self._target)
# Force screen in PORTRAIT mode
Screen.set_orientation(self._target, portrait=True)
# Set min brightness
Screen.set_brightness(self._target, auto=False, percent=0)
# Start app on the target device
System.start_activity(self._target, self.package, self.activity)
# Allow the activity to start
sleep(2)
# Parse logcat output lines to find beginning and end
logcat_cmd = self._adb(
'logcat ActivityManager:* System.out:I *:S GEEKBENCH_RESULT:*'\
.format(self._target.adb_name))
self._log.info("%s", logcat_cmd)
logcat = Popen(logcat_cmd, shell=True, stdout=PIPE)
# Click to accept the EULA
System.tap(self._target, 73, 55)
sleep(1)
# The main window opened will have the CPU benchmark
# Swipe to get the COMPUTE one
if test_name.upper() == 'COMPUTE':
System.hswipe(self._target, 10, 80, duration=100, swipe_right=False)
# Press the 'RUN <test_name> BENCHMARK' button
System.tap(self._target, 73, 72)
while True:
# read next logcat line (up to max 1024 chars)
message = logcat.stdout.readline(1024)
# Benchmark start trigger
match = GEEKBENCH_BENCHMARK_START_RE.search(message)
if match:
# Start tracing
self.tracingStart()
self._log.debug("Benchmark started!")
# Benchmark end trigger
match = GEEKBENCH_BENCHMARK_END_RE.search(message)
if match:
# Stop tracing
self.tracingStop()
remote_result_file = match.group('results_file')
self._log.debug("Benchmark finished! Results are in {}".format(remote_result_file))
break
# Get Geekbench Results file
target_result_file = self._target.path.basename(remote_result_file)
result_file = os.path.join(self.out_dir, target_result_file)
self._log.debug("result_file={}".format(result_file))
self._target.pull(remote_result_file, result_file)
# Close the app
System.force_stop(self._target, self.package, clear=False)
# Go back to home screen
System.home(self._target)
# Switch back to screen auto rotation
Screen.set_orientation(self._target, auto=True)
# Set brightness back to auto
Screen.set_brightness(self._target, auto=True)
def run(self, out_dir, collect, media_file, from_device=False, play_duration_s=None):
"""
Run Exoplayer workload
:param out_dir: Path to experiment directory on the host
where to store results.
:type out_dir: str
:param collect: Specifies what to collect. Possible values:
- 'energy'
- 'systrace'
- 'ftrace'
- any combination of the above as a single space-separated string.
:type collect: list(str)
:param media_file: Filepath of the media to play
Path on device if 'from_device' is True
Path on host if 'from_device' is False (default)
:type media_file: str
:param from_device: Whether file to play is already on the device
:type from_device: bool
:param play_duration_s: If set, maximum duration (seconds) of the media playback
If not set, media will play to completion
:type play_duration_s: int
"""
# Keep track of mandatory parameters
self.out_dir = out_dir
self.collect = collect
self.media_file = media_file
self.from_device = from_device
self.play_duration_s = play_duration_s
# Check media file exists
if from_device and not self._target.file_exists(self.media_file):
raise RuntimeError('Cannot find "{}" on target'.format(self.media_file))
elif not from_device and not os.path.isfile(self.media_file):
raise RuntimeError('Cannot find "{}" on host'.format(self.media_file))
# Unlock device screen (assume no password required)
Screen.unlock(self._target)
# Close and clear application
System.force_stop(self._target, self.package, clear=True)
# Enable airplane mode
System.set_airplane_mode(self._target, on=True)
# Set min brightness
Screen.set_brightness(self._target, auto=False, percent=0)
# Force screen in PORTRAIT mode
Screen.set_orientation(self._target, portrait=True)
# Launch Exoplayer benchmark
self._play()
# Go back to home screen
System.home(self._target)
# Set orientation back to auto
Screen.set_orientation(self._target, auto=True)
# Set brightness back to auto
Screen.set_brightness(self._target, auto=True)
# Turn off airplane mode
System.set_airplane_mode(self._target, on=False)
# Close and clear application
System.force_stop(self._target, self.package, clear=True)
def run(self, out_dir, test_name, iterations, collect='gfxinfo'):
"""
Run single SystemUi jank test workload.
:param out_dir: Path to experiment directory where to store results.
:type out_dir: str
:param test_name: Name of the test to run
:type test_name: str
:param iterations: Run benchmark for this required number of iterations
:type iterations: int
:param collect: Specifies what to collect. Possible values:
- 'systrace'
- 'ftrace'
- 'gfxinfo'
- 'surfaceflinger'
- any combination of the above
:type collect: list(str)
"""
if 'energy' in collect:
raise ValueError(
'SystemUi workload does not support energy data collection')
activity = '.' + test_name
# Keep track of mandatory parameters
self.out_dir = out_dir
self.collect = collect
# Unlock device screen (assume no password required)
Screen.unlock(self._target)
# Close and clear application
System.force_stop(self._target, self.package, clear=True)
# Set airplane mode
System.set_airplane_mode(self._target, on=True)
# Set min brightness
Screen.set_brightness(self._target, auto=False, percent=0)
# Force screen in PORTRAIT mode
Screen.set_orientation(self._target, portrait=True)
# Reset frame statistics
System.gfxinfo_reset(self._target, self.package)
sleep(1)
# Clear logcat
os.system(self._adb('logcat -c'))
# Regexps for benchmark synchronization
start_logline = r'TestRunner: started'
SYSTEMUI_BENCHMARK_START_RE = re.compile(start_logline)
self._log.debug("START string [%s]", start_logline)
finish_logline = r'TestRunner: finished'
SYSTEMUI_BENCHMARK_FINISH_RE = re.compile(finish_logline)
self._log.debug("FINISH string [%s]", finish_logline)
# Parse logcat output lines
logcat_cmd = self._adb(
'logcat TestRunner:* System.out:I *:S BENCH:*'\
.format(self._target.adb_name))
self._log.info("%s", logcat_cmd)
command = "nohup am instrument -e iterations {} -e class {}{} -w {}".format(
iterations, self.test_package, activity, self.test_package)
self._target.background(command)
logcat = Popen(logcat_cmd, shell=True, stdout=PIPE)
while True:
# read next logcat line (up to max 1024 chars)
message = logcat.stdout.readline(1024)
# Benchmark start trigger
match = SYSTEMUI_BENCHMARK_START_RE.search(message)
if match:
self.tracingStart()
self._log.debug("Benchmark started!")
match = SYSTEMUI_BENCHMARK_FINISH_RE.search(message)
if match:
self.tracingStop()
self._log.debug("Benchmark finished!")
break
sleep(5)
# Go back to home screen
System.home(self._target)
# Switch back to original settings
Screen.set_orientation(self._target, auto=True)
System.set_airplane_mode(self._target, on=False)
Screen.set_brightness(self._target, auto=True)
def run(self, out_dir, test_name, collect=''):
"""
Run single Vellamo workload. Returns a collection of results.
:param out_dir: Path to experiment directory where to store results.
:type out_dir: str
:param test_name: Name of the test to run
:type test_name: str
:param collect: Specifies what to collect. Possible values:
- 'energy'
- 'systrace'
- 'ftrace'
- any combination of the above
:type collect: list(str)
"""
self.out_dir = out_dir
self.collect = collect
# Check if the density of the target device screen is different from
# the one used to get the values below
density = Screen.get_screen_density(self._target)
if DEFAULT_DENSITY not in density:
msg = 'Screen density of target device differs from {}.\n'\
'Please set it to {}'
raise RuntimeError(msg.format(DEFAULT_DENSITY, DEFAULT_DENSITY))
if test_name.upper() not in VELLAMO_TESTS:
raise ValueError('Vellamo workload [%s] not supported', test_name)
# Set parameter depending on test
self._log.debug('Start Vellamo Benchmark [%s]', test_name)
test_x, test_y = (0, 0)
sleep_time = 0
if test_name.upper() == 'BROWSER':
test_x, test_y = (91, 33)
sleep_time = 3.5
elif test_name.upper() == 'METAL':
test_x, test_y = (91, 59)
sleep_time = 1
elif test_name.upper() == 'MULTI':
test_x, test_y = (91, 71)
sleep_time = 3.5
# Unlock device screen (assume no password required)
Screen.unlock(self._target)
System.force_stop(self._target, self.package, clear=True)
# Set min brightness
Screen.set_brightness(self._target, auto=False, percent=0)
# Clear logcat
os.system(self._adb('logcat -c'));
# Regexps for benchmark synchronization
start_logline = r'ActivityManager: Start.*'\
':com.quicinc.vellamo:benchmarking'
VELLAMO_BENCHMARK_START_RE = re.compile(start_logline)
self._log.debug("START string [%s]", start_logline)
# End of benchmark is marked by displaying results
end_logline = r'ActivityManager: START.*'\
'act=com.quicinc.vellamo.*_RESULTS'
VELLAMO_BENCHMARK_END_RE = re.compile(end_logline)
self._log.debug("END string [%s]", end_logline)
# Parse logcat output lines
logcat_cmd = self._adb(
'logcat ActivityManager:* System.out:I *:S BENCH:*'\
.format(self._target.adb_name))
self._log.info("%s", logcat_cmd)
# Start the activity
System.start_activity(self._target, self.package, self.activity)
logcat = Popen(logcat_cmd, shell=True, stdout=PIPE)
sleep(2)
# Accept EULA
System.tap(self._target, 80, 86)
sleep(1)
# Click Let's Roll
System.tap(self._target, 50, 67)
sleep(1)
# Skip Arrow
System.tap(self._target, 46, 78)
# Run Workload
System.tap(self._target, test_x, test_y)
# Skip instructions
System.hswipe(self._target, 10, 80, duration=100, swipe_right=False)
System.hswipe(self._target, 10, 80, duration=100, swipe_right=False)
System.hswipe(self._target, 10, 80, duration=100, swipe_right=False)
self._log.info("Vellamo - {} started!".format(test_name.upper()))
while True:
# read next logcat line (up to max 1024 chars)
message = logcat.stdout.readline(1024)
# Benchmark start trigger
if VELLAMO_BENCHMARK_START_RE.search(message):
# Start tracing
self.tracingStart()
self._log.debug("Benchmark started!")
elif VELLAMO_BENCHMARK_END_RE.search(message):
# Stop tracing
self.tracingStop()
break
else:
continue
# Gather scores file from the device
db_file = os.path.join(out_dir, VELLAMO_SCORE_NAME)
self._target.pull('{}/{}'.format(VELLAMO_DB_PATH, VELLAMO_SCORE_NAME),
db_file)
System.force_stop(self._target, self.package, clear=True)
# Go back to home screen
System.home(self._target)
# Set brightness back to auto
Screen.set_brightness(self._target, auto=True)
def run(self, out_dir, test_name, collect=''):
"""
Run single Geekbench workload.
:param out_dir: Path on host to experiment directory where
to store results.
:type out_dir: str
:param test_name: Name of the test to run
:type test_name: str
:param collect: Specifies what to collect. Possible values:
- 'energy'
- 'systrace'
- 'ftrace'
- any combination of the above in a space-separated string.
:type collect: list(str)
"""
# Initialize energy meter results
self.out_dir = out_dir
self.collect = collect
# Clear the stored data for the application, so we always start with
# an EULA to clear
System.force_stop(self._target, self.package, clear=True)
# Clear logcat from any previous runs
# do this on the target as then we don't need to build a string
self._target.clear_logcat()
# Unlock device screen (assume no password required)
System.menu(self._target)
# Press Back button to be sure we run the benchmark from the start
System.back(self._target)
# Force screen in PORTRAIT mode
Screen.set_orientation(self._target, portrait=True)
# Set min brightness
Screen.set_brightness(self._target, auto=False, percent=0)
# Start app on the target device
System.start_activity(self._target, self.package, self.activity)
# Allow the activity to start
sleep(2)
# Parse logcat output lines to find beginning and end
logcat_cmd = self._adb(
'logcat ActivityManager:* System.out:I *:S GEEKBENCH_RESULT:*'\
.format(self._target.adb_name))
self._log.info("%s", logcat_cmd)
logcat = Popen(logcat_cmd, shell=True, stdout=PIPE)
# Click to accept the EULA
System.tap(self._target, 73, 55)
sleep(1)
# The main window opened will have the CPU benchmark
# Swipe to get the COMPUTE one
if test_name.upper() == 'COMPUTE':
System.hswipe(self._target,
10,
80,
duration=100,
swipe_right=False)
# Press the 'RUN <test_name> BENCHMARK' button
System.tap(self._target, 73, 72)
while True:
# read next logcat line (up to max 1024 chars)
message = logcat.stdout.readline(1024)
# Benchmark start trigger
match = GEEKBENCH_BENCHMARK_START_RE.search(message)
if match:
# Start tracing
self.tracingStart()
self._log.debug("Benchmark started!")
# Benchmark end trigger
match = GEEKBENCH_BENCHMARK_END_RE.search(message)
if match:
# Stop tracing
self.tracingStop()
remote_result_file = match.group('results_file')
self._log.debug("Benchmark finished! Results are in {}".format(
remote_result_file))
break
# Get Geekbench Results file
target_result_file = self._target.path.basename(remote_result_file)
result_file = os.path.join(self.out_dir, target_result_file)
self._log.debug("result_file={}".format(result_file))
self._target.pull(remote_result_file, result_file)
# Close the app
System.force_stop(self._target, self.package, clear=False)
# Go back to home screen
System.home(self._target)
# Switch back to screen auto rotation
Screen.set_orientation(self._target, auto=True)
# Set brightness back to auto
Screen.set_brightness(self._target, auto=True)
def run(self, out_dir, test_name, collect=''):
"""
Run single Vellamo workload. Returns a collection of results.
:param out_dir: Path to experiment directory where to store results.
:type out_dir: str
:param test_name: Name of the test to run
:type test_name: str
:param collect: Specifies what to collect. Possible values:
- 'energy'
- 'systrace'
- 'ftrace'
- any combination of the above
:type collect: list(str)
"""
self.out_dir = out_dir
self.collect = collect
# Check if the density of the target device screen is different from
# the one used to get the values below
density = Screen.get_screen_density(self._target)
if DEFAULT_DENSITY not in density:
msg = 'Screen density of target device differs from {}.\n'\
'Please set it to {}'
raise RuntimeError(msg.format(DEFAULT_DENSITY, DEFAULT_DENSITY))
if test_name.upper() not in VELLAMO_TESTS:
raise ValueError('Vellamo workload [%s] not supported', test_name)
# Set parameter depending on test
self._log.debug('Start Vellamo Benchmark [%s]', test_name)
test_x, test_y = (0, 0)
sleep_time = 0
if test_name.upper() == 'BROWSER':
test_x, test_y = (91, 33)
sleep_time = 3.5
elif test_name.upper() == 'METAL':
test_x, test_y = (91, 59)
sleep_time = 1
elif test_name.upper() == 'MULTI':
test_x, test_y = (91, 71)
sleep_time = 3.5
# Unlock device screen (assume no password required)
Screen.unlock(self._target)
System.force_stop(self._target, self.package, clear=True)
# Set min brightness
Screen.set_brightness(self._target, auto=False, percent=0)
# Clear logcat
os.system(self._adb('logcat -c'))
# Regexps for benchmark synchronization
start_logline = r'ActivityManager: Start.*'\
':com.quicinc.vellamo:benchmarking'
VELLAMO_BENCHMARK_START_RE = re.compile(start_logline)
self._log.debug("START string [%s]", start_logline)
# End of benchmark is marked by displaying results
end_logline = r'ActivityManager: START.*'\
'act=com.quicinc.vellamo.*_RESULTS'
VELLAMO_BENCHMARK_END_RE = re.compile(end_logline)
self._log.debug("END string [%s]", end_logline)
# Parse logcat output lines
logcat_cmd = self._adb(
'logcat ActivityManager:* System.out:I *:S BENCH:*'\
.format(self._target.adb_name))
self._log.info("%s", logcat_cmd)
# Start the activity
System.start_activity(self._target, self.package, self.activity)
logcat = Popen(logcat_cmd, shell=True, stdout=PIPE)
sleep(2)
# Accept EULA
System.tap(self._target, 80, 86)
sleep(1)
# Click Let's Roll
System.tap(self._target, 50, 67)
sleep(1)
# Skip Arrow
System.tap(self._target, 46, 78)
# Run Workload
System.tap(self._target, test_x, test_y)
# Skip instructions
System.hswipe(self._target, 10, 80, duration=100, swipe_right=False)
System.hswipe(self._target, 10, 80, duration=100, swipe_right=False)
System.hswipe(self._target, 10, 80, duration=100, swipe_right=False)
self._log.info("Vellamo - {} started!".format(test_name.upper()))
while True:
# read next logcat line (up to max 1024 chars)
message = logcat.stdout.readline(1024)
# Benchmark start trigger
if VELLAMO_BENCHMARK_START_RE.search(message):
# Start tracing
self.tracingStart()
self._log.debug("Benchmark started!")
elif VELLAMO_BENCHMARK_END_RE.search(message):
# Stop tracing
self.tracingStop()
break
else:
continue
# Gather scores file from the device
db_file = os.path.join(out_dir, VELLAMO_SCORE_NAME)
self._target.pull('{}/{}'.format(VELLAMO_DB_PATH, VELLAMO_SCORE_NAME),
db_file)
System.force_stop(self._target, self.package, clear=True)
# Go back to home screen
System.home(self._target)
# Set brightness back to auto
Screen.set_brightness(self._target, auto=True)
def run(self, out_dir, video_url, video_duration_s, collect=''):
"""
Run single YouTube workload.
:param out_dir: Path to experiment directory where to store results.
:type out_dir: str
:param video_url: Video URL to be played
:type video_url: str
:param video_duration_s: Play video for this required number of seconds
:type video_duration_s: int
:param collect: Specifies what to collect. Possible values:
- 'energy'
- 'systrace'
- 'ftrace'
- any combination of the above
:type collect: list(str)
"""
# Keep track of mandatory parameters
self.out_dir = out_dir
self.collect = collect
# Unlock device screen (assume no password required)
Screen.unlock(self._target)
# Stop youtube if already running
System.force_stop(self._target, self.package)
# Use the monkey tool to start YouTube without playing any video.
# This allows to subsequently set the screen orientation to LANDSCAPE
# and to reset the frame statistics.
System.monkey(self._target, self.package)
# Force screen in LANDSCAPE mode
Screen.set_orientation(self._target, portrait=False)
# Set min brightness
Screen.set_brightness(self._target, auto=False, percent=0)
System.gfxinfo_reset(self._target, self.package)
sleep(1)
# Start YouTube video on the target device
System.start_action(self._target, self.action, video_url)
# Allow the activity to start
sleep(1)
# Wait until the end of the video
self.tracingStart()
self._log.info('Play video for %d [s]', video_duration_s)
sleep(video_duration_s)
self.tracingStop()
# Get frame stats
self.db_file = os.path.join(out_dir, "framestats.txt")
System.gfxinfo_get(self._target, self.package, self.db_file)
# Close the app without clearing the local data to
# avoid the dialog to select the account at next start
System.force_stop(self._target, self.package, clear=False)
# Go back to home screen
System.home(self._target)
# Switch back to screen auto rotation
Screen.set_orientation(self._target, auto=True)
# Set brightness back to auto
Screen.set_brightness(self._target, auto=True)
def run(self, out_dir, test_name, duration_s, collect=''):
"""
Run single UiBench workload.
:param out_dir: Path to experiment directory where to store results.
:type out_dir: str
:param test_name: Name of the test to run
:type test_name: str
:param duration_s: Run benchmak for this required number of seconds
:type duration_s: int
:param collect: Specifies what to collect. Possible values:
- 'energy'
- 'systrace'
- 'ftrace'
- any combination of the above
:type collect: list(str)
"""
activity = '.' + test_name
# Keep track of mandatory parameters
self.out_dir = out_dir
self.collect = collect
# Unlock device screen (assume no password required)
Screen.unlock(self._target)
# Close and clear application
System.force_stop(self._target, self.package, clear=True)
# Set airplane mode
System.set_airplane_mode(self._target, on=True)
# Set min brightness
Screen.set_brightness(self._target, auto=False, percent=0)
# Start the main view of the app which must be running
# to reset the frame statistics.
System.monkey(self._target, self.package)
# Force screen in PORTRAIT mode
Screen.set_orientation(self._target, portrait=True)
# Reset frame statistics
System.gfxinfo_reset(self._target, self.package)
sleep(1)
# Clear logcat
os.system(self._adb('logcat -c'));
# Regexps for benchmark synchronization
start_logline = r'ActivityManager: START.*'\
'cmp=com.android.test.uibench/{}'.format(activity)
UIBENCH_BENCHMARK_START_RE = re.compile(start_logline)
self._log.debug("START string [%s]", start_logline)
# Parse logcat output lines
logcat_cmd = self._adb(
'logcat ActivityManager:* System.out:I *:S BENCH:*'\
.format(self._target.adb_name))
self._log.info("%s", logcat_cmd)
# Start the activity
System.start_activity(self._target, self.package, activity)
logcat = Popen(logcat_cmd, shell=True, stdout=PIPE)
while True:
# read next logcat line (up to max 1024 chars)
message = logcat.stdout.readline(1024)
# Benchmark start trigger
match = UIBENCH_BENCHMARK_START_RE.search(message)
if match:
self.tracingStart()
self._log.debug("Benchmark started!")
break
# Run the workload for the required time
self._log.info('Benchmark [%s] started, waiting %d [s]',
activity, duration_s)
sleep(duration_s)
self._log.debug("Benchmark done!")
self.tracingStop()
# Get frame stats
self.db_file = os.path.join(out_dir, "framestats.txt")
System.gfxinfo_get(self._target, self.package, self.db_file)
# Close and clear application
System.force_stop(self._target, self.package, clear=True)
# Go back to home screen
System.home(self._target)
# Switch back to original settings
Screen.set_orientation(self._target, auto=True)
System.set_airplane_mode(self._target, on=False)
Screen.set_brightness(self._target, auto=True)
def run(self, out_dir, test_name, duration_s, collect=''):
"""
Run single UiBench workload.
:param out_dir: Path to experiment directory where to store results.
:type out_dir: str
:param test_name: Name of the test to run
:type test_name: str
:param duration_s: Run benchmak for this required number of seconds
:type duration_s: int
:param collect: Specifies what to collect. Possible values:
- 'energy'
- 'systrace'
- 'ftrace'
- any combination of the above
:type collect: list(str)
"""
activity = '.' + test_name
# Keep track of mandatory parameters
self.out_dir = out_dir
self.collect = collect
# Unlock device screen (assume no password required)
Screen.unlock(self._target)
# Close and clear application
System.force_stop(self._target, self.package, clear=True)
# Set airplane mode
System.set_airplane_mode(self._target, on=True)
# Set min brightness
Screen.set_brightness(self._target, auto=False, percent=0)
# Start the main view of the app which must be running
# to reset the frame statistics.
System.monkey(self._target, self.package)
# Force screen in PORTRAIT mode
Screen.set_orientation(self._target, portrait=True)
# Reset frame statistics
System.gfxinfo_reset(self._target, self.package)
sleep(1)
# Clear logcat
os.system(self._adb('logcat -c'))
# Regexps for benchmark synchronization
start_logline = r'TestRunner: started'
UIBENCH_BENCHMARK_START_RE = re.compile(start_logline)
self._log.debug("START string [%s]", start_logline)
# Parse logcat output lines
logcat_cmd = self._adb(
'logcat TestRunner:* System.out:I *:S BENCH:*'\
.format(self._target.adb_name))
self._log.info("%s", logcat_cmd)
# Run benchmark with a lot of iterations to avoid finishing before duration_s elapses
command = "nohup am instrument -e iterations 1000000 -e class {}{} -w {}".format(
self.test_package, activity, self.test_package)
self._target.background(command)
logcat = Popen(logcat_cmd, shell=True, stdout=PIPE)
while True:
# read next logcat line (up to max 1024 chars)
message = logcat.stdout.readline(1024)
# Benchmark start trigger
match = UIBENCH_BENCHMARK_START_RE.search(message)
if match:
self.tracingStart()
self._log.debug("Benchmark started!")
break
# Run the workload for the required time
self._log.info('Benchmark [%s] started, waiting %d [s]', activity,
duration_s)
sleep(duration_s)
self._log.debug("Benchmark done!")
self.tracingStop()
# Get frame stats
self.db_file = os.path.join(out_dir, "framestats.txt")
System.gfxinfo_get(self._target, self.package, self.db_file)
# Close and clear application
System.force_stop(self._target, self.package, clear=True)
# Go back to home screen
System.home(self._target)
# Switch back to original settings
Screen.set_orientation(self._target, auto=True)
System.set_airplane_mode(self._target, on=False)
Screen.set_brightness(self._target, auto=True)
def experiment():
# Check if the dhyrstone binary is on the device
dhrystone = os.path.join(target.executables_directory, 'dhrystone')
if not target.file_exists(dhrystone):
raise RuntimeError('dhrystone could not be located here: {}'.format(
dhrystone))
# Create results directory
outdir=te.res_dir + '_' + args.out_prefix
if not args.cont:
try:
shutil.rmtree(outdir)
except:
print "couldn't remove " + outdir
pass
if not os.path.exists(outdir):
os.makedirs(outdir)
# Get clusters and cpus
cpus = [cpu for cluster in CLUSTERS for cpu in cluster]
# Prevent screen from dozing
Screen.set_doze_always_on(target, on=False)
# Turn on airplane mode
System.set_airplane_mode(target, on=True)
# Turn off screen
Screen.set_screen(target, on=False)
# Stop thermal engine and perfd
target.execute("stop thermal-engine")
target.execute("stop perfd")
# Take a wakelock
System.wakelock(target, take=True)
# Store governors so they can be restored later
governors = [ target.cpufreq.get_governor(cpu) for cpu in cpus]
# Set the governer to userspace so the cpu frequencies can be set
target.hotplug.online_all()
target.cpufreq.set_all_governors('userspace')
# Freeze all non critical tasks
target.cgroups.freeze(exclude=CRITICAL_TASKS)
# Remove all userspace tasks from the cluster
sandbox_cg, isolated_cg = target.cgroups.isolate([])
# Run measurements on single cluster
single_cluster(cpus, sandbox_cg, isolated_cg, dhrystone, outdir)
# Run measurements on multiple clusters
multiple_clusters(cpus, sandbox_cg, isolated_cg, dhrystone, outdir)
# Restore all governors
for i, governor in enumerate(governors):
target.cpufreq.set_governor(cpus[i], governor)
# Restore non critical tasks
target.cgroups.freeze(thaw=True)
# Release wakelock
System.wakelock(target, take=False)
# Stop thermal engine and perfd
target.execute("start thermal-engine")
target.execute("start perfd")
# Dump platform
te.platform_dump(outdir)
te._log.info('RESULTS are in out directory: {}'.format(outdir))
def run(self, out_dir, duration_s, collect=''):
"""
Run single idle/suspend workload.
:param out_dir: Path to experiment directory where to store results.
:type out_dir: str
:param duration_s: Duration of test
:type duration_s: int
:type force: str
:param collect: Specifies what to collect. Possible values:
- 'energy'
- 'systrace'
- 'ftrace'
- any combination of the above
:type collect: list(str)
"""
# Idle/resume should always disconnect USB so rely on
# the Energy meter to disconnect USB (We only support USB disconnect
# mode for energy measurement, like with Monsoon
if 'energy' not in collect:
collect = collect + ' energy'
# Keep track of mandatory parameters
self.out_dir = out_dir
self.collect = collect
# Set min brightness
Screen.set_brightness(self._target, auto=False, percent=0)
# Set timeout to min value
Screen.set_timeout(self._target, seconds=0)
# Prevent screen from dozing
Screen.set_doze_always_on(self._target, on=False)
# Turn on airplane mode
System.set_airplane_mode(self._target, on=True)
# Unlock device screen (assume no password required)
Screen.unlock(self._target)
# Force the device to suspend
System.force_suspend_start(self._target)
# Prevent the device from fully suspending by holding a partial wakelock
System.wakelock(self._target, take=True)
sleep(1)
Screen.set_screen(self._target, on=False)
sleep(1)
self.tracingStart(screen_always_on=False)
self._log.info('Waiting for duration {}'.format(duration_s))
sleep(duration_s)
self.tracingStop(screen_always_on=False)
# Resume normal function
System.force_suspend_stop(self._target)
# Release wakelock
System.wakelock(self._target, take=False)
Screen.set_defaults(self._target)
System.set_airplane_mode(self._target, on=False)
def run(self, exp_dir, test_name, iterations, collect=""):
# Setup test id
try:
test_id = _jankbench[test_name]
except KeyError:
raise ValueError("Jankbench test [%s] not supported", test_name)
# Initialize energy meter results
nrg_report = None
self.target.execute("input keyevent 82")
# Press Back button to be sure we run the video from the start
self.target.execute("input keyevent KEYCODE_BACK")
# Close and clear application
self.target.execute("am force-stop com.android.benchmark")
self.target.execute("pm clear com.android.benchmark")
# Set airplane mode
System.set_airplane_mode(self.target, on=True)
# Force screen in PORTRAIT mode
Screen.set_orientation(self.target, portrait=True)
# Clear logcat
os.system(self._adb("logcat -c"))
self.logger.debug("Start Jank Benchmark [%d:%s]", test_id, test_name)
test_cmd = (
'am start -n "com.android.benchmark/.app.RunLocalBenchmarksActivity" '
'--eia "com.android.benchmark.EXTRA_ENABLED_BENCHMARK_IDS" {0} '
'--ei "com.android.benchmark.EXTRA_RUN_COUNT" {1}'.format(test_id, iterations)
)
self.logger.info(test_cmd)
self.target.execute(test_cmd)
# Parse logcat output lines
logcat_cmd = self._adb("logcat ActivityManager:* System.out:I *:S BENCH:*".format(self.target.adb_name))
self.logger.info("%s", logcat_cmd)
self.logger.debug("Iterations:")
logcat = Popen(logcat_cmd, shell=True, stdout=PIPE)
while True:
# read next logcat line (up to max 1024 chars)
message = logcat.stdout.readline(1024)
# Benchmark start trigger
match = JANKBENCH_BENCHMARK_START_RE.search(message)
if match:
if "energy" in collect and self.te.emeter:
self.te.emeter.reset()
self.logger.debug("Benchmark started!")
# Benchmark completed trigger
match = JANKBENCH_BENCHMARK_DONE_RE.search(message)
if match:
if "energy" in collect and self.te.emeter:
nrg_report = self.te.emeter.report(exp_dir)
self.logger.info("Estimated energy: %7.3f", float(nrg_report.channels["BAT"]))
self.logger.debug("Benchmark done!")
break
# Iteration completd
match = JANKBENCH_ITERATION_COUNT_RE.search(message)
if match:
self.logger.debug("Iteration %2d:", int(match.group("iteration")) + 1)
# Iteration metrics
match = JANKBENCH_ITERATION_METRICS_RE.search(message)
if match:
self.logger.info(
" Mean: %7.3f JankP: %7.3f StdDev: %7.3f Count Bad: %4d Count Jank: %4d",
float(match.group("mean")),
float(match.group("junk_p")),
float(match.group("std_dev")),
int(match.group("count_bad")),
int(match.group("count_junk")),
)
# get results
db_file = os.path.join(exp_dir, JANKBENCH_DB_NAME)
self.target.pull(JANKBENCH_DB_PATH + JANKBENCH_DB_NAME, db_file)
# Close and clear application
self.target.execute("am force-stop com.android.benchmark")
self.target.execute("pm clear com.android.benchmark")
# Go back to home screen
self.target.execute("input keyevent KEYCODE_HOME")
# Switch back to screen auto rotation
Screen.set_orientation(self.target, auto=True)
return db_file, nrg_report
def run(self, out_dir, collect, media_file, from_device=False, play_duration_s=None):
"""
Run Exoplayer workload
:param out_dir: Path to experiment directory on the host
where to store results.
:type out_dir: str
:param collect: Specifies what to collect. Possible values:
- 'energy'
- 'systrace'
- 'ftrace'
- any combination of the above as a single space-separated string.
:type collect: list(str)
:param media_file: Filepath of the media to play
Path on device if 'from_device' is True
Path on host if 'from_device' is False (default)
:type media_file: str
:param from_device: Whether file to play is already on the device
:type from_device: bool
:param play_duration_s: If set, maximum duration (seconds) of the media playback
If not set, media will play to completion
:type play_duration_s: int
"""
# Keep track of mandatory parameters
self.out_dir = out_dir
self.collect = collect
self.media_file = media_file
self.from_device = from_device
self.play_duration_s = play_duration_s
# Check media file exists
if from_device and not self._target.file_exists(self.media_file):
raise RuntimeError('Cannot find "{}" on target'.format(self.media_file))
elif not from_device and not os.path.isfile(self.media_file):
raise RuntimeError('Cannot find "{}" on host'.format(self.media_file))
# Unlock device screen (assume no password required)
Screen.unlock(self._target)
# Close and clear application
System.force_stop(self._target, self.package, clear=True)
# Enable airplane mode
System.set_airplane_mode(self._target, on=True)
# Set min brightness
Screen.set_brightness(self._target, auto=False, percent=0)
# Force screen in PORTRAIT mode
Screen.set_orientation(self._target, portrait=True)
# Launch Exoplayer benchmark
self._play()
# Go back to home screen
System.home(self._target)
# Set orientation back to auto
Screen.set_orientation(self._target, auto=True)
# Set brightness back to auto
Screen.set_brightness(self._target, auto=True)
# Turn off airplane mode
System.set_airplane_mode(self._target, on=False)
# Close and clear application
System.force_stop(self._target, self.package, clear=True)
def run(self, out_dir, test_name, iterations, collect=''):
"""
Run single system app jank test workload.
Performance statistics are stored in self.results, and can be retrieved
after the fact by calling SystemUi.get_results()
:param out_dir: Path to experiment directory where to store results.
:type out_dir: str
:param test_name: Name of the test to run
:type test_name: str
:param iterations: Run benchmark for this required number of iterations
:type iterations: int
:param collect: Specifies what to collect. Possible values:
- 'systrace'
- 'ftrace'
- 'gfxinfo'
- 'surfaceflinger'
- any combination of the above
:type collect: list(str)
"""
if "energy" in collect:
raise ValueError('System app workload does not support energy data collection')
activity = "." + test_name
package = self._get_test_package(test_name)
# Keep track of mandatory parameters
self.out_dir = out_dir
self.collect = collect
# Filter out test overhead
filter_prop = System.get_boolean_property(self._target, "debug.hwui.filter_test_overhead")
if not filter_prop:
System.set_property(self._target, "debug.hwui.filter_test_overhead", "true", restart=True)
# Unlock device screen (assume no password required)
Screen.unlock(self._target)
# Close and clear application
System.force_stop(self._target, package, clear=True)
# Set min brightness
Screen.set_brightness(self._target, auto=False, percent=0)
# Force screen in PORTRAIT mode
Screen.set_orientation(self._target, portrait=True)
# Delete old test results
self._target.remove("/sdcard/results.log")
# Clear logcat
self._target.execute("logcat -c")
# Regexps for benchmark synchronization
start_logline = r"TestRunner: started"
SYSAPP_BENCHMARK_START_RE = re.compile(start_logline)
self._log.debug("START string [%s]", start_logline)
finish_logline = r"TestRunner: finished"
SYSAPP_BENCHMARK_FINISH_RE = re.compile(finish_logline)
self._log.debug("FINISH string [%s]", finish_logline)
# Parse logcat output lines
logcat_cmd = self._adb("logcat TestRunner:* System.out:I *:S BENCH:*")
self._log.info("%s", logcat_cmd)
command = "am instrument -e iterations {} -e class {}{} -w {}".format(
iterations, self.test_package, activity, self.test_package)
logcat = Popen(logcat_cmd, shell=True, stdout=PIPE)
test_proc = self._target.background(command)
while True:
# read next logcat line (up to max 1024 chars)
message = logcat.stdout.readline(1024)
# Benchmark start
match = SYSAPP_BENCHMARK_START_RE.search(message)
if match:
self.tracingStart()
self._log.debug("Benchmark started!")
# Benchmark finish
match = SYSAPP_BENCHMARK_FINISH_RE.search(message)
if match:
self.tracingStop()
self._log.debug("Benchmark finished!")
test_proc.wait()
break
sleep(5)
self._target.pull("/sdcard/results.log", os.path.join(out_dir, "results.log"))
self.db_file = os.path.join(out_dir, "results.log")
self.results = self.get_results(out_dir)
# Close and clear application
System.force_stop(self._target, package, clear=True)
# Go back to home screen
System.home(self._target)
# Switch back to original settings
Screen.set_orientation(self._target, auto=True)
Screen.set_brightness(self._target, auto=True)
def run(self, out_dir, video_url, video_duration_s, collect=''):
"""
Run single YouTube workload.
:param out_dir: Path to experiment directory where to store results.
:type out_dir: str
:param video_url: Video URL to be played
:type video_url: str
:param video_duration_s: Play video for this required number of seconds
:type video_duration_s: int
:param collect: Specifies what to collect. Possible values:
- 'energy'
- 'systrace'
- 'ftrace'
- any combination of the above
:type collect: list(str)
"""
# Keep track of mandatory parameters
self.out_dir = out_dir
self.collect = collect
# Unlock device screen (assume no password required)
Screen.unlock(self._target)
# Stop youtube if already running
System.force_stop(self._target, self.package)
# Use the monkey tool to start YouTube without playing any video.
# This allows to subsequently set the screen orientation to LANDSCAPE
# and to reset the frame statistics.
System.monkey(self._target, self.package)
# Force screen in LANDSCAPE mode
Screen.set_orientation(self._target, portrait=False)
# Set min brightness
Screen.set_brightness(self._target, auto=False, percent=0)
System.gfxinfo_reset(self._target, self.package)
sleep(1)
# Start YouTube video on the target device
System.start_action(self._target, self.action, video_url)
# Allow the activity to start
sleep(1)
# Wait until the end of the video
self.tracingStart()
self._log.info('Play video for %d [s]', video_duration_s)
sleep(video_duration_s)
self.tracingStop()
# Get frame stats
self.db_file = os.path.join(out_dir, "framestats.txt")
System.gfxinfo_get(self._target, self.package, self.db_file)
# Close the app without clearing the local data to
# avoid the dialog to select the account at next start
System.force_stop(self._target, self.package, clear=False)
# Go back to home screen
System.home(self._target)
# Switch back to screen auto rotation
Screen.set_orientation(self._target, auto=True)
# Set brightness back to auto
Screen.set_brightness(self._target, auto=True)
def run(self, out_dir, package, permissions, duration_s, collect=''):
"""
Run single AppStartup workload.
:param out_dir: Path to experiment directory where to store results.
:type out_dir: str
:param package: Name of the apk package
:type package: str
:param permissions: List of permissions the app requires
:type permissions: list of str
:param duration_s: Run benchmak for this required number of seconds
:type duration_s: int
:param collect: Specifies what to collect. Possible values:
- 'energy'
- 'time_in_state'
- 'systrace'
- 'ftrace'
- any combination of the above
:type collect: list(str)
"""
# Keep track of mandatory parameters
self.out_dir = out_dir
self.collect = collect
# Check if the package exists on the device
if not System.contains_package(self._target, package):
raise RuntimeError('System does not contain the requried package')
# Unlock device screen (assume no password required)
Screen.unlock(self._target)
# Close and clear application
System.force_stop(self._target, package, clear=True)
sleep(3)
# Grant permissions
for permission in permissions:
System.grant_permission(self._target, package, permission)
sleep(3)
# Set min brightness
Screen.set_brightness(self._target, auto=False, percent=100)
# Set screen timeout
Screen.set_timeout(self._target, seconds=duration_s + 60)
# Force screen in PORTRAIT mode
Screen.set_orientation(self._target, portrait=True)
# Start the main view of the app
System.monkey(self._target, package)
# Start tracing
self.tracingStart()
# Sleep for duration
sleep(duration_s)
# Stop tracing
self.tracingStop()
# Reset permissions
System.reset_permissions(self._target, package)
# Close and clear application
System.force_stop(self._target, package, clear=True)
# Go back to home screen
System.home(self._target)
# Switch back to original settings
Screen.set_defaults(self._target)
def run(self, out_dir, duration_s, brightness, filepath, collect=''):
"""
Run single display image workload.
:param out_dir: Path to experiment directory where to store results.
:type out_dir: str
:param duration_s: Duration of test
:type duration_s: int
:param brightness: Brightness of the screen 0-100
:type brightness: int
:param out_dir: Path to the image to display
:type out_dir: str
:param collect: Specifies what to collect. Possible values:
- 'energy'
- 'time_in_state'
- 'systrace'
- 'ftrace'
- any combination of the above, except energy and display-energy
cannot be collected simultaneously.
:type collect: list(str)
"""
# Keep track of mandatory parameters
self.out_dir = out_dir
self.collect = collect
# Set brightness
Screen.set_brightness(self._target, auto=False, percent=brightness)
# Set timeout to be sufficiently longer than the duration of the test
Screen.set_timeout(self._target, seconds=(duration_s+60))
# Turn on airplane mode
System.set_airplane_mode(self._target, on=True)
# Unlock device screen (assume no password required)
Screen.unlock(self._target)
# Push the image to the device
device_filepath = '/data/local/tmp/image'
self._target.push(filepath, device_filepath)
sleep(1)
# Put image on the screen
System.start_action(self._target, self.action,
'-t image/* -d file://{}'.format(device_filepath))
sleep(1)
# Dimiss the navigation bar by tapping the center of the screen
System.tap(self._target, 50, 50)
self.tracingStart(screen_always_on=False)
self._log.info('Waiting for duration {}'.format(duration_s))
sleep(duration_s)
self.tracingStop(screen_always_on=False)
# Close app and dismiss image
System.force_stop(self._target, self.package, clear=True)
# Remove image
self._target.execute('rm /data/local/tmp/image')
Screen.set_defaults(self._target)
System.set_airplane_mode(self._target, on=False)
Screen.set_screen(self._target, on=False)
def run(self, out_dir, test_name, duration_s, collect='', actions='default'):
"""
Run single UiBench workload.
:param out_dir: Path to experiment directory where to store results.
:type out_dir: str
:param test_name: Name of the test to run
:type test_name: str
:param duration_s: Run benchmak for this required number of seconds
:type duration_s: int
:param collect: Specifies what to collect. Possible values:
- 'energy'
- 'systrace'
- 'ftrace'
- any combination of the above
:type collect: list(str)
:param actions: Specifies what actions to perform. Possible values:
- None : Perform no action
- 'default' : Use the predefined default actions from the `test_actions` dict
- 'vswipe' : Perform a vertical swipe
- 'tap' : Perform a centre tap
- A list with any combination of vswipe and tap, in execution order
:type actions: list(str)
"""
activity = '.' + test_name
# If default, get the default actions from test_actions
# If this is an undefined test, no action will be the default
if actions == 'default':
actions = None
if test_name in self.test_actions:
actions = self.test_actions[test_name]
# Format actions as a list if it is just a single string item,
# or an empty list if it is None
actions = [actions] if isinstance(actions, basestring) else actions if actions else []
# Keep track of mandatory parameters
self.out_dir = out_dir
self.collect = collect
# Unlock device screen (assume no password required)
Screen.unlock(self._target)
# Close and clear application
System.force_stop(self._target, self.package, clear=True)
# Set airplane mode
System.set_airplane_mode(self._target, on=True)
# Set min brightness
Screen.set_brightness(self._target, auto=False, percent=0)
# Start the main view of the app which must be running
# to reset the frame statistics.
System.monkey(self._target, self.package)
# Force screen in PORTRAIT mode
Screen.set_orientation(self._target, portrait=True)
# Reset frame statistics
System.gfxinfo_reset(self._target, self.package)
sleep(1)
# Clear logcat
os.system(self._adb('logcat -c'));
# Regexps for benchmark synchronization
start_logline = r'ActivityManager: START.*'\
'cmp=com.android.test.uibench/{}'.format(activity)
UIBENCH_BENCHMARK_START_RE = re.compile(start_logline)
self._log.debug("START string [%s]", start_logline)
# Parse logcat output lines
logcat_cmd = self._adb(
'logcat ActivityManager:* System.out:I *:S BENCH:*'\
.format(self._target.adb_name))
self._log.info("%s", logcat_cmd)
# Prepare user-input commands on the device
if actions:
script = TargetScript(self._te, "uibench.sh")
for action in actions:
self._perform_action(script, action)
script.push()
# Start the activity
System.start_activity(self._target, self.package, activity)
logcat = Popen(logcat_cmd, shell=True, stdout=PIPE)
while True:
# read next logcat line (up to max 1024 chars)
message = logcat.stdout.readline(1024)
# Benchmark start trigger
match = UIBENCH_BENCHMARK_START_RE.search(message)
if match:
self.tracingStart()
self._log.debug("Benchmark started!")
break
# Run the workload for the required time
self._log.info('Benchmark [%s] started, waiting %d [s]',
activity, duration_s)
start = time()
if actions:
script.run()
while (time() - start) < duration_s:
sleep(1)
self._log.debug("Benchmark done!")
self.tracingStop()
# Get frame stats
self.db_file = os.path.join(out_dir, "framestats.txt")
System.gfxinfo_get(self._target, self.package, self.db_file)
# Close and clear application
System.force_stop(self._target, self.package, clear=True)
# Go back to home screen
System.home(self._target)
# Switch back to original settings
Screen.set_orientation(self._target, auto=True)
System.set_airplane_mode(self._target, on=False)
Screen.set_brightness(self._target, auto=True)
def run(self, exp_dir, test_name, iterations, collect=''):
# Setup test id
try:
test_id = _jankbench[test_name]
except KeyError:
raise ValueError('Jankbench test [%s] not supported', test_name)
# Initialize energy meter results
nrg_report = None
# Make sure we exit the app if already open
System.menu(self.target)
System.back(self.target)
# Close and clear application
System.force_stop(self.target, self.package, clear=True)
# Set airplane mode
System.set_airplane_mode(self.target, on=True)
# Force screen in PORTRAIT mode
Screen.set_orientation(self.target, portrait=True)
# Clear logcat
os.system(self._adb('logcat -c'))
self.logger.debug('Start Jank Benchmark [%d:%s]', test_id, test_name)
test_cmd = 'am start -n "com.android.benchmark/.app.RunLocalBenchmarksActivity" '\
'--eia "com.android.benchmark.EXTRA_ENABLED_BENCHMARK_IDS" {0} '\
'--ei "com.android.benchmark.EXTRA_RUN_COUNT" {1}'\
.format(test_id, iterations)
self.logger.info(test_cmd)
self.target.execute(test_cmd)
# Parse logcat output lines
logcat_cmd = self._adb(
'logcat ActivityManager:* System.out:I *:S BENCH:*'\
.format(self.target.adb_name))
self.logger.info("%s", logcat_cmd)
self.logger.debug("Iterations:")
logcat = Popen(logcat_cmd, shell=True, stdout=PIPE)
while True:
# read next logcat line (up to max 1024 chars)
message = logcat.stdout.readline(1024)
# Benchmark start trigger
match = JANKBENCH_BENCHMARK_START_RE.search(message)
if match:
if 'energy' in collect and self.te.emeter:
self.te.emeter.reset()
self.logger.debug("Benchmark started!")
# Benchmark completed trigger
match = JANKBENCH_BENCHMARK_DONE_RE.search(message)
if match:
if 'energy' in collect and self.te.emeter:
nrg_report = self.te.emeter.report(exp_dir)
self.logger.debug("Benchmark done!")
break
# Iteration completd
match = JANKBENCH_ITERATION_COUNT_RE.search(message)
if match:
self.logger.debug("Iteration %2d:",
int(match.group('iteration')) + 1)
# Iteration metrics
match = JANKBENCH_ITERATION_METRICS_RE.search(message)
if match:
self.logger.info(
" Mean: %7.3f JankP: %7.3f StdDev: %7.3f Count Bad: %4d Count Jank: %4d",
float(match.group('mean')), float(match.group('junk_p')),
float(match.group('std_dev')),
int(match.group('count_bad')),
int(match.group('count_junk')))
# get results
db_file = os.path.join(exp_dir, JANKBENCH_DB_NAME)
self.target.pull(JANKBENCH_DB_PATH + JANKBENCH_DB_NAME, db_file)
System.force_stop(self.target, self.package, clear=True)
# Go back to home screen
System.home(self.target)
# Switch back to screen auto rotation
Screen.set_orientation(self.target, auto=True)
return db_file, nrg_report
def run(self, out_dir, duration_s=30, collect='systrace'):
"""
Run a camera preview workload
:param out_dir: Path to experiment directory where to store results.
:type out_dir: str
:param duration_s: Duration of test
:type duration_s: int
:param collect: Specifies what to collect. Possible values:
- 'energy'
- 'systrace'
- 'ftrace'
- any combination of the above
:type collect: list(str)
"""
self._log.info("Running CameraPreview for {}s and collecting {}".format(duration_s, collect))
# Keep track of mandatory parameters
self.out_dir = out_dir
self.collect = collect
# Unlock device screen (assume no password required)
Screen.unlock(self._target)
# Set airplane mode
System.set_airplane_mode(self._target, on=True)
# Set min brightness
Screen.set_brightness(self._target, auto=False, percent=0)
# Use the monkey tool to start CameraPreview
# This allows to subsequently set the screen orientation to LANDSCAPE
# and to reset the frame statistics.
System.monkey(self._target, self.package)
# Force screen in PORTRAIT mode
Screen.set_orientation(self._target, portrait=True)
sleep(2)
self.tracingStart()
sleep(duration_s)
self.tracingStop()
# Close the app without clearing the local data to
# avoid the dialog to select the account at next start
System.force_stop(self._target, self.package, clear=False)
# Go back to home screen
System.home(self._target)
# Set brightness back to auto
Screen.set_brightness(self._target, auto=True)
# Switch back to screen auto rotation
Screen.set_orientation(self._target, auto=True)
# Switch off airplane mode
System.set_airplane_mode(self._target, on=False)