def experiment():
def run_test(outdir, test_name):
te._log.info("Running test {}".format(test_name))
wload.run(outdir, test_name=test_name, iterations=args.iterations, collect=args.collect)
if args.reimage:
System.reimage(te, args.kernel_path, args.reimage)
# Get workload
wload = Workload.getInstance(te, 'UiBench', args.reinstall)
outdir=te.res_dir + '_' + args.out_prefix
make_dir(outdir)
# Run UiBench
if args.run_all:
te._log.info("Running all tests: {}".format(wload.test_list))
for test in wload.get_test_list():
test_outdir = os.path.join(outdir, test)
make_dir(test_outdir)
run_test(test_outdir, test)
else:
test_outdir = os.path.join(outdir, args.test_name)
make_dir(test_outdir)
run_test(test_outdir, args.test_name)
# Dump platform descriptor
te.platform_dump(te.res_dir)
te._log.info('RESULTS are in out directory: {}'.format(outdir))
def experiment():
# Get workload
wload = Workload.getInstance(te, 'AppStartup')
outdir=te.res_dir + '_' + args.out_prefix
try:
shutil.rmtree(outdir)
except:
print "couldn't remove " + outdir
pass
os.makedirs(outdir)
package = 'com.example.android.powerprofile.cameraflashlight'
permissions = []
# Set airplane mode
System.set_airplane_mode(target, on=True)
# Run AppStartup workload with the flashlight app
wload.run(outdir, package=package, permissions=permissions,
duration_s=args.duration_s, collect=args.collect)
# Turn off airplane mode
System.set_airplane_mode(target, on=False)
# Dump platform descriptor
te.platform_dump(te.res_dir)
te._log.info('RESULTS are in out directory: {}'.format(outdir))
def experiment():
# Get workload
wload = Workload.getInstance(te, 'AppStartup')
outdir = te.res_dir + '_' + args.out_prefix
try:
shutil.rmtree(outdir)
except:
print "couldn't remove " + outdir
pass
os.makedirs(outdir)
package = 'com.example.android.powerprofile.gpson'
permissions = ['android.permission.ACCESS_FINE_LOCATION']
# Run AppStartup workload with the gps on app
wload.run(outdir,
package=package,
permissions=permissions,
duration_s=args.duration_s,
collect=args.collect)
# Dump platform descriptor
te.platform_dump(te.res_dir)
te._log.info('RESULTS are in out directory: {}'.format(outdir))
def _getWorkload(self):
if self.bm_name is None:
msg = 'Benchmark subclasses must override the `bm_name` attribute'
raise NotImplementedError(msg)
# Get a referench to the worload to run
wl = Workload.getInstance(self.te, self.bm_name)
if wl is None:
raise ValueError('Specified benchmark [{}] is not supported'\
.format(self.bm_name))
return wl
def _getWorkload(self):
if self.bm_name is None:
msg = 'Benchmark subclasses must override the `bm_name` attribute'
raise NotImplementedError(msg)
# Get a referench to the worload to run
wl = Workload.getInstance(self.te, self.bm_name)
if wl is None:
raise ValueError('Specified benchmark [{}] is not supported'\
.format(self.bm_name))
return wl
def experiment():
# Get workload
wload = Workload.getInstance(te, 'IdleResume')
outdir = te.res_dir + '_' + args.out_prefix
try:
shutil.rmtree(outdir)
except:
print "couldn't remove " + outdir
pass
os.makedirs(outdir)
# Run IdleResume
wload.run(outdir, duration_s=args.duration_s, collect=args.collect)
# Dump platform descriptor
te.platform_dump(te.res_dir)
te._log.info('RESULTS are in out directory: {}'.format(outdir))
def experiment():
# Get workload
wload = Workload.getInstance(te, 'YouTube')
outdir = te.res_dir + '_' + args.out_prefix
try:
shutil.rmtree(outdir)
except:
print "coulnd't remove " + outdir
pass
os.makedirs(outdir)
wload.run(outdir,
video_url='https://www.youtube.com/watch?v=NLZRYQMLDW4',
video_duration_s=args.duration_s,
collect=args.collect)
# Dump platform descriptor
te.platform_dump(te.res_dir)
te._log.info('RESULTS are in out directory: {}'.format(outdir))
def experiment(outdir):
# Get workload
wload = Workload.getInstance(te, 'UiBench')
try:
shutil.rmtree(outdir)
except:
print "coulnd't remove " + outdir
pass
os.makedirs(outdir)
# Run UiBench
wload.run(outdir,
test_name=args.test_name,
iterations=args.iterations,
collect=args.collect)
# Dump platform descriptor
te.platform_dump(te.res_dir)
te._log.info('RESULTS are in out directory: {}'.format(outdir))
def experiment():
# Get workload
wload = Workload.getInstance(te, args.workload)
outdir = te.res_dir + '_' + args.out_prefix
try:
shutil.rmtree(outdir)
except:
print "coulnd't remove " + outdir
pass
os.makedirs(outdir)
# Run Camera
# Note: The default time duration of the test is specificified within the workload
if args.duration_s:
wload.run(outdir, duration_s=args.duration_s, collect=args.collect)
else:
wload.run(outdir, collect=args.collect)
# Dump platform descriptor
te.platform_dump(te.res_dir)
te._log.info('RESULTS are in out directory: {}'.format(outdir))
def experiment():
# Get workload
wload = Workload.getInstance(te, 'DisplayImage')
outdir = te.res_dir + '_' + args.out_prefix
try:
shutil.rmtree(outdir)
except:
print "couldn't remove " + outdir
pass
os.makedirs(outdir)
# Run DisplayImage
wload.run(outdir,
duration_s=args.duration_s,
brightness=args.brightness,
filepath=os.path.join(os.environ["LISA_HOME"],
'experiments/data', args.image),
collect=args.collect)
# Dump platform descriptor
te.platform_dump(te.res_dir)
te._log.info('RESULTS are in out directory: {}'.format(outdir))
class UiBench(Workload):
"""
Android UiBench workload
"""
# Packages required by this workload
packages = [
Workload.WorkloadPackage("com.android.test.uibench",
"data/app/UiBench/UiBench.apk",
"frameworks/base/tests/UiBench"),
Workload.WorkloadPackage("com.android.uibench.janktests",
"data/app/UiBenchJankTests/UiBenchJankTests.apk",
"platform_testing/tests/jank/uibench"),
]
# Package required by this workload
package = packages[0].package_name
# Instrumentation required to run tests
test_package = packages[1].package_name
# Supported tests list
test_list = \
['UiBenchJankTests#testClippedListView',
'UiBenchJankTests#testDialogListFling',
'UiBenchJankTests#testFadingEdgeListViewFling',
'UiBenchJankTests#testFullscreenOverdraw',
'UiBenchJankTests#testGLTextureView',
'UiBenchJankTests#testInflatingListViewFling',
'UiBenchJankTests#testInvalidate',
'UiBenchJankTests#testInvalidateTree',
'UiBenchJankTests#testOpenNavigationDrawer',
'UiBenchJankTests#testOpenNotificationShade',
'UiBenchJankTests#testResizeHWLayer',
'UiBenchJankTests#testSaveLayerAnimation',
'UiBenchJankTests#testSlowBindRecyclerViewFling',
'UiBenchJankTests#testSlowNestedRecyclerViewFling',
'UiBenchJankTests#testSlowNestedRecyclerViewInitialFling',
'UiBenchJankTests#testTrivialAnimation',
'UiBenchJankTests#testTrivialListViewFling',
'UiBenchJankTests#testTrivialRecyclerListViewFling',
'UiBenchRenderingJankTests#testBitmapUploadJank',
'UiBenchRenderingJankTests#testShadowGridListFling',
'UiBenchTextJankTests#testEditTextTyping',
'UiBenchTextJankTests#testLayoutCacheHighHitrateFling',
'UiBenchTextJankTests#testLayoutCacheLowHitrateFling',
'UiBenchTransitionsJankTests#testActivityTransitionsAnimation',
'UiBenchWebView#testWebViewFling']
def __init__(self, test_env):
super(UiBench, self).__init__(test_env)
self._log = logging.getLogger('UiBench')
self._log.debug('Workload created')
# Set of output data reported by UiBench
self.db_file = None
def get_test_list(self):
return UiBench.test_list
def run(self, out_dir, test_name, iterations=10, collect=''):
"""
Run single UiBench workload.
:param out_dir: Path to experiment directory where to store results.
:type out_dir: str
:param test_name: Name of the test to run
:type test_name: str
:param iterations: Run benchmak for this required number of iterations
:type iterations: int
:param collect: Specifies what to collect. Possible values:
- 'systrace'
- 'ftrace'
:type collect: list(str)
"""
if 'energy' in collect:
raise ValueError('UiBench workload does not support energy data collection')
activity = '.' + test_name
# Keep track of mandatory parameters
self.out_dir = out_dir
self.collect = collect
# Filter out test overhead
filter_prop = System.get_boolean_property(self._target, 'debug.hwui.filter_test_overhead')
if not filter_prop:
System.set_property(
self._target, 'debug.hwui.filter_test_overhead', 'true', restart=True)
# Unlock device screen (assume no password required)
Screen.unlock(self._target)
# Close and clear application
System.force_stop(self._target, self.package, clear=True)
# Set airplane mode
System.set_airplane_mode(self._target, on=True)
# Set min brightness
Screen.set_brightness(self._target, auto=False, percent=0)
# Force screen in PORTRAIT mode
Screen.set_orientation(self._target, portrait=True)
# Clear logcat
os.system(self._adb('logcat -c'));
# Regexps for benchmark synchronization
start_logline = r'TestRunner: started'
UIBENCH_BENCHMARK_START_RE = re.compile(start_logline)
self._log.debug("START string [%s]", start_logline)
finish_logline = r'TestRunner: finished'
UIBENCH_BENCHMARK_FINISH_RE = re.compile(finish_logline)
self._log.debug("FINISH string [%s]", start_logline)
# Parse logcat output lines
logcat_cmd = self._adb(
'logcat TestRunner:* System.out:I *:S BENCH:*'\
.format(self._target.adb_name))
self._log.info("%s", logcat_cmd)
command = "am instrument -e iterations {} -e class {}{} -w {}".format(
iterations, self.test_package, activity, self.test_package)
test_proc = self._target.background(command)
logcat = Popen(logcat_cmd, shell=True, stdout=PIPE)
while True:
# read next logcat line (up to max 1024 chars)
message = logcat.stdout.readline(1024)
# Benchmark start trigger
match = UIBENCH_BENCHMARK_START_RE.search(message)
if match:
self.tracingStart()
self._log.debug("Benchmark started!")
match = UIBENCH_BENCHMARK_FINISH_RE.search(message)
if match:
self.tracingStop()
self._log.debug("Benchmark finished!")
test_proc.wait()
break
# Get frame stats
self.db_file = os.path.join(out_dir, "framestats.txt")
with open(self.db_file, 'w') as f:
f.writelines(test_proc.stdout.readlines())
self.results = self.get_results(out_dir)
# Close and clear application
System.force_stop(self._target, self.package, clear=True)
# Go back to home screen
System.home(self._target)
# Switch back to original settings
Screen.set_orientation(self._target, auto=True)
System.set_airplane_mode(self._target, on=False)
Screen.set_brightness(self._target, auto=True)
@staticmethod
def get_results(res_dir):
path = os.path.join(res_dir, 'framestats.txt')
with open(path, "r") as f:
lines = f.readlines()
values = []
columns = []
RESULTS_PARSE_RE = re.compile(r'gfx-([^=]+)=([0-9.]+)')
for line in lines:
matches = RESULTS_PARSE_RE.search(line)
if matches:
columns.append(matches.group(1))
values.append(float(matches.group(2)))
return pd.DataFrame([values], columns=columns)
class SysApp(Workload):
"""
Android system app jank test workload.
"""
packages = [
Workload.WorkloadPackage("com.android.sysapp.janktests",
"data/app/SystemAppJankTests/SystemAppJankTests.apk",
"vendor/google_testing/integration/tests/jank/sysapp"),
Workload.WorkloadPackage("com.android.chrome",
"system/app/Chrome/Chrome.apk",
"vendor/unbundled_google/packages/Chrome"),
Workload.WorkloadPackage("com.google.android.youtube",
"system/app/Youtube/Youtube.apk",
"vendor/unbundled_google/packages/YouTube")
]
test_package = packages[0].package_name
test_list = [
"ChromeJankTests#testChromeOverflowMenuTap",
"YouTubeJankTests#testYouTubeRecomendationWindowFling"
]
def __init__(self, test_env):
super(SysApp, self).__init__(test_env)
def get_test_list(self):
return SysApp.test_list
def _get_test_package(self, test_name):
name_start = test_name.partition('JankTests')[0]
name_map = {
"Chrome": "com.android.chrome",
"YouTube": "com.google.android.youtube",
}
return name_map[name_start]
def run(self, out_dir, test_name, iterations, collect=''):
"""
Run single system app jank test workload.
Performance statistics are stored in self.results, and can be retrieved
after the fact by calling SystemUi.get_results()
:param out_dir: Path to experiment directory where to store results.
:type out_dir: str
:param test_name: Name of the test to run
:type test_name: str
:param iterations: Run benchmark for this required number of iterations
:type iterations: int
:param collect: Specifies what to collect. Possible values:
- 'systrace'
- 'ftrace'
- 'gfxinfo'
- 'surfaceflinger'
- any combination of the above
:type collect: list(str)
"""
if "energy" in collect:
raise ValueError('System app workload does not support energy data collection')
activity = "." + test_name
package = self._get_test_package(test_name)
# Keep track of mandatory parameters
self.out_dir = out_dir
self.collect = collect
# Filter out test overhead
filter_prop = System.get_boolean_property(self._target, "debug.hwui.filter_test_overhead")
if not filter_prop:
System.set_property(self._target, "debug.hwui.filter_test_overhead", "true", restart=True)
# Unlock device screen (assume no password required)
Screen.unlock(self._target)
# Close and clear application
System.force_stop(self._target, package, clear=True)
# Set min brightness
Screen.set_brightness(self._target, auto=False, percent=0)
# Force screen in PORTRAIT mode
Screen.set_orientation(self._target, portrait=True)
# Delete old test results
self._target.remove("/sdcard/results.log")
# Clear logcat
self._target.execute("logcat -c")
# Regexps for benchmark synchronization
start_logline = r"TestRunner: started"
SYSAPP_BENCHMARK_START_RE = re.compile(start_logline)
self._log.debug("START string [%s]", start_logline)
finish_logline = r"TestRunner: finished"
SYSAPP_BENCHMARK_FINISH_RE = re.compile(finish_logline)
self._log.debug("FINISH string [%s]", finish_logline)
# Parse logcat output lines
logcat_cmd = self._adb("logcat TestRunner:* System.out:I *:S BENCH:*")
self._log.info("%s", logcat_cmd)
command = "am instrument -e iterations {} -e class {}{} -w {}".format(
iterations, self.test_package, activity, self.test_package)
logcat = Popen(logcat_cmd, shell=True, stdout=PIPE)
test_proc = self._target.background(command)
while True:
# read next logcat line (up to max 1024 chars)
message = logcat.stdout.readline(1024)
# Benchmark start
match = SYSAPP_BENCHMARK_START_RE.search(message)
if match:
self.tracingStart()
self._log.debug("Benchmark started!")
# Benchmark finish
match = SYSAPP_BENCHMARK_FINISH_RE.search(message)
if match:
self.tracingStop()
self._log.debug("Benchmark finished!")
test_proc.wait()
break
sleep(5)
self._target.pull("/sdcard/results.log", os.path.join(out_dir, "results.log"))
self.db_file = os.path.join(out_dir, "results.log")
self.results = self.get_results(out_dir)
# Close and clear application
System.force_stop(self._target, package, clear=True)
# Go back to home screen
System.home(self._target)
# Switch back to original settings
Screen.set_orientation(self._target, auto=True)
Screen.set_brightness(self._target, auto=True)
@staticmethod
def get_results(out_dir):
"""
Parse SysApp test output log and return a pandas dataframe of test results.
:param out_dir: Output directory for a run of the SysApp workload.
:type out_dir: str
"""
path = os.path.join(out_dir, "results.log")
with open(path, "r") as f:
lines = f.readlines()
cols = []
vals = []
for line in lines:
name, val = str.split(line)
if name == "Result":
cols.append("test-name")
vals.append(val)
elif name.startswith("gfx-"):
cols.append(name[4:])
vals.append(float(val))
else:
raise ValueError("Unrecognized line in results file")
return pd.DataFrame([vals], columns=cols)
class SystemUi(Workload):
"""
Android SystemUi jank test workload
"""
# Packages required by this workload
packages = [
Workload.WorkloadPackage(
"android.platform.systemui.tests.jank",
"data/app/UbSystemUiJankTests/UbSystemUiJankTests.apk",
"platform_testing/tests/jank/UbSystemUiJankTests")
]
# Instrumentation required to run tests
test_package = packages[0].package_name
test_list = \
["LauncherJankTests#testOpenAllAppsContainer",
"LauncherJankTests#testAllAppsContainerSwipe",
"LauncherJankTests#testHomeScreenSwipe",
"LauncherJankTests#testWidgetsContainerFling",
"SettingsJankTests#testSettingsFling",
"SystemUiJankTests#testRecentAppsFling",
"SystemUiJankTests#testRecentAppsDismiss",
"SystemUiJankTests#testNotificationListPull",
"SystemUiJankTests#testNotificationListPull_manyNotifications",
"SystemUiJankTests#testNotificationListScroll",
"SystemUiJankTests#testQuickSettingsPull",
"SystemUiJankTests#testUnlock",
"SystemUiJankTests#testExpandGroup",
"SystemUiJankTests#testClearAll",
"SystemUiJankTests#testChangeBrightness",
"SystemUiJankTests#testNotificationAppear",
"SystemUiJankTests#testCameraFromLockscreen",
"SystemUiJankTests#testAmbientWakeUp",
"SystemUiJankTests#testGoToFullShade",
"SystemUiJankTests#testInlineReply",
"SystemUiJankTests#testPinAppearance",
"SystemUiJankTests#testLaunchSettings"]
def __init__(self, test_env):
super(SystemUi, self).__init__(test_env)
self._log = logging.getLogger('SystemUi')
self._log.debug('Workload created')
# Set of output data reported by SystemUi
self.db_file = None
def get_test_list(self):
return SystemUi.test_list
def run(self, out_dir, test_name, iterations, collect=''):
"""
Run single SystemUi jank test workload.
Performance statistics are stored in self.results, and can be retrieved after
the fact by calling SystemUi.get_results()
:param out_dir: Path to experiment directory where to store results.
:type out_dir: str
:param test_name: Name of the test to run
:type test_name: str
:param iterations: Run benchmark for this required number of iterations
:type iterations: int
:param collect: Specifies what to collect. Possible values:
- 'systrace'
- 'ftrace'
- 'gfxinfo'
- 'surfaceflinger'
- any combination of the above
:type collect: list(str)
"""
if 'energy' in collect:
raise ValueError(
'SystemUi workload does not support energy data collection')
activity = '.' + test_name
# Keep track of mandatory parameters
self.out_dir = out_dir
self.collect = collect
# Filter out test overhead
filter_prop = System.get_boolean_property(
self._target, 'debug.hwui.filter_test_overhead')
if not filter_prop:
System.set_property(self._target,
'debug.hwui.filter_test_overhead',
'true',
restart=True)
# Unlock device screen (assume no password required)
Screen.unlock(self._target)
# Set airplane mode
System.set_airplane_mode(self._target, on=True)
# Set min brightness
Screen.set_brightness(self._target, auto=False, percent=0)
# Force screen in PORTRAIT mode
Screen.set_orientation(self._target, portrait=True)
# Delete old test results
self._target.remove('/sdcard/results.log')
# Clear logcat
os.system(self._adb('logcat -c'))
# Regexps for benchmark synchronization
start_logline = r'TestRunner: started'
SYSTEMUI_BENCHMARK_START_RE = re.compile(start_logline)
self._log.debug("START string [%s]", start_logline)
finish_logline = r'TestRunner: finished'
SYSTEMUI_BENCHMARK_FINISH_RE = re.compile(finish_logline)
self._log.debug("FINISH string [%s]", finish_logline)
# Parse logcat output lines
logcat_cmd = self._adb(
'logcat TestRunner:* System.out:I *:S BENCH:*'\
.format(self._target.adb_name))
self._log.info("%s", logcat_cmd)
command = "nohup am instrument -e iterations {} -e class {}{} -w {}".format(
iterations, self.test_package, activity, self.test_package)
print "command: {}".format(command)
self._target.background(command)
logcat = Popen(logcat_cmd, shell=True, stdout=PIPE)
while True:
# read next logcat line (up to max 1024 chars)
message = logcat.stdout.readline(1024)
# Benchmark start trigger
match = SYSTEMUI_BENCHMARK_START_RE.search(message)
if match:
self.tracingStart()
self._log.debug("Benchmark started!")
match = SYSTEMUI_BENCHMARK_FINISH_RE.search(message)
if match:
self.tracingStop()
self._log.debug("Benchmark finished!")
break
sleep(5)
self._target.pull('/sdcard/results.log',
os.path.join(out_dir, 'results.log'))
self._target.remove('/sdcard/results.log')
self.results = self.get_results(out_dir)
# Go back to home screen
System.home(self._target)
# Switch back to original settings
Screen.set_orientation(self._target, auto=True)
System.set_airplane_mode(self._target, on=False)
Screen.set_brightness(self._target, auto=True)
@staticmethod
def get_results(out_dir):
"""
Parse SystemUi test output log and return a pandas dataframe of test results.
:param out_dir: Output directory for a run of the SystemUi workload.
:type out_dir: str
"""
path = os.path.join(out_dir, 'results.log')
with open(path, "r") as f:
lines = f.readlines()
data = {}
for line in lines:
key, val = str.split(line)
if key == 'Result':
key = 'test-name'
elif key.startswith('gfx-'):
key = key[4:]
val = float(val)
else:
raise ValueError('Unrecognized line in results file')
data[key] = val
columns, values = zip(*((k, v) for k, v in data.iteritems()))
return pd.DataFrame([values], columns=columns)
