def main():
parser = argparse.ArgumentParser()
parser.add_argument('agi_dir', help='Path to AGI build')
parser.add_argument('out_dir', help='Path to output directory')
args = parser.parse_args()
#### Early checks and sanitization
assert os.path.isdir(args.agi_dir)
agi_dir = os.path.normpath(args.agi_dir)
assert os.path.isdir(args.out_dir)
out_dir = os.path.normpath(args.out_dir)
#### Test parameters
test_params = {
'startframe': '100',
'numframes': '5',
'observe_frames': '1',
'api': 'vulkan',
}
required_keys = ['apk', 'package', 'activity']
botutil.load_params(test_params, required_keys=required_keys)
#### Install APK
botutil.install_apk(test_params)
#### Trace the app
gapit = os.path.join(agi_dir, 'gapit')
gfxtrace = os.path.join(out_dir, test_params['package'] + '.gfxtrace')
cmd = [
gapit, 'trace', '-api', test_params['api'], '-start-at-frame',
test_params['startframe'], '-capture-frames', test_params['numframes'],
'-observe-frames', test_params['observe_frames'], '-out', gfxtrace
]
if 'additionalargs' in test_params.keys():
cmd += ['-additionalargs', test_params['additionalargs']]
cmd += [test_params['package'] + '/' + test_params['activity']]
p = botutil.runcmd(cmd)
if p.returncode != 0:
return 1
#### Stop the app asap for device cool-down
botutil.adb(['shell', 'am', 'force-stop', test_params['package']])
#### Replay
videofile = os.path.join(out_dir, test_params['package'] + '.mp4')
cmd = [
gapit, 'video', '-gapir-nofallback', '-type', 'sxs', '-frames-minimum',
test_params['numframes'], '-out', videofile, gfxtrace
]
p = botutil.runcmd(cmd)
return p.returncode
def main():
parser = argparse.ArgumentParser()
parser.add_argument('agi_dir', help='Path to AGI build')
parser.add_argument('out_dir', help='Path to output directory')
args = parser.parse_args()
#### Early checks and sanitization
assert os.path.isdir(args.agi_dir)
agi_dir = os.path.normpath(args.agi_dir)
assert os.path.isdir(args.out_dir)
out_dir = os.path.normpath(args.out_dir)
#### Test parameters
test_params = {
'startframe': '100',
'numframes': '5',
'observe_frames': '1',
'api': 'vulkan',
}
required_keys = ['apk', 'package', 'activity']
botutil.load_params(test_params, required_keys=required_keys)
#### Install APK
botutil.install_apk(test_params)
#### Trace the app
gapit = os.path.join(agi_dir, 'gapit')
gfxtrace = os.path.join(out_dir, test_params['package'] + '.gfxtrace')
cmd = [
gapit, 'trace', '-api', test_params['api'], '-start-at-frame',
test_params['startframe'], '-capture-frames', test_params['numframes'],
'-observe-frames', test_params['observe_frames'], '-out', gfxtrace
]
if 'additionalargs' in test_params.keys():
cmd += ['-additionalargs', test_params['additionalargs']]
cmd += [test_params['package'] + '/' + test_params['activity']]
p = botutil.runcmd(cmd)
if p.returncode != 0:
return 1
#### Stop the app asap for device cool-down
botutil.adb(['shell', 'am', 'force-stop', test_params['package']])
#### Replay
# Use the 'sxs-frames' mode that generates a series of PNGs rather
# than an mp4 video. This makes inspection easier, and removes the
# dependency on ffmpeg on the running hosts.
videooutfile = os.path.join(out_dir, test_params['package'] + '.frame.png')
cmd = [
gapit, 'video', '-gapir-nofallback', '-type', 'sxs-frames',
'-frames-minimum', test_params['numframes'], '-out', videooutfile,
gfxtrace
]
p = botutil.runcmd(cmd)
if p.returncode != 0:
return p.returncode
#### Screenshot test to retrieve mid-frame resources
screenshotfile = os.path.join(out_dir, test_params['package'] + '.png')
cmd = [
gapit, 'screenshot', '-executeddraws', '5', '-out', screenshotfile,
gfxtrace
]
p = botutil.runcmd(cmd)
if p.returncode != 0:
return p.returncode
#### Frame profiler
# Check that frame profiling generates valid JSON
profile_json = os.path.join(out_dir,
test_params['package'] + '.profiling.json')
cmd = [gapit, 'profile', '-json', '-out', profile_json, gfxtrace]
p = botutil.runcmd(cmd)
if p.returncode != 0:
return p.returncode
assert botutil.is_valid_json(profile_json)
#### Frame graph
# Check that framegraph generates valid JSON
framegraph_json = os.path.join(out_dir,
test_params['package'] + '.framegraph.json')
cmd = [gapit, 'framegraph', '-json', framegraph_json, gfxtrace]
p = botutil.runcmd(cmd)
if p.returncode != 0:
return p.returncode
assert botutil.is_valid_json(framegraph_json)
#### All tests have passed, return success
return 0
def main():
parser = argparse.ArgumentParser()
parser.add_argument(
'timeout',
type=int,
help='Timeout (duration limit for this test), in seconds')
parser.add_argument('test_dir',
help='Path to test directory, e.g. tests/foobar')
parser.add_argument('out_dir', help='Path to output directory')
args = parser.parse_args()
#### Early checks and sanitization
assert os.path.isdir(args.test_dir)
test_dir = os.path.abspath(args.test_dir)
assert os.path.isdir(args.out_dir)
out_dir = os.path.abspath(args.out_dir)
# bot-scripts/ contains test scripts
assert os.path.isdir('bot-scripts')
# agi/ contains the AGI build
assert os.path.isdir('agi')
agi_dir = os.path.abspath('agi')
#### Print AGI build properties (AGI version, build commit SHA)
cmd = ['cat', os.path.join(agi_dir, 'build.properties')]
botutil.runcmd(cmd)
#### Check test parameters
test_params = {}
params_file = os.path.join(test_dir, 'params.json')
assert os.path.isfile(params_file)
with open(params_file, 'r') as f:
test_params = json.load(f)
assert 'script' in test_params.keys()
test_script = os.path.abspath(
os.path.join('bot-scripts', test_params['script']))
assert os.path.isfile(test_script)
#### Timeout: make room for pre-script checks and post-script cleanup.
# All durations are in seconds.
cleanup_timeout = 15
if args.timeout < cleanup_timeout:
print(
'Error: timeout must be higher than the time for cleanup duration ({} sec)'
.format(cleanup_timeout))
return 1
test_timeout = args.timeout - cleanup_timeout
#### Check Android device access
# This first adb command may take a while if the adb deamon has to launch
p = botutil.adb(['shell', 'true'], timeout=10)
if p.returncode != 0:
print('Error: zero or more than one device connected')
return 1
# Print device fingerprint
p = botutil.adb(['shell', 'getprop', 'ro.build.fingerprint'])
print('Device fingerprint: ' + p.stdout)
#### Prepare device
# Wake up (224) and unlock (82) screen, sleep to pass any kind of animation
# The screen wakeup (224) call sometimes takes more than a second to return,
# hence the extended timeout.
botutil.adb(['shell', 'input', 'keyevent', '224'], timeout=2)
time.sleep(2)
# TODO(b/157444640): Temporary workaround: touch the screen before unlocking it to bypass a possible "Android preview" notification
botutil.adb(['shell', 'input', 'touchscreen', 'tap', '100', '100'])
time.sleep(1)
botutil.adb(['shell', 'input', 'keyevent', '82'])
time.sleep(1)
# Turn brightness to a minimum, to prevent device to get too hot
botutil.adb(
['shell', 'settings', 'put', 'system', 'screen_brightness', '0'])
# Make sure to have the screen "stay awake" during the test, we turn off the screen ourselves at the end
botutil.adb([
'shell', 'settings', 'put', 'global', 'stay_on_while_plugged_in', '7'
])
# Avoid "Viewing full screen" notifications that makes app loose focus
botutil.adb([
'shell', 'settings', 'put', 'secure', 'immersive_mode_confirmations',
'confirmed'
])
# Remove any implicit vulkan layers
botutil.adb(
['shell', 'settings', 'delete', 'global', 'enable_gpu_debug_layers'])
botutil.adb(['shell', 'settings', 'delete', 'global', 'gpu_debug_app'])
botutil.adb(['shell', 'settings', 'delete', 'global', 'gpu_debug_layers'])
botutil.adb(
['shell', 'settings', 'delete', 'global', 'gpu_debug_layer_app'])
# Clean up logcat, can take a few seconds
botutil.adb(['logcat', '-c'], timeout=5)
#### Launch test script
print('Start test script "{}" with timeout of {} seconds'.format(
test_script, test_timeout))
cmd = [test_script, agi_dir, out_dir]
test_returncode = None
stdout_filename = os.path.abspath(os.path.join(out_dir, 'stdout.txt'))
stderr_filename = os.path.abspath(os.path.join(out_dir, 'stderr.txt'))
with open(stdout_filename, 'w') as stdout_file:
with open(stderr_filename, 'w') as stderr_file:
try:
p = subprocess.run(cmd,
timeout=test_timeout,
cwd=test_dir,
stdout=stdout_file,
stderr=stderr_file)
test_returncode = p.returncode
except subprocess.TimeoutExpired as err:
# Mirror returncode from unix 'timeout' command
test_returncode = 124
#### Dump the logcat
logcat_file = os.path.join(out_dir, 'logcat.txt')
with open(logcat_file, 'w') as f:
cmd = ['adb', 'logcat', '-d']
p = subprocess.run(cmd, timeout=5, check=True, stdout=f)
#### Dump test outputs
with open(stdout_filename, 'r') as f:
print('#### Test stdout:')
print(f.read())
with open(stderr_filename, 'r') as f:
print('#### Test stderr:')
print(f.read())
print('#### Test returncode:')
print(test_returncode)
#### Turn off the device screen
# Key "power" (26) toggle between screen off and on, so first make sure to
# have the screen on with key "wake up" (224), then press "power" (26)
botutil.adb(['shell', 'input', 'keyevent', '224'])
# Wait a bit to let any kind of device wake up animation terminate
time.sleep(2)
botutil.adb(['shell', 'input', 'keyevent', '26'])
#### Force-stop AGI and test app
for abi in ['armeabiv7a', 'arm64v8a']:
botutil.adb(
['shell', 'am', 'force-stop', 'com.google.android.gapid.' + abi])
if 'package' in test_params.keys():
botutil.adb(['shell', 'am', 'force-stop', test_params['package']])
#### Test may fail halfway through, salvage any gfxtrace
gfxtraces = glob.glob(os.path.join(test_dir, '*.gfxtrace'))
# Do not salvage a gfxtrace that is listed as a test input
if ('gfxtrace' in test_params.keys()):
g = os.path.join(test_dir, test_params['gfxtrace'])
if g in gfxtraces:
gfxtraces.remove(g)
if len(gfxtraces) != 0:
salvage_dir = os.path.join(out_dir, 'harness-salvage')
os.makedirs(salvage_dir, exist_ok=True)
for gfx in gfxtraces:
dest = os.path.join(salvage_dir, os.path.basename(gfx))
os.rename(gfx, dest)
#### Analyze the return code
print('#### Test status:')
if test_returncode == 124:
print('TIMEOUT')
print(
'Sleep a bit more to trigger a Swarming-level timeout, to disambiguate a timeout from a crash'
)
time.sleep(cleanup_timeout)
elif test_returncode != 0:
print('FAIL')
else:
print('PASS')
return test_returncode
def main():
parser = argparse.ArgumentParser()
parser.add_argument('agi_dir', help='Path to AGI build')
parser.add_argument('out_dir', help='Path to output directory')
args = parser.parse_args()
#### Early checks and sanitization
assert os.path.isdir(args.agi_dir)
agi_dir = os.path.normpath(args.agi_dir)
assert os.path.isdir(args.out_dir)
out_dir = os.path.normpath(args.out_dir)
#### Test parameters
test_params = {}
required_keys = ['apk', 'package', 'activity', 'perfetto_config']
botutil.load_params(test_params, required_keys=required_keys)
#### Install APK
botutil.install_apk(test_params)
#### Retrieve device-specific perfetto config
p = botutil.adb(['shell', 'getprop', 'ro.product.device'])
device = p.stdout.rstrip()
if not device in test_params['perfetto_config'].keys():
botutil.log('Error: no perfetto config found for device: ' + device)
return 1
perfetto_config = test_params['perfetto_config'][device]
if not os.path.isfile(perfetto_config):
botutil.log('Error: perfetto config file not found: ' +
perfetto_config)
return 1
#### Trace the app
gapit = os.path.join(agi_dir, 'gapit')
perfetto_trace = os.path.join(out_dir,
test_params['package'] + '.perfetto')
cmd = [
gapit, 'trace', '-api', 'perfetto', '-for', '5s', '-perfetto',
perfetto_config, '-out', perfetto_trace
]
if 'additionalargs' in test_params.keys():
cmd += ['-additionalargs', test_params['additionalargs']]
cmd += [test_params['package'] + '/' + test_params['activity']]
p = botutil.runcmd(cmd)
if p.returncode != 0:
return 1
#### Stop the app asap for device cool-down
botutil.adb(['shell', 'am', 'force-stop', test_params['package']])
#### Check perfetto trace validity by formatting it to JSON
perfetto_json = perfetto_trace.replace('.perfetto', '.json')
cmd = [
gapit, 'perfetto', '-mode', 'metrics', '-format', 'json', '-out',
perfetto_json, perfetto_trace
]
p = botutil.runcmd(cmd)
return p.returncode