def main(args=sys.argv[1:]):
usage = "usage: %prog [options] <product> <test> <output dir>"
parser = eideticker.OptionParser(usage=usage)
parser.add_option("--enable-profiling",
action="store_true", dest = "enable_profiling",
help = "Create SPS profile to go along with capture")
parser.add_option("--device-id", action="store", dest="device_id",
help="id of device (used in output json)")
parser.add_option("--apk", action="store", dest="apk",
help = "Product apk to get metadata from " \
"(Android-specific)")
parser.add_option("--num-runs", action="store",
type = "int", dest = "num_runs",
help = "number of runs (default: 1)")
options, args = parser.parse_args()
if len(args) != 3:
parser.error("incorrect number of arguments")
(productname, testkey, outputdir) = args
num_runs = 1
if options.num_runs:
num_runs = options.num_runs
manifest = manifestparser.TestManifest(manifests=[os.path.join(
os.path.dirname(__file__), '../src/tests/manifest.ini')])
# sanity check... does the test match a known test key?
testkeys = [test["key"] for test in manifest.active_tests()]
if testkey not in testkeys:
print "ERROR: No tests matching '%s' (options: %s)" % (testkey, ", ".join(testkeys))
sys.exit(1)
testinfo = [test for test in manifest.active_tests() if test['key'] == testkey][0]
device_id = options.device_id
if not device_id:
device_id = os.environ.get('DEVICE_ID')
if not device_id:
print "ERROR: Must specify device id (either with --device-id or with DEVICE_ID environment variable)"
sys.exit(1)
product = eideticker.get_product(productname)
current_date = time.strftime("%Y-%m-%d")
datafile = os.path.join(outputdir, device_id, '%s.json' % testkey)
data = NestedDict()
if os.path.isfile(datafile):
data.update(json.loads(open(datafile).read()))
devicePrefs = eideticker.getDevicePrefs(options)
device = eideticker.getDevice(**devicePrefs)
# update the device list for the dashboard
devices = {}
devicefile = os.path.join(outputdir, 'devices.json')
if os.path.isfile(devicefile):
devices = json.loads(open(devicefile).read())['devices']
testfile = os.path.join(outputdir, '%s' % device_id, 'tests.json')
if os.path.isfile(testfile):
tests = json.loads(open(testfile).read())['tests']
else:
tests = {}
tests[testkey] = { 'shortDesc': testinfo['shortDesc'],
'defaultMeasure': testinfo['defaultMeasure'] }
devices[device_id] = { 'name': device.model,
'version': device.getprop('ro.build.version.release') }
with open(devicefile, 'w') as f:
f.write(json.dumps({ 'devices': devices }))
testfiledir = os.path.dirname(testfile)
if not os.path.exists(testfiledir):
os.mkdir(testfiledir)
with open(testfile, 'w') as f:
f.write(json.dumps({ 'tests': tests }))
if options.apk:
appinfo = eideticker.get_fennec_appinfo(options.apk)
appname = appinfo['appname']
print "Using application name '%s' from apk '%s'" % (appname, options.apk)
capture_name = "%s %s" % (product['name'], appinfo['date'])
else:
# no apk, assume it's something static on the device
appinfo = { 'date': 'today' }
appname = product['appname']
capture_name = "%s (taken on %s)" % (product['name'], current_date)
# Run the test the specified number of times
for i in range(num_runs):
# Now run the test
runtest(device, product, appname, appinfo, testinfo,
capture_name + " #%s" % i, outputdir, datafile, data,
enable_profiling=options.enable_profiling, **devicePrefs)
# Kill app after test complete
device.killProcess(appname)
def main(args=sys.argv[1:]):
usage = "usage: %prog [options] <product> <test> <output dir>"
parser = eideticker.TestOptionParser(usage=usage)
parser.add_option("--enable-profiling",
action="store_true",
dest="enable_profiling",
help="Create SPS profile to go along with capture")
parser.add_option("--device-id",
action="store",
dest="device_id",
help="id of device (used in output json)",
default=os.environ.get('DEVICE_ID'))
parser.add_option("--device-name",
action="store",
dest="device_name",
help="name of device to display in dashboard (if not "
"specified, display model name)",
default=os.environ.get('DEVICE_NAME'))
parser.add_option("--apk",
action="store",
dest="apk",
help="Product apk to get metadata from "
"(Android-specific)")
parser.add_option("--baseline",
action="store_true",
dest="baseline",
help="Create baseline results for dashboard")
parser.add_option("--num-runs",
action="store",
type="int",
dest="num_runs",
help="number of runs (default: 1)")
parser.add_option("--app-version",
action="store",
dest="app_version",
help="Specify app version (if not automatically "
"available; Android-specific)")
parser.add_option("--sources-xml",
action="store",
dest="sources_xml",
help="Path to sources XML file for getting revision "
"information (B2G-specific)")
options, args = parser.parse_args()
if len(args) != 3:
parser.print_usage()
sys.exit(1)
(productname, testkey, outputdir) = args
num_runs = 1
if options.num_runs:
num_runs = options.num_runs
testinfo = eideticker.get_testinfo(testkey)
device_id = options.device_id
if not device_id:
print "ERROR: Must specify device id (either with --device-id or with "
"DEVICE_ID environment variable)"
sys.exit(1)
# we'll log http requests for webstartup tests only
log_http_requests = False
if testinfo['type'] == 'webstartup':
log_http_requests = True
# likewise, log actions only for web tests and b2g tests
log_actions = False
if testinfo['type'] == 'web' or testinfo['type'] == 'b2g':
log_actions = True
product = eideticker.get_product(productname)
current_date = time.strftime("%Y-%m-%d")
capture_name = "%s - %s (taken on %s)" % (testkey, product['name'],
current_date)
datafile = os.path.join(outputdir, device_id, '%s.json' % testkey)
data = NestedDict()
if os.path.isfile(datafile):
data.update(json.loads(open(datafile).read()))
device_prefs = eideticker.getDevicePrefs(options)
device = eideticker.getDevice(**device_prefs)
devices = {}
devicefile = os.path.join(outputdir, 'devices.json')
if os.path.isfile(devicefile):
devices = json.loads(open(devicefile).read())['devices']
testfile = os.path.join(outputdir, '%s' % device_id, 'tests.json')
if os.path.isfile(testfile):
tests = json.loads(open(testfile).read())['tests']
else:
tests = {}
tests[testkey] = {
'shortDesc': testinfo['shortDesc'],
'defaultMeasure': testinfo['defaultMeasure']
}
device_name = options.device_name
if not device_name:
device_name = device.model
if options.devicetype == "android":
devices[device_id] = {
'name': device_name,
'version': device.getprop('ro.build.version.release')
}
if options.apk:
if options.app_version:
raise Exception("Should specify either --app-version or "
"--apk, not both!")
appinfo = eideticker.get_fennec_appinfo(options.apk)
appname = appinfo['appname']
print "Using application name '%s' from apk '%s'" % (appname,
options.apk)
capture_name = "%s %s" % (product['name'], appinfo['appdate'])
else:
if not options.app_version:
raise Exception("Should specify --app-version if not --apk!")
# no apk, assume it's something static on the device
appinfo = {
'appdate': time.strftime("%Y-%m-%d"),
'version': options.app_version
}
appname = product['appname']
elif options.devicetype == "b2g":
if not options.sources_xml:
raise Exception("Must specify --sources-xml on b2g!")
devices[device_id] = {'name': device_name}
appinicontents = device.pullFile('/system/b2g/application.ini')
sfh = StringIO.StringIO(appinicontents)
appinfo = eideticker.get_appinfo(sfh)
appinfo.update(get_revision_data(options.sources_xml))
appname = None
else:
print "Unknown device type '%s'!" % options.devicetype
# update the device / test list for the dashboard
with open(devicefile, 'w') as f:
f.write(json.dumps({'devices': devices}))
testfiledir = os.path.dirname(testfile)
if not os.path.exists(testfiledir):
os.mkdir(testfiledir)
with open(testfile, 'w') as f:
f.write(json.dumps({'tests': tests}))
if options.prepare_test:
eideticker.prepare_test(testkey, device_prefs)
# Run the test the specified number of times
for i in range(num_runs):
runtest(device,
device_prefs,
options.capture_device,
options.capture_area,
product,
appname,
appinfo,
testinfo,
capture_name + " #%s" % i,
outputdir,
datafile,
data,
enable_profiling=options.enable_profiling,
log_http_requests=log_http_requests,
log_actions=log_actions,
baseline=options.baseline,
wifi_settings_file=options.wifi_settings_file,
sync_time=options.sync_time)
if options.devicetype == "android":
# Kill app after test complete
device.killProcess(appname)
def runtest(device_prefs, testname, options, apk=None, appname=None,
appdate=None):
if apk:
appinfo = eideticker.get_fennec_appinfo(apk)
appname = appinfo['appname']
print "Installing %s (version: %s, revision %s)" % (
appinfo['appname'], appinfo['version'], appinfo['revision'])
device = eideticker.getDevice(**device_prefs)
device.updateApp(apk)
else:
appinfo = None
options.appname = appname
testinfo = eideticker.get_testinfo(testname)
stableframecapture = (testinfo['type'] in ('startup', 'webstartup') or
testinfo['defaultMeasure'] == 'timetostableframe')
capture_results = []
if options.prepare_test:
eideticker.prepare_test(testname, options)
for i in range(options.num_runs):
# Now run the test
curtime = int(time.time())
capture_file = os.path.join(CAPTURE_DIR,
"metric-test-%s-%s.zip" % (appname,
curtime))
if options.enable_profiling and options.outputdir:
profile_relpath = os.path.join(
'profiles', 'sps-profile-%s.zip' % time.time())
profile_filename = os.path.join(options.outputdir, profile_relpath)
else:
profile_filename = None
current_date = time.strftime("%Y-%m-%d")
capture_name = "%s - %s (taken on %s)" % (
testname, appname, current_date)
testlog = eideticker.run_test(testname, options,
capture_filename=capture_file,
profile_filename=profile_filename,
capture_name=capture_name)
capture_uuid = uuid.uuid1().hex
datapoint = { 'uuid': capture_uuid }
metadata = {}
metrics = {}
if options.capture:
capture = videocapture.Capture(capture_file)
datapoint['captureFile'] = metadata['captureFile'] = capture_file
metadata['captureFPS'] = capture.fps
metadata['generatedVideoFPS'] = capture.generated_video_fps
if stableframecapture:
metrics['timetostableframe'] = \
eideticker.get_stable_frame_time(capture)
else:
metrics.update(
eideticker.get_standard_metrics(capture, testlog.actions))
metadata['metrics'] = metrics
metadata.update(eideticker.get_standard_metric_metadata(capture))
if options.outputdir:
# video
video_relpath = os.path.join(
'videos', 'video-%s.webm' % time.time())
video_path = os.path.join(options.outputdir, video_relpath)
open(video_path, 'w').write(capture.get_video().read())
metadata['video'] = video_relpath
if options.log_checkerboard_stats:
metrics['internalcheckerboard'] = \
testlog.checkerboard_percent_totals
# Want metrics data in data, so we can graph everything at once
datapoint.update(metrics)
if options.enable_profiling:
metadata['profile'] = profile_filename
# dump metadata
if options.outputdir:
# metadata
metadata_path = os.path.join(options.outputdir, 'metadata',
'%s.json' % capture_uuid)
open(metadata_path, 'w').write(json.dumps(metadata))
capture_results.append(datapoint)
if options.devicetype == "b2g":
# FIXME: get information from sources.xml and application.ini on
# device, as we do in update-dashboard.py
display_key = appkey = "FirefoxOS"
else:
appkey = appname
if appdate:
appkey = appdate.isoformat()
else:
appkey = appname
if appinfo and appinfo.get('revision'):
display_key = "%s (%s)" % (appkey, appinfo['revision'])
else:
display_key = appkey
print "=== Results on %s for %s ===" % (testname, display_key)
if options.capture:
measures = [ ('timetostableframe',
'Times to first stable frame (seconds)'),
('uniqueframes', 'Number of unique frames'),
('fps', 'Average number of unique frames per second'),
('overallentropy',
'Overall entropy over length of capture'),
('checkerboard',
'Checkerboard area/duration (sum of percents NOT '
'percentage)'),
('timetoresponse',
'Time to first input response (seconds)') ]
for measure in measures:
if capture_results[0].get(measure[0]):
print " %s:" % measure[1]
print " %s" % map(lambda c: c[measure[0]], capture_results)
print
print " Capture files:"
print " Capture files: %s" % map(lambda c: c['captureFile'], capture_results)
print
if options.log_checkerboard_stats:
print " Internal Checkerboard Stats (sum of percents, not "
"percentage):"
print " %s" % map(
lambda c: c['internalcheckerboard'], capture_results)
print
if options.outputdir:
outputfile = os.path.join(options.outputdir, "metric.json")
resultdict = {'title': testname, 'data': {}}
if os.path.isfile(outputfile):
resultdict.update(json.loads(open(outputfile).read()))
if not resultdict['data'].get(appkey):
resultdict['data'][appkey] = []
resultdict['data'][appkey].extend(capture_results)
with open(outputfile, 'w') as f:
f.write(json.dumps(resultdict))
def main(args=sys.argv[1:]):
usage = "usage: %prog [options] <product> <test> <output dir>"
parser = eideticker.CaptureOptionParser(usage=usage)
parser.add_option("--enable-profiling",
action="store_true", dest = "enable_profiling",
help = "Create SPS profile to go along with capture")
parser.add_option("--device-id", action="store", dest="device_id",
help="id of device (used in output json)")
parser.add_option("--apk", action="store", dest="apk",
help = "Product apk to get metadata from " \
"(Android-specific)")
parser.add_option("--baseline", action="store_true", dest="baseline",
help = "Create baseline results for dashboard")
parser.add_option("--num-runs", action="store",
type = "int", dest = "num_runs",
help = "number of runs (default: 1)")
parser.add_option("--app-version", action="store", dest="app_version",
help="Specify app version (if not automatically " \
"available; Android-specific)")
parser.add_option("--sources-xml", action="store", dest="sources_xml",
help="Path to sources XML file for getting revision " \
"information (B2G-specific)")
options, args = parser.parse_args()
parser.validate_options(options)
(productname, testkey, outputdir) = args
num_runs = 1
if options.num_runs:
num_runs = options.num_runs
manifest = eideticker.get_test_manifest()
# sanity check... does the test match a known test key?
testkeys = [test["key"] for test in manifest.active_tests()]
if testkey not in testkeys:
print "ERROR: No tests matching '%s' (options: %s)" % (testkey, ", ".join(testkeys))
sys.exit(1)
testinfo = [test for test in manifest.active_tests() if test['key'] == testkey][0]
device_id = options.device_id
if not device_id:
device_id = os.environ.get('DEVICE_ID')
if not device_id:
print "ERROR: Must specify device id (either with --device-id or with DEVICE_ID environment variable)"
sys.exit(1)
# we'll log http requests for webstartup tests only
log_http_requests = False
if testinfo['type'] == 'webstartup':
log_http_requests = True
product = eideticker.get_product(productname)
current_date = time.strftime("%Y-%m-%d")
capture_name = "%s (taken on %s)" % (product['name'], current_date)
datafile = os.path.join(outputdir, device_id, '%s.json' % testkey)
data = NestedDict()
if os.path.isfile(datafile):
data.update(json.loads(open(datafile).read()))
device_prefs = eideticker.getDevicePrefs(options)
device = eideticker.getDevice(**device_prefs)
devices = {}
devicefile = os.path.join(outputdir, 'devices.json')
if os.path.isfile(devicefile):
devices = json.loads(open(devicefile).read())['devices']
testfile = os.path.join(outputdir, '%s' % device_id, 'tests.json')
if os.path.isfile(testfile):
tests = json.loads(open(testfile).read())['tests']
else:
tests = {}
tests[testkey] = { 'shortDesc': testinfo['shortDesc'],
'defaultMeasure': testinfo['defaultMeasure'] }
if options.devicetype == "android":
devices[device_id] = { 'name': device.model,
'version': device.getprop('ro.build.version.release') }
if options.apk:
if options.app_version:
raise Exception("Should specify either --app-version or --apk, not both!")
appinfo = eideticker.get_fennec_appinfo(options.apk)
appname = appinfo['appname']
print "Using application name '%s' from apk '%s'" % (appname, options.apk)
capture_name = "%s %s" % (product['name'], appinfo['appdate'])
else:
if not options.app_version:
raise Exception("Should specify --app-version if not --apk!")
# no apk, assume it's something static on the device
appinfo = { 'appdate': time.strftime("%Y-%m-%d"), 'version': options.app_version }
appname = product['appname']
elif options.devicetype == "b2g":
if not options.sources_xml:
raise Exception("Must specify --sources-xml on b2g!")
devices[device_id] = { 'name': device.model }
appinicontents = device.pullFile('/system/b2g/application.ini')
sfh = StringIO.StringIO(appinicontents)
appinfo = eideticker.get_appinfo(sfh)
appinfo.update(get_revision_data(options.sources_xml))
appname = None
else:
print "Unknown device type '%s'!" % options.devicetype
# update the device / test list for the dashboard
with open(devicefile, 'w') as f:
f.write(json.dumps({ 'devices': devices }))
testfiledir = os.path.dirname(testfile)
if not os.path.exists(testfiledir):
os.mkdir(testfiledir)
with open(testfile, 'w') as f:
f.write(json.dumps({ 'tests': tests }))
capture_area = None
if options.capture_area:
# we validated this previously...
capture_area = json.loads(options.capture_area)
# Run the test the specified number of times
for i in range(num_runs):
# Now run the test
runtest(device, device_prefs, options.capture_device, capture_area,
product, appname, appinfo, testinfo,
capture_name + " #%s" % i, outputdir, datafile, data,
enable_profiling=options.enable_profiling,
log_http_requests=log_http_requests,
baseline=options.baseline)
if options.devicetype == "android":
# Kill app after test complete
device.killProcess(appname)
def runtest(device_prefs,
testname,
options,
apk=None,
appname=None,
appdate=None):
if apk:
appinfo = eideticker.get_fennec_appinfo(apk)
appname = appinfo['appname']
print "Installing %s (version: %s, revision %s)" % (
appinfo['appname'], appinfo['version'], appinfo['revision'])
device = eideticker.getDevice(**device_prefs)
device.updateApp(apk)
else:
appinfo = None
testinfo = eideticker.get_testinfo(testname)
stableframecapture = (testinfo['type'] in ('startup', 'webstartup')
or testinfo['defaultMeasure'] == 'timetostableframe')
capture_results = []
if options.prepare_test:
eideticker.prepare_test(testname, device_prefs,
options.wifi_settings_file)
for i in range(options.num_runs):
# Now run the test
curtime = int(time.time())
capture_file = os.path.join(
CAPTURE_DIR, "metric-test-%s-%s.zip" % (appname, curtime))
if options.enable_profiling and options.outputdir:
profile_relpath = os.path.join('profiles',
'sps-profile-%s.zip' % time.time())
profile_file = os.path.join(options.outputdir, profile_relpath)
else:
profile_file = None
current_date = time.strftime("%Y-%m-%d")
capture_name = "%s - %s (taken on %s)" % (testname, appname,
current_date)
testlog = eideticker.run_test(
testname,
options.capture_device,
appname,
capture_name,
device_prefs,
extra_prefs=options.extra_prefs,
extra_env_vars=options.extra_env_vars,
log_checkerboard_stats=options.get_internal_checkerboard_stats,
profile_file=profile_file,
capture_area=options.capture_area,
camera_settings_file=options.camera_settings_file,
capture=options.capture,
fps=options.fps,
capture_file=capture_file,
wifi_settings_file=options.wifi_settings_file,
sync_time=options.sync_time,
use_vpxenc=options.use_vpxenc)
capture_uuid = uuid.uuid1().hex
datapoint = {'uuid': capture_uuid}
metadata = {}
metrics = {}
if options.capture:
capture = videocapture.Capture(capture_file)
datapoint['captureFile'] = metadata['captureFile'] = capture_file
metadata['captureFPS'] = capture.fps
metadata['generatedVideoFPS'] = capture.generated_video_fps
if stableframecapture:
metrics['timetostableframe'] = \
eideticker.get_stable_frame_time(capture)
else:
metrics.update(
eideticker.get_standard_metrics(capture, testlog.actions))
metadata['metrics'] = metrics
metadata.update(eideticker.get_standard_metric_metadata(capture))
if options.outputdir:
# video
video_relpath = os.path.join('videos',
'video-%s.webm' % time.time())
video_path = os.path.join(options.outputdir, video_relpath)
open(video_path, 'w').write(capture.get_video().read())
metadata['video'] = video_relpath
if options.get_internal_checkerboard_stats:
metrics['internalcheckerboard'] = \
testlog.checkerboard_percent_totals
# Want metrics data in data, so we can graph everything at once
datapoint.update(metrics)
if options.enable_profiling:
metadata['profile'] = profile_file
# dump metadata
if options.outputdir:
# metadata
metadata_path = os.path.join(options.outputdir, 'metadata',
'%s.json' % capture_uuid)
open(metadata_path, 'w').write(json.dumps(metadata))
capture_results.append(datapoint)
if options.devicetype == "b2g":
# FIXME: get information from sources.xml and application.ini on
# device, as we do in update-dashboard.py
display_key = appkey = "FirefoxOS"
else:
appkey = appname
if appdate:
appkey = appdate.isoformat()
else:
appkey = appname
if appinfo and appinfo.get('revision'):
display_key = "%s (%s)" % (appkey, appinfo['revision'])
else:
display_key = appkey
print "=== Results on %s for %s ===" % (testname, display_key)
if options.capture:
measures = [
('timetostableframe', 'Times to first stable frame (seconds)'),
('uniqueframes', 'Number of unique frames'),
('fps', 'Average number of unique frames per second'),
('overallentropy', 'Overall entropy over length of capture'),
('checkerboard', 'Checkerboard area/duration (sum of percents NOT '
'percentage)'),
('timetoresponse', 'Time to first input response (seconds)')
]
for measure in measures:
if capture_results[0].get(measure[0]):
print " %s:" % measure[1]
print " %s" % map(lambda c: c[measure[0]], capture_results)
print
print " Capture files:"
print " Capture files: %s" % map(lambda c: c['captureFile'],
capture_results)
print
if options.get_internal_checkerboard_stats:
print " Internal Checkerboard Stats (sum of percents, not "
"percentage):"
print " %s" % map(lambda c: c['internalcheckerboard'],
capture_results)
print
if options.outputdir:
outputfile = os.path.join(options.outputdir, "metric.json")
resultdict = {'title': testname, 'data': {}}
if os.path.isfile(outputfile):
resultdict.update(json.loads(open(outputfile).read()))
if not resultdict['data'].get(appkey):
resultdict['data'][appkey] = []
resultdict['data'][appkey].extend(capture_results)
with open(outputfile, 'w') as f:
f.write(json.dumps(resultdict))
def main(args=sys.argv[1:]):
usage = "usage: %prog [options] TEST..."
parser = eideticker.TestOptionParser(usage=usage)
eideticker.add_dashboard_options(parser)
parser.add_option("--enable-profiling",
action="store_true", dest="enable_profiling",
help="Create SPS profile to go along with capture")
parser.add_option("--device-id", action="store", dest="device_id",
help="id of device (used in output json)",
default=os.environ.get('DEVICE_ID'))
parser.add_option("--branch", action="store", dest="branch_id",
help="branch under test (used in output json)",
default=os.environ.get('BRANCH'))
parser.add_option("--device-name", action="store", dest="device_name",
help="name of device to display in dashboard (if not "
"specified, display model name)",
default=os.environ.get('DEVICE_NAME'))
parser.add_option("--apk", action="store", dest="apk",
help="Product apk to get metadata from "
"(Android-specific)")
parser.add_option("--baseline", action="store_true", dest="baseline",
help="Create baseline results for dashboard")
parser.add_option("--num-runs", action="store",
type="int", dest="num_runs",
help="number of runs (default: %default)", default=1)
parser.add_option("--app-version", action="store", dest="app_version",
help="Specify app version (if not automatically "
"available; Android-specific)")
parser.add_option("--sources-xml", action="store", dest="sources_xml",
help="Path to sources XML file for getting revision "
"information (B2G-specific)")
parser.add_option("--product", action="store",
type="string", dest="product_name",
default="nightly",
help="product name (android-specific, default: "
"%default)")
options, args = parser.parse_args()
if not args: # need to specify at least one test to run!
parser.print_usage()
sys.exit(1)
if not options.device_id:
print "ERROR: Must specify device id (either with --device-id or with " \
"DEVICE_ID environment variable)"
sys.exit(1)
if not options.branch_id:
print "ERROR: Must specify branch (either with --branch or with " \
"BRANCH environment variable)"
sys.exit(1)
# get device info
device_prefs = eideticker.getDevicePrefs(options)
device = eideticker.getDevice(**device_prefs)
device_name = options.device_name
if not device_name:
device_name = device.model
# copy dashboard files to output directory (if applicable)
eideticker.copy_dashboard_files(options.dashboard_dir)
if options.devicetype == 'android':
product = eideticker.get_product(options.product_name)
device_info = { 'name': device_name,
'version': device.getprop('ro.build.version.release')}
elif options.devicetype == 'b2g':
product = eideticker.get_product('b2g-nightly')
device_info = { 'name': device_name }
else:
print "ERROR: Unknown device type '%s'" % options.devicetype
# update device index
eideticker.update_dashboard_device_list(options.dashboard_dir, options.device_id,
options.branch_id, device_info)
# get application/build info
if options.devicetype == "android":
if options.apk:
if options.app_version:
raise Exception("Should specify either --app-version or "
"--apk, not both!")
appinfo = eideticker.get_fennec_appinfo(options.apk)
options.appname = appinfo['appname']
print "Using application name '%s' from apk '%s'" % (
options.appname, options.apk)
options.capture_name = "%s %s" % (product['name'], appinfo['appdate'])
else:
if not options.app_version:
raise Exception("Should specify --app-version if not --apk!")
# no apk, assume it's something static on the device
appinfo = {
'appdate': time.strftime("%Y-%m-%d"),
'version': options.app_version}
elif options.devicetype == "b2g":
if not options.sources_xml:
raise Exception("Must specify --sources-xml on b2g!")
appinicontents = device.pullFile('/system/b2g/application.ini')
sfh = StringIO.StringIO(appinicontents)
appinfo = eideticker.get_appinfo(sfh)
appinfo.update(get_revision_data(options.sources_xml))
options.appname = None
else:
print "Unknown device type '%s'!" % options.devicetype
# run through the tests...
failed_tests = []
for testkey in args:
testinfo = eideticker.get_testinfo(testkey)
eideticker.update_dashboard_test_list(options.dashboard_dir, options.device_id,
options.branch_id,
testinfo)
current_date = time.strftime("%Y-%m-%d")
options.capture_name = "%s - %s (taken on %s)" % (testkey, product['name'],
current_date)
if options.prepare_test:
eideticker.prepare_test(testkey, options)
# Run the test the specified number of times
for i in range(options.num_runs):
try:
runtest(device, device_prefs, options,
product, appinfo, testinfo,
options.capture_name + " #%s" % i)
except eideticker.TestException:
print "Unable to run test '%s'. Skipping and continuing." % testkey
failed_tests.append(testkey)
break
# synchronize with dashboard (if we have a server to upload to)
if options.dashboard_server:
eideticker.upload_dashboard(options)
else:
print "No dashboard server specified. Skipping upload."
if failed_tests:
print "The following tests failed: %s" % ", ".join(failed_tests)
sys.exit(1)
def run_test(device, outputdir, outputfile, test, url_params, num_runs,
startup_test, no_capture, get_internal_checkerboard_stats,
apk=None, appname = None, appdate = None, enable_profiling=False,
extra_prefs="{}", dmtype="adb", devicetype="android", host=None,
port=None):
if apk:
appinfo = eideticker.get_fennec_appinfo(apk)
appname = appinfo['appname']
print "Installing %s (version: %s, revision %s)" % (appinfo['appname'],
appinfo['version'],
appinfo['revision'])
device.updateApp(apk)
else:
appinfo = None
captures = []
for i in range(num_runs):
# Kill any existing instances of the processes
device.killProcess(appname)
# Now run the test
curtime = int(time.time())
capture_file = os.path.join(CAPTURE_DIR,
"metric-test-%s-%s.zip" % (appname,
curtime))
if enable_profiling:
profile_file = os.path.join(PROFILE_DIR,
"profile-%s-%s.zip" % (appname, curtime))
args = ["runtest.py", "--url-params", url_params,
"--extra-prefs", extra_prefs, appname, test]
if get_internal_checkerboard_stats:
checkerboard_logfile = tempfile.NamedTemporaryFile()
args.extend(["--checkerboard-log-file", checkerboard_logfile.name])
if startup_test:
args.extend(["--startup-test"])
if no_capture:
args.extend(["--no-capture"])
else:
args.extend(["--capture-file", capture_file])
if enable_profiling:
args.extend(["--profile-file", profile_file])
if dmtype:
args.extend(["-m", dmtype])
if devicetype:
args.extend(["-d", devicetype])
if host:
args.extend(["--host", host])
if port:
args.extend(["--port", port])
print args
retval = subprocess.call(args)
if retval != 0:
raise Exception("Failed to run test %s for %s" % (test, appname))
capture_result = {}
if not no_capture:
capture_result['file'] = capture_file
capture = videocapture.Capture(capture_file)
if startup_test:
capture_result['stableframe'] = videocapture.get_stable_frame(capture)
else:
capture_result['uniqueframes'] = videocapture.get_num_unique_frames(capture)
capture_result['fps'] = videocapture.get_fps(capture)
capture_result['checkerboard'] = videocapture.get_checkerboarding_area_duration(capture)
if outputdir:
video_path = os.path.join('videos', 'video-%s.webm' % time.time())
video_file = os.path.join(outputdir, video_path)
open(video_file, 'w').write(capture.get_video().read())
capture_result['video'] = video_path
if enable_profiling:
capture_result['profile'] = profile_file
if get_internal_checkerboard_stats:
internal_checkerboard_totals = parse_checkerboard_log(checkerboard_logfile.name)
capture_result['internalcheckerboard'] = internal_checkerboard_totals
captures.append(capture_result)
appkey = appname
if appdate:
appkey = appdate.isoformat()
else:
appkey = appname
if appinfo and appinfo.get('revision'):
display_key = "%s (%s)" % (appkey, appinfo['revision'])
else:
display_key = appkey
print "=== Results for %s ===" % display_key
if not no_capture:
if startup_test:
print " First stable frames:"
print " %s" % map(lambda c: c['stableframe'], captures)
print
else:
print " Number of unique frames:"
print " %s" % map(lambda c: c['uniqueframes'], captures)
print
print " Average number of unique frames per second:"
print " %s" % map(lambda c: c['fps'], captures)
print
print " Checkerboard area/duration (sum of percents NOT percentage):"
print " %s" % map(lambda c: c['checkerboard'], captures)
print
print " Capture files:"
print " Capture files: %s" % map(lambda c: c['file'], captures)
print
if enable_profiling:
print " Profile files:"
print " Profile files: %s" % map(lambda c: c['profile'], captures)
print
if get_internal_checkerboard_stats:
print " Internal Checkerboard Stats (sum of percents, not percentage):"
print " %s" % map(lambda c: c['internalcheckerboard'], captures)
print
if outputfile:
resultdict = { 'title': test, 'data': {} }
if os.path.isfile(outputfile):
resultdict.update(json.loads(open(outputfile).read()))
if not resultdict['data'].get(appkey):
resultdict['data'][appkey] = []
resultdict['data'][appkey].extend(captures)
with open(outputfile, 'w') as f:
f.write(json.dumps(resultdict))
def run_test(device,
outputdir,
outputfile,
test,
url_params,
num_runs,
startup_test,
no_capture,
get_internal_checkerboard_stats,
apk=None,
appname=None,
appdate=None,
profile_file=None,
dmtype="adb",
host=None,
port=None):
if apk:
appinfo = eideticker.get_fennec_appinfo(apk)
appname = appinfo['appname']
print "Installing %s (version: %s, revision %s)" % (
appinfo['appname'], appinfo['version'], appinfo['revision'])
device.updateApp(apk)
else:
appinfo = None
captures = []
for i in range(num_runs):
# Kill any existing instances of the processes
device.killProcess(appname)
# Now run the test
capture_file = os.path.join(
CAPTURE_DIR, "metric-test-%s-%s.zip" % (appname, int(time.time())))
args = ["runtest.py", "--url-params", url_params, appname, test]
if get_internal_checkerboard_stats:
checkerboard_logfile = tempfile.NamedTemporaryFile()
args.extend(["--checkerboard-log-file", checkerboard_logfile.name])
if startup_test:
args.extend(["--startup-test"])
if no_capture:
args.extend(["--no-capture"])
else:
args.extend(["--capture-file", capture_file])
if profile_file:
args.extend(["--profile-file", profile_file])
if dmtype:
args.extend(["-m", dmtype])
if host:
args.extend(["--host", host])
if port:
args.extend(["--port", port])
print args
retval = subprocess.call(args)
if retval != 0:
raise Exception("Failed to run test %s for %s" % (test, appname))
capture_result = {}
if not no_capture:
capture_result['file'] = capture_file
capture = videocapture.Capture(capture_file)
framediff_sums = videocapture.get_framediff_sums(capture)
if startup_test:
capture_result['stableframe'] = videocapture.get_stable_frame(
capture)
else:
capture_result[
'uniqueframes'] = videocapture.get_num_unique_frames(
capture)
capture_result['fps'] = videocapture.get_fps(capture)
capture_result[
'checkerboard'] = videocapture.get_checkerboarding_area_duration(
capture)
if outputdir:
video_path = os.path.join('videos',
'video-%s.webm' % time.time())
video_file = os.path.join(outputdir, video_path)
open(video_file, 'w').write(capture.get_video().read())
capture_result['video'] = video_path
if get_internal_checkerboard_stats:
internal_checkerboard_totals = parse_checkerboard_log(
checkerboard_logfile.name)
capture_result[
'internalcheckerboard'] = internal_checkerboard_totals
captures.append(capture_result)
appkey = appname
if appdate:
appkey = appdate.isoformat()
else:
appkey = appname
if appinfo and appinfo.get('revision'):
display_key = "%s (%s)" % (appkey, appinfo['revision'])
else:
display_key = appkey
print "=== Results for %s ===" % display_key
if not no_capture:
if startup_test:
print " First stable frames:"
print " %s" % map(lambda c: c['stableframe'], captures)
print
else:
print " Number of unique frames:"
print " %s" % map(lambda c: c['uniqueframes'], captures)
print
print " Average number of unique frames per second:"
print " %s" % map(lambda c: c['fps'], captures)
print
print " Checkerboard area/duration (sum of percents NOT percentage):"
print " %s" % map(lambda c: c['checkerboard'], captures)
print
print " Capture files (for further reference):"
print " Capture files: %s" % map(lambda c: c['file'], captures)
print
if get_internal_checkerboard_stats:
print " Internal Checkerboard Stats (sum of percents, not percentage):"
print " %s" % map(lambda c: c['internalcheckerboard'], captures)
print
if outputfile:
resultdict = {'title': test, 'data': {}}
if os.path.isfile(outputfile):
resultdict.update(json.loads(open(outputfile).read()))
if not resultdict['data'].get(appkey):
resultdict['data'][appkey] = []
resultdict['data'][appkey].extend(captures)
with open(outputfile, 'w') as f:
f.write(json.dumps(resultdict))
def main(args=sys.argv[1:]):
usage = "usage: %prog [options] <test> <output dir>"
parser = eideticker.OptionParser(usage=usage)
parser.add_option("--enable-profiling",
action="store_true",
dest="enable_profiling",
help="Create SPS profile to go along with capture")
parser.add_option("--device-id",
action="store",
dest="device_id",
help="id of device (used in output json)")
parser.add_option(
"--product",
action="store",
dest="product",
help="Restrict testing to product (options: %s)" %
", ".join([product["name"] for product in default_products]))
parser.add_option("--num-runs",
action="store",
type="int",
dest="num_runs",
help="number of runs (default: 1)")
options, args = parser.parse_args()
if len(args) != 2:
parser.error("incorrect number of arguments")
(testname, outputdir) = args
num_runs = 1
if options.num_runs:
num_runs = options.num_runs
testnames = [test["name"] for test in default_tests]
if testname not in testnames:
print "ERROR: No tests matching '%s' (options: %s)" % (
testname, ", ".join(testnames))
sys.exit(1)
else:
test = [test for test in default_tests if test['name'] == testname][0]
device_id = options.device_id
if not device_id:
device_id = os.environ.get('DEVICE_ID')
if not device_id:
print "ERROR: Must specify device id (either with --device-id or with DEVICE_ID environment variable)"
sys.exit(1)
products = default_products
if options.product:
products = [
product for product in default_products
if product['name'] == options.product
]
if not products:
print "ERROR: No products matching '%s'" % options.product
sys.exit(1)
current_date = time.strftime("%Y-%m-%d")
datafile = os.path.join(outputdir, 'data-%s.json' % device_id)
data = NestedDict()
if os.path.isfile(datafile):
data.update(json.loads(open(datafile).read()))
devicePrefs = eideticker.getDevicePrefs(options)
device = eideticker.getDevice(**devicePrefs)
# update the device list for the dashboard
devices = {}
devicefile = os.path.join(outputdir, 'devices.json')
if os.path.isfile(devicefile):
devices = json.loads(open(devicefile).read())['devices']
devices[device_id] = {
'name': device.model,
'version': device.getprop('ro.build.version.release')
}
with open(devicefile, 'w') as f:
f.write(json.dumps({'devices': devices}))
for product in products:
if product.get('url'):
product_fname = os.path.join(DOWNLOAD_DIR,
"%s.apk" % product['name'])
appinfo = eideticker.get_fennec_appinfo(product_fname)
appname = appinfo['appname']
print "Using application name '%s' from apk '%s'" % (appname,
product_fname)
capture_name = "%s %s" % (product['name'], appinfo['date'])
else:
appinfo = {}
appname = product['appname']
capture_name = "%s (taken on %s)" % (product['name'], current_date)
if appinfo.get('appname'):
appname = appinfo['appname']
else:
appname = product['appname']
# Run the test the specified number of times
for i in range(num_runs):
# Now run the test
runtest(device,
product,
current_date,
appname,
appinfo,
test,
capture_name + " #%s" % i,
outputdir,
datafile,
data,
enable_profiling=options.enable_profiling,
**devicePrefs)
# Kill app after test complete
device.killProcess(appname)
def runtest(device_prefs, testname, options, apk=None, appname=None, appdate=None):
device = None
if apk:
appinfo = eideticker.get_fennec_appinfo(apk)
appname = appinfo["appname"]
print "Installing %s (version: %s, revision %s)" % (appinfo["appname"], appinfo["version"], appinfo["revision"])
device = eideticker.getDevice(**device_prefs)
device.updateApp(apk)
else:
appinfo = None
testinfo = eideticker.get_testinfo(testname)
stableframecapture = (
testinfo["type"] in ("startup", "webstartup") or testinfo["defaultMeasure"] == "timetostableframe"
)
capture_results = []
for i in range(options.num_runs):
# Kill any existing instances of the processes (for Android)
if device:
device.killProcess(appname)
# Now run the test
curtime = int(time.time())
capture_file = os.path.join(CAPTURE_DIR, "metric-test-%s-%s.zip" % (appname, curtime))
if options.enable_profiling:
profile_file = os.path.join(PROFILE_DIR, "profile-%s-%s.zip" % (appname, curtime))
else:
profile_file = None
current_date = time.strftime("%Y-%m-%d")
capture_name = "%s - %s (taken on %s)" % (testname, appname, current_date)
if options.prepare_test:
eideticker.prepare_test(testname, device_prefs)
testlog = eideticker.run_test(
testname,
options.capture_device,
appname,
capture_name,
device_prefs,
extra_prefs=options.extra_prefs,
extra_env_vars=options.extra_env_vars,
log_checkerboard_stats=options.get_internal_checkerboard_stats,
profile_file=profile_file,
capture_area=options.capture_area,
no_capture=options.no_capture,
fps=options.fps,
capture_file=capture_file,
wifi_settings_file=options.wifi_settings_file,
sync_time=options.sync_time,
)
capture_result = {}
if not options.no_capture:
capture_result["file"] = capture_file
capture = videocapture.Capture(capture_file)
capture_result["captureFPS"] = capture.fps
if stableframecapture:
capture_result["timetostableframe"] = eideticker.get_stable_frame_time(capture)
else:
capture_result.update(eideticker.get_standard_metrics(capture, testlog.actions))
if options.outputdir:
# video
video_relpath = os.path.join("videos", "video-%s.webm" % time.time())
video_path = os.path.join(options.outputdir, video_relpath)
open(video_path, "w").write(capture.get_video().read())
capture_result["video"] = video_relpath
# framediff
framediff_relpath = os.path.join("framediffs", "framediff-%s.json" % time.time())
framediff_path = os.path.join(options.outputdir, framediff_relpath)
with open(framediff_path, "w") as f:
framediff = videocapture.get_framediff_sums(capture)
f.write(json.dumps({"diffsums": framediff}))
capture_result["frameDiff"] = framediff_relpath
if options.enable_profiling:
capture_result["profile"] = profile_file
if options.get_internal_checkerboard_stats:
capture_result["internalcheckerboard"] = testlog.checkerboard_percent_totals
capture_results.append(capture_result)
if options.devicetype == "b2g":
# FIXME: get information from sources.xml and application.ini on
# device, as we do in update-dashboard.py
display_key = appkey = "FirefoxOS"
else:
appkey = appname
if appdate:
appkey = appdate.isoformat()
else:
appkey = appname
if appinfo and appinfo.get("revision"):
display_key = "%s (%s)" % (appkey, appinfo["revision"])
else:
display_key = appkey
print "=== Results on %s for %s ===" % (testname, display_key)
if not options.no_capture:
if stableframecapture:
print " Times to first stable frame (seconds):"
print " %s" % map(lambda c: c["timetostableframe"], capture_results)
print
else:
print " Number of unique frames:"
print " %s" % map(lambda c: c["uniqueframes"], capture_results)
print
print " Average number of unique frames per second:"
print " %s" % map(lambda c: c["fps"], capture_results)
print
print " Checkerboard area/duration (sum of percents NOT percentage):"
print " %s" % map(lambda c: c["checkerboard"], capture_results)
print
print " Time to first input response: "
print " %s" % map(lambda c: c.get("timetoresponse"), capture_results)
print
print " Capture files:"
print " Capture files: %s" % map(lambda c: c["file"], capture_results)
print
if options.enable_profiling:
print " Profile files:"
print " Profile files: %s" % map(lambda c: c["profile"], capture_results)
print
if options.get_internal_checkerboard_stats:
print " Internal Checkerboard Stats (sum of percents, not percentage):"
print " %s" % map(lambda c: c["internalcheckerboard"], capture_results)
print
if options.outputdir:
outputfile = os.path.join(options.outputdir, "metric.json")
resultdict = {"title": testname, "data": {}}
if os.path.isfile(outputfile):
resultdict.update(json.loads(open(outputfile).read()))
if not resultdict["data"].get(appkey):
resultdict["data"][appkey] = []
resultdict["data"][appkey].extend(capture_results)
with open(outputfile, "w") as f:
f.write(json.dumps(resultdict))
def main(args=sys.argv[1:]):
usage = "usage: %prog [options] <test> <output dir>"
parser = eideticker.OptionParser(usage=usage)
parser.add_option("--enable-profiling",
action="store_true", dest = "enable_profiling",
help = "Create SPS profile to go along with capture")
parser.add_option("--device-id", action="store", dest="device_id",
help="id of device (used in output json)")
parser.add_option("--product",
action="store", dest="product",
help = "Restrict testing to product (options: %s)" %
", ".join([product["name"] for product in default_products]))
parser.add_option("--num-runs", action="store",
type = "int", dest = "num_runs",
help = "number of runs (default: 1)")
options, args = parser.parse_args()
if len(args) != 2:
parser.error("incorrect number of arguments")
(testname, outputdir) = args
num_runs = 1
if options.num_runs:
num_runs = options.num_runs
testnames = [test["name"] for test in default_tests]
if testname not in testnames:
print "ERROR: No tests matching '%s' (options: %s)" % (testname, ", ".join(testnames))
sys.exit(1)
else:
test = [test for test in default_tests if test['name'] == testname][0]
device_id = options.device_id
if not device_id:
device_id = os.environ.get('DEVICE_ID')
if not device_id:
print "ERROR: Must specify device id (either with --device-id or with DEVICE_ID environment variable)"
sys.exit(1)
products = default_products
if options.product:
products = [product for product in default_products if product['name'] == options.product]
if not products:
print "ERROR: No products matching '%s'" % options.product
sys.exit(1)
current_date = time.strftime("%Y-%m-%d")
datafile = os.path.join(outputdir, 'data-%s.json' % device_id)
data = NestedDict()
if os.path.isfile(datafile):
data.update(json.loads(open(datafile).read()))
devicePrefs = eideticker.getDevicePrefs(options)
device = eideticker.getDevice(**devicePrefs)
# update the device list for the dashboard
devices = {}
devicefile = os.path.join(outputdir, 'devices.json')
if os.path.isfile(devicefile):
devices = json.loads(open(devicefile).read())['devices']
devices[device_id] = { 'name': device.model,
'version': device.getprop('ro.build.version.release') }
with open(devicefile, 'w') as f:
f.write(json.dumps({ 'devices': devices }))
for product in products:
if product.get('url'):
product_fname = os.path.join(DOWNLOAD_DIR, "%s.apk" % product['name'])
appinfo = eideticker.get_fennec_appinfo(product_fname)
appname = appinfo['appname']
print "Using application name '%s' from apk '%s'" % (appname, product_fname)
capture_name = "%s %s" % (product['name'], appinfo['date'])
else:
appinfo = { }
appname = product['appname']
capture_name = "%s (taken on %s)" % (product['name'], current_date)
if appinfo.get('appname'):
appname = appinfo['appname']
else:
appname = product['appname']
# Run the test the specified number of times
for i in range(num_runs):
# Now run the test
runtest(device, product, current_date, appname, appinfo, test,
capture_name + " #%s" % i, outputdir, datafile, data,
enable_profiling=options.enable_profiling, **devicePrefs)
# Kill app after test complete
device.killProcess(appname)
def main(args=sys.argv[1:]):
usage = "usage: %prog [options] TEST..."
parser = eideticker.TestOptionParser(usage=usage)
eideticker.add_dashboard_options(parser)
parser.add_option("--enable-profiling",
action="store_true",
dest="enable_profiling",
help="Create SPS profile to go along with capture")
parser.add_option("--dashboard-id",
action="store",
dest="dashboard_id",
help="id of dashboard (used in output json)",
default=os.environ.get('DASHBOARD_ID'))
parser.add_option("--dashboard-name",
action="store",
dest="dashboard_name",
help="name of dashboard to display",
default=os.environ.get('DASHBOARD_NAME'))
parser.add_option("--device-id",
action="store",
dest="device_id",
help="id of device (used in output json)",
default=os.environ.get('DEVICE_ID'))
parser.add_option("--branch",
action="store",
dest="branch_id",
help="branch under test (used in output json)",
default=os.environ.get('BRANCH'))
parser.add_option("--device-name",
action="store",
dest="device_name",
help="name of device to display in dashboard (if not "
"specified, display model name)",
default=os.environ.get('DEVICE_NAME'))
parser.add_option("--apk",
action="store",
dest="apk",
help="Product apk to get metadata from "
"(Android-specific)")
parser.add_option("--baseline",
action="store_true",
dest="baseline",
help="Create baseline results for dashboard")
parser.add_option("--num-runs",
action="store",
type="int",
dest="num_runs",
help="number of runs (default: %default)",
default=1)
parser.add_option("--app-version",
action="store",
dest="app_version",
help="Specify app version (if not automatically "
"available; Android-specific)")
parser.add_option("--sources-xml",
action="store",
dest="sources_xml",
help="Path to sources XML file for getting revision "
"information (B2G-specific)")
parser.add_option("--product",
action="store",
type="string",
dest="product_name",
default="nightly",
help="product name (android-specific, default: "
"%default)")
options, args = parser.parse_args()
if not args: # need to specify at least one test to run!
parser.print_usage()
sys.exit(1)
if not options.dashboard_id:
parser.error("Must specify dashboard id (either with --dashboard-id "
"or with DASHBOARD_ID environment variable)")
if not options.dashboard_name:
parser.error("Must specify dashboard name (either with "
"--dashboard-name or with DASHBOARD_NAME environment "
"varaiable)")
if not options.device_id:
parser.error("Must specify device id (either with --device-id or with "
"DEVICE_ID environment variable)")
if not options.branch_id:
parser.error("Must specify branch (either with --branch or with "
"BRANCH environment variable)")
# get device info
device_prefs = eideticker.getDevicePrefs(options)
device = eideticker.getDevice(**device_prefs)
device_name = options.device_name
if not device_name:
device_name = device.model
# copy dashboard files to output directory (if applicable)
eideticker.copy_dashboard_files(options.dashboard_dir)
if options.devicetype == 'android':
product = eideticker.get_product(options.product_name)
device_info = {
'name': device_name,
'version': device.getprop('ro.build.version.release')
}
elif options.devicetype == 'b2g':
product = eideticker.get_product('b2g-nightly')
device_info = {'name': device_name}
else:
print "ERROR: Unknown device type '%s'" % options.devicetype
# update dashboard / device index
eideticker.update_dashboard_list(options.dashboard_dir,
options.dashboard_id,
options.dashboard_name)
eideticker.update_dashboard_device_list(options.dashboard_dir,
options.dashboard_id,
options.device_id,
options.branch_id, device_info)
# get application/build info
if options.devicetype == "android":
if options.apk:
if options.app_version:
raise Exception("Should specify either --app-version or "
"--apk, not both!")
appinfo = eideticker.get_fennec_appinfo(options.apk)
options.appname = appinfo['appname']
print "Using application name '%s' from apk '%s'" % (
options.appname, options.apk)
options.capture_name = "%s %s" % (product['name'],
appinfo['appdate'])
else:
if not options.app_version:
raise Exception("Should specify --app-version if not --apk!")
# no apk, assume it's something static on the device
appinfo = {
'appdate': time.strftime("%Y-%m-%d"),
'version': options.app_version
}
elif options.devicetype == "b2g":
if not options.sources_xml:
raise Exception("Must specify --sources-xml on b2g!")
appinicontents = device.pullFile('/system/b2g/application.ini')
sfh = StringIO.StringIO(appinicontents)
appinfo = eideticker.get_appinfo(sfh)
appinfo.update(get_revision_data(options.sources_xml))
options.appname = None
else:
print "Unknown device type '%s'!" % options.devicetype
# run through the tests...
failed_tests = []
for testkey in args:
testinfo = eideticker.get_testinfo(testkey)
eideticker.update_dashboard_test_list(options.dashboard_dir,
options.dashboard_id,
options.device_id,
options.branch_id, testinfo)
current_date = time.strftime("%Y-%m-%d")
options.capture_name = "%s - %s (taken on %s)" % (
testkey, product['name'], current_date)
if options.prepare_test:
eideticker.prepare_test(testkey, options)
# Run the test the specified number of times
for i in range(options.num_runs):
try:
runtest(device, device_prefs, options, product, appinfo,
testinfo, options.capture_name + " #%s" % i)
except eideticker.TestException:
print "Unable to run test '%s'. Skipping and continuing." % testkey
failed_tests.append(testkey)
break
# synchronize with dashboard (if we have a server to upload to)
if options.dashboard_server:
eideticker.upload_dashboard(options)
else:
print "No dashboard server specified. Skipping upload."
if failed_tests:
print "The following tests failed: %s" % ", ".join(failed_tests)
sys.exit(1)
def main(args=sys.argv[1:]):
usage = "usage: %prog [options] <product> <test>"
parser = eideticker.TestOptionParser(usage=usage)
parser.add_option("--enable-profiling",
action="store_true", dest="enable_profiling",
help="Create SPS profile to go along with capture")
parser.add_option("--device-id", action="store", dest="device_id",
help="id of device (used in output json)",
default=os.environ.get('DEVICE_ID'))
parser.add_option("--device-name", action="store", dest="device_name",
help="name of device to display in dashboard (if not "
"specified, display model name)",
default=os.environ.get('DEVICE_NAME'))
parser.add_option("--apk", action="store", dest="apk",
help="Product apk to get metadata from "
"(Android-specific)")
parser.add_option("--baseline", action="store_true", dest="baseline",
help="Create baseline results for dashboard")
parser.add_option("--num-runs", action="store",
type="int", dest="num_runs",
help="number of runs (default: 1)")
parser.add_option("--app-version", action="store", dest="app_version",
help="Specify app version (if not automatically "
"available; Android-specific)")
parser.add_option("--sources-xml", action="store", dest="sources_xml",
help="Path to sources XML file for getting revision "
"information (B2G-specific)")
parser.add_option("--output-dir", action="store",
type="string", dest="outputdir", default=eideticker.DASHBOARD_DIR,
help="output results to directory instead of src/dashboard")
options, args = parser.parse_args()
if len(args) != 2:
parser.print_usage()
sys.exit(1)
(productname, testkey) = args
num_runs = 1
if options.num_runs:
num_runs = options.num_runs
testinfo = eideticker.get_testinfo(testkey)
device_id = options.device_id
if not device_id:
print "ERROR: Must specify device id (either with --device-id or with "
"DEVICE_ID environment variable)"
sys.exit(1)
# we'll log http requests for webstartup tests only
log_http_requests = False
if testinfo['type'] == 'webstartup':
log_http_requests = True
# likewise, log actions only for web tests and b2g tests
log_actions = False
if testinfo['type'] == 'web' or testinfo['type'] == 'b2g':
log_actions = True
product = eideticker.get_product(productname)
current_date = time.strftime("%Y-%m-%d")
capture_name = "%s - %s (taken on %s)" % (testkey, product['name'],
current_date)
datafile = os.path.join(options.outputdir, device_id, '%s.json' % testkey)
data = NestedDict()
if os.path.isfile(datafile):
data.update(json.loads(open(datafile).read()))
device_prefs = eideticker.getDevicePrefs(options)
device = eideticker.getDevice(**device_prefs)
devices = {}
devicefile = os.path.join(options.outputdir, 'devices.json')
if os.path.isfile(devicefile):
devices = json.loads(open(devicefile).read())['devices']
testfile = os.path.join(options.outputdir, '%s' % device_id, 'tests.json')
if os.path.isfile(testfile):
tests = json.loads(open(testfile).read())['tests']
else:
tests = {}
tests[testkey] = {'shortDesc': testinfo['shortDesc'],
'defaultMeasureId': testinfo['defaultMeasure']}
device_name = options.device_name
if not device_name:
device_name = device.model
if options.devicetype == "android":
devices[device_id] = {
'name': device_name,
'version': device.getprop('ro.build.version.release')}
if options.apk:
if options.app_version:
raise Exception("Should specify either --app-version or "
"--apk, not both!")
appinfo = eideticker.get_fennec_appinfo(options.apk)
appname = appinfo['appname']
print "Using application name '%s' from apk '%s'" % (
appname, options.apk)
capture_name = "%s %s" % (product['name'], appinfo['appdate'])
else:
if not options.app_version:
raise Exception("Should specify --app-version if not --apk!")
# no apk, assume it's something static on the device
appinfo = {
'appdate': time.strftime("%Y-%m-%d"),
'version': options.app_version}
appname = product['appname']
elif options.devicetype == "b2g":
if not options.sources_xml:
raise Exception("Must specify --sources-xml on b2g!")
devices[device_id] = {'name': device_name}
appinicontents = device.pullFile('/system/b2g/application.ini')
sfh = StringIO.StringIO(appinicontents)
appinfo = eideticker.get_appinfo(sfh)
appinfo.update(get_revision_data(options.sources_xml))
appname = None
else:
print "Unknown device type '%s'!" % options.devicetype
# copy dashboard files to output directory (if applicable)
eideticker.copy_dashboard_files(options.outputdir)
# update the device / test list for the dashboard
with open(devicefile, 'w') as f:
f.write(json.dumps({'devices': devices}))
testfiledir = os.path.dirname(testfile)
if not os.path.exists(testfiledir):
os.mkdir(testfiledir)
with open(testfile, 'w') as f:
f.write(json.dumps({'tests': tests}))
if options.prepare_test:
eideticker.prepare_test(
testkey, device_prefs, options.wifi_settings_file)
# Run the test the specified number of times
for i in range(num_runs):
runtest(device, device_prefs, options,
product, appname, appinfo, testinfo,
capture_name + " #%s" % i, datafile, data,
log_http_requests=log_http_requests,
log_actions=log_actions)
def runtest(device_prefs, capture_device, outputdir, outputfile, testname, url_params, num_runs,
startup_test, no_capture, get_internal_checkerboard_stats,
apk=None, appname = None, appdate = None, enable_profiling=False,
extra_prefs={}, extra_env_vars={}):
device = None
if apk:
appinfo = eideticker.get_fennec_appinfo(apk)
appname = appinfo['appname']
print "Installing %s (version: %s, revision %s)" % (appinfo['appname'],
appinfo['version'],
appinfo['revision'])
device = eideticker.getDevice(**device_prefs)
device.updateApp(apk)
else:
appinfo = None
captures = []
for i in range(num_runs):
# Kill any existing instances of the processes (for Android)
if device:
device.killProcess(appname)
# Now run the test
curtime = int(time.time())
capture_file = os.path.join(CAPTURE_DIR,
"metric-test-%s-%s.zip" % (appname,
curtime))
if enable_profiling:
profile_file = os.path.join(PROFILE_DIR,
"profile-%s-%s.zip" % (appname, curtime))
else:
profile_file = None
if get_internal_checkerboard_stats:
checkerboard_log_file = tempfile.NamedTemporaryFile()
else:
checkerboard_log_file = None
current_date = time.strftime("%Y-%m-%d")
capture_name = "%s - %s (taken on %s)" % (testname, appname, current_date)
eideticker.run_test(testname, capture_device,
appname, capture_name, device_prefs,
extra_prefs=extra_prefs,
extra_env_vars=extra_env_vars,
checkerboard_log_file=checkerboard_log_file,
profile_file=profile_file,
no_capture=no_capture,
capture_file=capture_file)
capture_result = {}
if not no_capture:
capture_result['file'] = capture_file
capture = videocapture.Capture(capture_file)
if startup_test:
capture_result['stableframe'] = videocapture.get_stable_frame(capture)
else:
capture_result['uniqueframes'] = videocapture.get_num_unique_frames(capture)
capture_result['fps'] = videocapture.get_fps(capture)
capture_result['checkerboard'] = videocapture.get_checkerboarding_area_duration(capture)
if outputdir:
video_path = os.path.join('videos', 'video-%s.webm' % time.time())
video_file = os.path.join(outputdir, video_path)
open(video_file, 'w').write(capture.get_video().read())
capture_result['video'] = video_path
if enable_profiling:
capture_result['profile'] = profile_file
if get_internal_checkerboard_stats:
internal_checkerboard_totals = parse_checkerboard_log(checkerboard_log_file.name)
capture_result['internalcheckerboard'] = internal_checkerboard_totals
captures.append(capture_result)
appkey = appname
if appdate:
appkey = appdate.isoformat()
else:
appkey = appname
if appinfo and appinfo.get('revision'):
display_key = "%s (%s)" % (appkey, appinfo['revision'])
else:
display_key = appkey
print "=== Results for %s ===" % display_key
if not no_capture:
if startup_test:
print " First stable frames:"
print " %s" % map(lambda c: c['stableframe'], captures)
print
else:
print " Number of unique frames:"
print " %s" % map(lambda c: c['uniqueframes'], captures)
print
print " Average number of unique frames per second:"
print " %s" % map(lambda c: c['fps'], captures)
print
print " Checkerboard area/duration (sum of percents NOT percentage):"
print " %s" % map(lambda c: c['checkerboard'], captures)
print
print " Capture files:"
print " Capture files: %s" % map(lambda c: c['file'], captures)
print
if enable_profiling:
print " Profile files:"
print " Profile files: %s" % map(lambda c: c['profile'], captures)
print
if get_internal_checkerboard_stats:
print " Internal Checkerboard Stats (sum of percents, not percentage):"
print " %s" % map(lambda c: c['internalcheckerboard'], captures)
print
if outputfile:
resultdict = { 'title': testname, 'data': {} }
if os.path.isfile(outputfile):
resultdict.update(json.loads(open(outputfile).read()))
if not resultdict['data'].get(appkey):
resultdict['data'][appkey] = []
resultdict['data'][appkey].extend(captures)
with open(outputfile, 'w') as f:
f.write(json.dumps(resultdict))