def main(args=sys.argv[1:]):
parser = eideticker.OptionParser(usage="usage: %prog")
options, args = parser.parse_args()
# Create a device object to interface with the phone
devicePrefs = eideticker.getDevicePrefs(options)
device = eideticker.getDevice(**devicePrefs)
print "READY"
sys.stdout.flush()
while 1:
try:
line = sys.stdin.readline()
except KeyboardInterrupt:
break
if not line:
break
tokens = line.rstrip().split()
if len(tokens) < 1:
raise Exception("No command")
(cmd, args) = (tokens[0], tokens[1:])
device.executeCommand(cmd, args)
def main(args=sys.argv[1:]):
usage = "usage: %prog <product> <date>"
parser = eideticker.OptionParser(usage=usage)
options, args = parser.parse_args()
if len(args) != 2:
parser.error("incorrect number of arguments")
(productname, datestr) = args
product = eideticker.get_product(productname)
devicePrefs = eideticker.getDevicePrefs(options)
device = eideticker.getDevice(**devicePrefs)
if device.type != 'android':
print "Device type '%s' does not currently support updates" % \
device.type
sys.exit(0)
if not product.get('reponame'):
print "No download / installation needed for %s" % product['name']
else:
print "Downloading %s (date: %s)" % (product['name'], datestr)
br = eideticker.BuildRetriever()
if datestr == 'latest':
date = None
else:
date = eideticker.BuildRetriever.get_date(datestr)
filename = br.get_build(product, date)
print "Installing %s (%s)" % (product['name'], filename)
device.updateApp(filename)
def main(args=sys.argv[1:]):
usage = "usage: %prog <product> <date>"
parser = eideticker.OptionParser(usage=usage)
options, args = parser.parse_args()
if len(args) != 2:
parser.error("incorrect number of arguments")
(productname, datestr) = args
product = eideticker.get_product(productname)
devicePrefs = eideticker.getDevicePrefs(options)
device = eideticker.getDevice(**devicePrefs)
if device.type != 'android':
print "Device type '%s' does not currently support updates" % device.type
sys.exit(0)
if not product.get('reponame'):
print "No download / installation needed for %s" % product['name']
else:
print "Downloading %s" % product['name']
br = eideticker.BuildRetriever()
if datestr == 'latest':
date = None
else:
date = eideticker.BuildRetriever.get_date(datestr)
filename = br.get_build(product, date)
print "Installing %s (%s)" % (product['name'], filename)
device.updateApp(filename)
def main(args=sys.argv[1:]):
parser = eideticker.OptionParser(usage="usage: %prog")
options, args = parser.parse_args()
# Create a device object to interface with the phone
devicePrefs = eideticker.getDevicePrefs(options)
device = eideticker.getDevice(**devicePrefs)
print "READY"
sys.stdout.flush()
while 1:
try:
line = sys.stdin.readline()
except KeyboardInterrupt:
break
if not line:
break
tokens = line.rstrip().split()
if len(tokens) < 1:
raise Exception("No command")
(cmd, args) = (tokens[0], tokens[1:])
device.executeCommand(cmd, args)
def main(args=sys.argv[1:]):
usage = "usage: %prog [options] <test key>"
parser = eideticker.TestOptionParser(usage=usage)
parser.add_option("--url-params", action="store",
dest="url_params",
help="additional url parameters for test")
parser.add_option("--name", action="store",
type="string", dest="capture_name",
help="name to give capture")
parser.add_option("--capture-file", action="store",
type="string", dest="capture_file",
help="name to give to capture file")
parser.add_option("--no-capture", action="store_true",
dest="no_capture",
help="run through the test, but don't actually "
"capture anything")
parser.add_option("--app-name", action="store",
type="string", dest="appname",
help="Specify an application name (android only)")
parser.add_option("--test-type", action="store", type="string",
dest="test_type", help="override test type")
parser.add_option("--profile-file", action="store",
type="string", dest="profile_file",
help="Collect a performance profile using the built in "
"profiler (fennec only).")
parser.add_option("--request-log-file", action="store",
type="string", dest="request_log_file",
help="Collect a log of HTTP requests during test")
parser.add_option("--actions-log-file", action="store",
type="string", dest="actions_log_file",
help="Collect a log of actions requests during test")
options, args = parser.parse_args()
if len(args) != 1:
parser.error("You must specify (only) a test key")
sys.exit(1)
testkey = args[0]
device_prefs = eideticker.getDevicePrefs(options)
if options.prepare_test:
eideticker.prepare_test(testkey, device_prefs)
testlog = eideticker.run_test(testkey, options.capture_device,
options.appname,
options.capture_name, device_prefs,
extra_prefs=options.extra_prefs,
extra_env_vars=options.extra_env_vars,
test_type=options.test_type,
profile_file=options.profile_file,
no_capture=options.no_capture,
capture_area=options.capture_area,
capture_file=options.capture_file,
wifi_settings_file=options.wifi_settings_file,
sync_time=options.sync_time)
# save logs if applicable
testlog.save_logs(http_request_log_path=options.request_log_file,
actions_log_path=options.actions_log_file)
def main(args=sys.argv[1:]):
usage = "usage: %prog"
parser = eideticker.OptionParser(usage=usage)
options, args = parser.parse_args()
devicePrefs = eideticker.getDevicePrefs(options)
device = eideticker.getDevice(**devicePrefs)
if device.type == 'b2g':
print "No cleanup presently required on b2g"
return
for product in eideticker.products:
if product['platform'] == device.type:
device.killProcess(product['appname'])
# clean up any test stuff (profiles, etc.)
device.removeDir(device.getDeviceRoot())
# profile files are dumped directly to the sdcard,
# not the testroot so we need to clean them up
# seperately
files = device.listFiles('/mnt/sdcard/')
for file in files:
if re.match('profile_.*txt', file):
device.removeFile('/mnt/sdcard/%s' % file)
def main(args=sys.argv[1:]):
usage = "usage: %prog"
parser = eideticker.OptionParser(usage=usage)
options, args = parser.parse_args()
devicePrefs = eideticker.getDevicePrefs(options)
device = eideticker.getDevice(**devicePrefs)
if device.type == 'b2g':
print "No cleanup presently required on b2g"
return
for product in eideticker.products:
if product['platform'] == device.type:
device.killProcess(product['appname'])
# clean up any test stuff (profiles, etc.)
device.removeDir(device.getDeviceRoot())
# profile files are dumped directly to the sdcard,
# not the testroot so we need to clean them up
# seperately
files = device.listFiles('/mnt/sdcard/')
for file in files:
if re.match('profile_.*txt', file):
device.removeFile('/mnt/sdcard/%s' % file)
def main(args=sys.argv[1:]):
usage = "usage: %prog [product1] [product2]"
parser = eideticker.OptionParser(usage=usage)
options, args = parser.parse_args()
if len(args):
products = [
product for product in default_products if product['name'] in args
]
else:
products = default_products
if not products:
print "No products matching arguments!"
sys.exit(1)
print products
devicePrefs = eideticker.getDevicePrefs(options)
device = eideticker.getDevice(**devicePrefs)
for product in products:
if not product.get('url'):
print "No download / installation needed for %s" % product['name']
else:
print "Downloading %s" % product['name']
product_fname = os.path.join(DOWNLOAD_DIR,
"%s.apk" % product['name'])
dl = urllib2.urlopen(product['url'])
with open(product_fname, 'w') as f:
f.write(dl.read())
print "Installing %s" % product['name']
device.updateApp(product_fname)
def main(args=sys.argv[1:]):
usage = "usage: %prog [product1] [product2]"
parser = eideticker.OptionParser(usage=usage)
options, args = parser.parse_args()
if len(args):
products = [product for product in default_products if product['name'] in args]
else:
products = default_products
if not products:
print "No products matching arguments!"
sys.exit(1)
print products
devicePrefs = eideticker.getDevicePrefs(options)
device = eideticker.getDevice(**devicePrefs)
for product in products:
if not product.get('url'):
print "No download / installation needed for %s" % product['name']
else:
print "Downloading %s" % product['name']
product_fname = os.path.join(DOWNLOAD_DIR, "%s.apk" % product['name'])
dl = urllib2.urlopen(product['url'])
with open(product_fname, 'w') as f:
f.write(dl.read())
print "Installing %s" % product['name']
device.updateApp(product_fname)
def main(args=sys.argv[1:]):
usage = "usage: %prog"
parser = eideticker.OptionParser(usage=usage)
options, args = parser.parse_args()
devicePrefs = eideticker.getDevicePrefs(options)
device = eideticker.getDevice(**devicePrefs)
device.cleanup()
def main(args=sys.argv[1:]):
usage = "usage: %prog"
parser = eideticker.OptionParser(usage=usage)
options, args = parser.parse_args()
devicePrefs = eideticker.getDevicePrefs(options)
device = eideticker.getDevice(**devicePrefs)
device.cleanup()
def run_capture(options, capture_file):
device_prefs = eideticker.getDevicePrefs(options)
capture_server = CaptureServer(capture_file,
options.capture_device,
options.mode,
capture=options.capture)
host = moznetwork.get_ip()
docroot = eideticker.runtest.TEST_DIR
httpd = mozhttpd.MozHttpd(port=0,
host=host,
docroot=docroot,
urlhandlers=[{
'method':
'GET',
'path':
'/api/captures/start/?',
'function':
capture_server.start_capture
}, {
'method':
'GET',
'path':
'/api/captures/end/?',
'function':
capture_server.end_capture
}])
httpd.start(block=False)
print "Serving '%s' at %s:%s" % (httpd.docroot, httpd.host,
httpd.httpd.server_port)
device = eideticker.getDevice(**device_prefs)
mode = options.mode
if not mode:
mode = device.hdmiResolution
url = "http://%s:%s/getdimensions.html" % (host, httpd.httpd.server_port)
device.executeCommand("tap", [100, 100])
if device_prefs['devicetype'] == 'android':
device.launchFennec(options.appname, url=url)
else:
if not options.wifi_settings_file:
print "WIFI settings file (see --help) required for B2G!"
sys.exit(1)
device.restartB2G()
device.connectWIFI(json.loads(open(options.wifi_settings_file).read()))
device.marionette.execute_script("window.location.href='%s';" % url)
while not capture_server.finished:
time.sleep(0.25)
capture_server.convert_capture()
httpd.stop()
def run_capture(options, capture_file):
device_prefs = eideticker.getDevicePrefs(options)
capture_server = CaptureServer(capture_file, options.capture_device,
options.mode,
no_capture=options.no_capture)
host = moznetwork.get_ip()
docroot = eideticker.runtest.TEST_DIR
httpd = mozhttpd.MozHttpd(port=0, host=host, docroot=docroot, urlhandlers=[
{'method': 'GET',
'path': '/api/captures/start/?',
'function': capture_server.start_capture},
{'method': 'GET',
'path': '/api/captures/end/?',
'function': capture_server.end_capture}])
httpd.start(block=False)
print "Serving '%s' at %s:%s" % (
httpd.docroot, httpd.host, httpd.httpd.server_port)
device = eideticker.getDevice(**device_prefs)
mode = options.mode
if not mode:
mode = device.hdmiResolution
url = "http://%s:%s/getdimensions.html" % (host, httpd.httpd.server_port)
device.executeCommand("tap", [100, 100])
if device_prefs['devicetype'] == 'android':
device.launchFennec(options.appname, url=url)
else:
if not options.wifi_settings_file:
print "WIFI settings file (see --help) required for B2G!"
sys.exit(1)
device.restartB2G()
device.connectWIFI(json.loads(open(options.wifi_settings_file).read()))
session = device.marionette.session
if 'b2g' not in session:
raise Exception("bad session value %s returned by start_session" %
session)
device.unlock()
# wait for device to become ready (yes, this is terrible, can we
# detect this condition in marionette somehow?)
time.sleep(5)
device.marionette.execute_script("window.location.href='%s';" % url)
while not capture_server.finished:
time.sleep(0.25)
capture_server.convert_capture()
device.killProcess(options.appname)
httpd.stop()
def main(args=sys.argv[1:]):
usage = "usage: %prog"
parser = eideticker.OptionParser(usage=usage)
options, args = parser.parse_args()
devicePrefs = eideticker.getDevicePrefs(options)
device = eideticker.getDevice(**devicePrefs)
for product in default_products:
device.killProcess(product['appname'])
# clean up any test stuff (profiles, etc.)
device.removeDir(device.getDeviceRoot())
def main(args=sys.argv[1:]):
usage = "usage: %prog"
parser = eideticker.OptionParser(usage=usage)
options, args = parser.parse_args()
devicePrefs = eideticker.getDevicePrefs(options)
device = eideticker.getDevice(**devicePrefs)
for product in default_products:
device.killProcess(product['appname'])
# clean up any test stuff (profiles, etc.)
device.removeDir(device.getDeviceRoot())
def main(args=sys.argv[1:]):
usage = "usage: %prog"
parser = eideticker.OptionParser(usage=usage)
options, args = parser.parse_args()
devicePrefs = eideticker.getDevicePrefs(options)
device = eideticker.getDevice(**devicePrefs)
if device.type == 'b2g':
print "No cleanup presently required on b2g"
return
for product in eideticker.products:
if product['platform'] == device.type:
device.killProcess(product['appname'])
# clean up any test stuff (profiles, etc.)
device.removeDir(device.getDeviceRoot())
def run_capture(options, capture_filename):
device_prefs = eideticker.getDevicePrefs(options)
capture_server = CaptureServer(capture_filename, options)
host = moznetwork.get_ip()
docroot = eideticker.test.TEST_DIR
httpd = mozhttpd.MozHttpd(port=0, host=host, docroot=docroot, urlhandlers=[
{'method': 'GET',
'path': '/api/captures/start/?',
'function': capture_server.start_capture},
{'method': 'GET',
'path': '/api/captures/end/?',
'function': capture_server.end_capture}])
httpd.start(block=False)
print "Serving '%s' at %s:%s" % (
httpd.docroot, httpd.host, httpd.httpd.server_port)
device = eideticker.getDevice(**device_prefs)
url = "http://%s:%s/getdimensions.html" % (host, httpd.httpd.server_port)
device.executeCommand("tap", [100, 100])
if device_prefs['devicetype'] == 'android':
device.launchFennec(options.appname, url=url)
else:
if not options.wifi_settings_file:
print "WIFI settings file (see --help) required for B2G!"
sys.exit(1)
device.restartB2G()
device.connectWIFI(json.loads(open(options.wifi_settings_file).read()))
device.marionette.execute_script("window.location.href='%s';" % url)
while not capture_server.finished:
time.sleep(0.25)
capture_server.convert_capture()
httpd.stop()
def main(args=sys.argv[1:]):
usage = "usage: %prog <test> [appname1] [appname2] ..."
parser = eideticker.TestOptionParser(usage=usage)
parser.add_option("--num-runs", action="store",
type="int", dest="num_runs",
default=1,
help="number of runs (default: 1)")
parser.add_option("--output-dir", action="store",
type="string", dest="outputdir",
help="output results to web site")
parser.add_option("--enable-profiling", action="store_true",
dest="enable_profiling",
help="Collect performance profiles using the built in "
"profiler.")
parser.add_option("--url-params", action="store",
dest="url_params", default="",
help="additional url parameters for test")
parser.add_option("--use-apks", action="store_true", dest="use_apks",
help="use and install android APKs as part of test "
"(instead of specifying appnames)")
parser.add_option("--date", action="store", dest="date",
metavar="YYYY-MM-DD",
help="get and test nightly build for date")
parser.add_option("--start-date", action="store", dest="start_date",
metavar="YYYY-MM-DD",
help="start date for range of nightlies to test")
parser.add_option("--end-date", action="store", dest="end_date",
metavar="YYYY-MM-DD",
help="end date for range of nightlies to test")
options, args = parser.parse_args()
if len(args) == 0:
parser.error("Must specify at least one argument: the test")
if options.enable_profiling and not options.outputdir:
parser.error("Must specify output directory if profiling enabled")
dates = []
appnames = []
apks = []
if options.start_date and options.end_date and len(args) == 1:
testname = args[0]
start_date = eideticker.BuildRetriever.get_date(options.start_date)
end_date = eideticker.BuildRetriever.get_date(options.end_date)
days = (end_date - start_date).days
for numdays in range(days + 1):
dates.append(start_date + datetime.timedelta(days=numdays))
elif options.date and len(args) == 1:
testname = args[0]
dates = [eideticker.BuildRetriever.get_date(options.date)]
elif not options.date and len(args) >= 2:
testname = args[0]
if options.use_apks:
apks = args[1:]
else:
appnames = args[1:]
elif options.devicetype == "b2g":
testname = args[0]
elif not options.date or (not options.start_date and not options.end_date):
parser.error("On Android, must specify date, date range, a set of "
"appnames (e.g. org.mozilla.fennec) or a set of apks (if "
"--use-apks is specified)")
device_prefs = eideticker.getDevicePrefs(options)
if options.outputdir:
eideticker.copy_dashboard_files(options.outputdir,
indexfile='metric.html')
if options.devicetype == "b2g":
runtest(device_prefs, testname, options)
elif appnames:
for appname in appnames:
runtest(device_prefs, testname, options, appname=appname)
elif apks:
for apk in apks:
runtest(device_prefs, testname, options, apk=apk)
else:
br = eideticker.BuildRetriever()
productname = "nightly"
product = eideticker.get_product(productname)
for date in dates:
apk = br.get_build(product, date)
runtest(device_prefs, testname, options, apk=apk, appdate=date)
def main(args=sys.argv[1:]):
usage = "usage: %prog <test> [appname1] [appname2] ..."
parser = eideticker.TestOptionParser(usage=usage)
parser.add_option("--num-runs",
action="store",
type="int",
dest="num_runs",
default=1,
help="number of runs (default: 1)")
parser.add_option("--output-dir",
action="store",
type="string",
dest="outputdir",
help="output results to web site")
parser.add_option("--enable-profiling",
action="store_true",
dest="enable_profiling",
help="Collect performance profiles using the built in "
"profiler.")
parser.add_option(
"--get-internal-checkerboard-stats",
action="store_true",
dest="get_internal_checkerboard_stats",
help="get and calculate internal checkerboard stats (Android only)")
parser.add_option("--url-params",
action="store",
dest="url_params",
default="",
help="additional url parameters for test")
parser.add_option("--use-apks",
action="store_true",
dest="use_apks",
help="use and install android APKs as part of test "
"(instead of specifying appnames)")
parser.add_option("--date",
action="store",
dest="date",
metavar="YYYY-MM-DD",
help="get and test nightly build for date")
parser.add_option("--start-date",
action="store",
dest="start_date",
metavar="YYYY-MM-DD",
help="start date for range of nightlies to test")
parser.add_option("--end-date",
action="store",
dest="end_date",
metavar="YYYY-MM-DD",
help="end date for range of nightlies to test")
options, args = parser.parse_args()
if len(args) == 0:
parser.error("Must specify at least one argument: the test")
if options.enable_profiling and not options.outputdir:
parser.error("Must specify output directory if profiling enabled")
dates = []
appnames = []
apks = []
if options.start_date and options.end_date and len(args) == 1:
testname = args[0]
start_date = eideticker.BuildRetriever.get_date(options.start_date)
end_date = eideticker.BuildRetriever.get_date(options.end_date)
days = (end_date - start_date).days
for numdays in range(days + 1):
dates.append(start_date + datetime.timedelta(days=numdays))
elif options.date and len(args) == 1:
testname = args[0]
dates = [eideticker.BuildRetriever.get_date(options.date)]
elif not options.date and len(args) >= 2:
testname = args[0]
if options.use_apks:
apks = args[1:]
else:
appnames = args[1:]
elif options.devicetype == "b2g":
testname = args[0]
elif not options.date or (not options.start_date and not options.end_date):
parser.error("On Android, must specify date, date range, a set of "
"appnames (e.g. org.mozilla.fennec) or a set of apks (if "
"--use-apks is specified)")
device_prefs = eideticker.getDevicePrefs(options)
if options.outputdir:
eideticker.copy_dashboard_files(options.outputdir,
indexfile='metric.html')
if options.devicetype == "b2g":
runtest(device_prefs, testname, options)
elif appnames:
for appname in appnames:
runtest(device_prefs, testname, options, appname=appname)
elif apks:
for apk in apks:
runtest(device_prefs, testname, options, apk=apk)
else:
br = eideticker.BuildRetriever()
productname = "nightly"
product = eideticker.get_product(productname)
for date in dates:
apk = br.get_build(product, date)
runtest(device_prefs, testname, options, apk=apk, appdate=date)
def main(args=sys.argv[1:]):
usage = "usage: %prog [options] <product> <test> <output dir>"
parser = eideticker.TestOptionParser(usage=usage)
parser.add_option("--enable-profiling",
action="store_true",
dest="enable_profiling",
help="Create SPS profile to go along with capture")
parser.add_option("--device-id",
action="store",
dest="device_id",
help="id of device (used in output json)",
default=os.environ.get('DEVICE_ID'))
parser.add_option("--device-name",
action="store",
dest="device_name",
help="name of device to display in dashboard (if not "
"specified, display model name)",
default=os.environ.get('DEVICE_NAME'))
parser.add_option("--apk",
action="store",
dest="apk",
help="Product apk to get metadata from "
"(Android-specific)")
parser.add_option("--baseline",
action="store_true",
dest="baseline",
help="Create baseline results for dashboard")
parser.add_option("--num-runs",
action="store",
type="int",
dest="num_runs",
help="number of runs (default: 1)")
parser.add_option("--app-version",
action="store",
dest="app_version",
help="Specify app version (if not automatically "
"available; Android-specific)")
parser.add_option("--sources-xml",
action="store",
dest="sources_xml",
help="Path to sources XML file for getting revision "
"information (B2G-specific)")
options, args = parser.parse_args()
if len(args) != 3:
parser.print_usage()
sys.exit(1)
(productname, testkey, outputdir) = args
num_runs = 1
if options.num_runs:
num_runs = options.num_runs
testinfo = eideticker.get_testinfo(testkey)
device_id = options.device_id
if not device_id:
print "ERROR: Must specify device id (either with --device-id or with "
"DEVICE_ID environment variable)"
sys.exit(1)
# we'll log http requests for webstartup tests only
log_http_requests = False
if testinfo['type'] == 'webstartup':
log_http_requests = True
# likewise, log actions only for web tests and b2g tests
log_actions = False
if testinfo['type'] == 'web' or testinfo['type'] == 'b2g':
log_actions = True
product = eideticker.get_product(productname)
current_date = time.strftime("%Y-%m-%d")
capture_name = "%s - %s (taken on %s)" % (testkey, product['name'],
current_date)
datafile = os.path.join(outputdir, device_id, '%s.json' % testkey)
data = NestedDict()
if os.path.isfile(datafile):
data.update(json.loads(open(datafile).read()))
device_prefs = eideticker.getDevicePrefs(options)
device = eideticker.getDevice(**device_prefs)
devices = {}
devicefile = os.path.join(outputdir, 'devices.json')
if os.path.isfile(devicefile):
devices = json.loads(open(devicefile).read())['devices']
testfile = os.path.join(outputdir, '%s' % device_id, 'tests.json')
if os.path.isfile(testfile):
tests = json.loads(open(testfile).read())['tests']
else:
tests = {}
tests[testkey] = {
'shortDesc': testinfo['shortDesc'],
'defaultMeasure': testinfo['defaultMeasure']
}
device_name = options.device_name
if not device_name:
device_name = device.model
if options.devicetype == "android":
devices[device_id] = {
'name': device_name,
'version': device.getprop('ro.build.version.release')
}
if options.apk:
if options.app_version:
raise Exception("Should specify either --app-version or "
"--apk, not both!")
appinfo = eideticker.get_fennec_appinfo(options.apk)
appname = appinfo['appname']
print "Using application name '%s' from apk '%s'" % (appname,
options.apk)
capture_name = "%s %s" % (product['name'], appinfo['appdate'])
else:
if not options.app_version:
raise Exception("Should specify --app-version if not --apk!")
# no apk, assume it's something static on the device
appinfo = {
'appdate': time.strftime("%Y-%m-%d"),
'version': options.app_version
}
appname = product['appname']
elif options.devicetype == "b2g":
if not options.sources_xml:
raise Exception("Must specify --sources-xml on b2g!")
devices[device_id] = {'name': device_name}
appinicontents = device.pullFile('/system/b2g/application.ini')
sfh = StringIO.StringIO(appinicontents)
appinfo = eideticker.get_appinfo(sfh)
appinfo.update(get_revision_data(options.sources_xml))
appname = None
else:
print "Unknown device type '%s'!" % options.devicetype
# update the device / test list for the dashboard
with open(devicefile, 'w') as f:
f.write(json.dumps({'devices': devices}))
testfiledir = os.path.dirname(testfile)
if not os.path.exists(testfiledir):
os.mkdir(testfiledir)
with open(testfile, 'w') as f:
f.write(json.dumps({'tests': tests}))
if options.prepare_test:
eideticker.prepare_test(testkey, device_prefs)
# Run the test the specified number of times
for i in range(num_runs):
runtest(device,
device_prefs,
options.capture_device,
options.capture_area,
product,
appname,
appinfo,
testinfo,
capture_name + " #%s" % i,
outputdir,
datafile,
data,
enable_profiling=options.enable_profiling,
log_http_requests=log_http_requests,
log_actions=log_actions,
baseline=options.baseline,
wifi_settings_file=options.wifi_settings_file,
sync_time=options.sync_time)
if options.devicetype == "android":
# Kill app after test complete
device.killProcess(appname)
def main(args=sys.argv[1:]):
usage = "usage: %prog [options] <product> <test> <output dir>"
parser = eideticker.CaptureOptionParser(usage=usage)
parser.add_option("--enable-profiling",
action="store_true", dest = "enable_profiling",
help = "Create SPS profile to go along with capture")
parser.add_option("--device-id", action="store", dest="device_id",
help="id of device (used in output json)")
parser.add_option("--apk", action="store", dest="apk",
help = "Product apk to get metadata from " \
"(Android-specific)")
parser.add_option("--baseline", action="store_true", dest="baseline",
help = "Create baseline results for dashboard")
parser.add_option("--num-runs", action="store",
type = "int", dest = "num_runs",
help = "number of runs (default: 1)")
parser.add_option("--app-version", action="store", dest="app_version",
help="Specify app version (if not automatically " \
"available; Android-specific)")
parser.add_option("--sources-xml", action="store", dest="sources_xml",
help="Path to sources XML file for getting revision " \
"information (B2G-specific)")
options, args = parser.parse_args()
parser.validate_options(options)
(productname, testkey, outputdir) = args
num_runs = 1
if options.num_runs:
num_runs = options.num_runs
manifest = eideticker.get_test_manifest()
# sanity check... does the test match a known test key?
testkeys = [test["key"] for test in manifest.active_tests()]
if testkey not in testkeys:
print "ERROR: No tests matching '%s' (options: %s)" % (testkey, ", ".join(testkeys))
sys.exit(1)
testinfo = [test for test in manifest.active_tests() if test['key'] == testkey][0]
device_id = options.device_id
if not device_id:
device_id = os.environ.get('DEVICE_ID')
if not device_id:
print "ERROR: Must specify device id (either with --device-id or with DEVICE_ID environment variable)"
sys.exit(1)
# we'll log http requests for webstartup tests only
log_http_requests = False
if testinfo['type'] == 'webstartup':
log_http_requests = True
product = eideticker.get_product(productname)
current_date = time.strftime("%Y-%m-%d")
capture_name = "%s (taken on %s)" % (product['name'], current_date)
datafile = os.path.join(outputdir, device_id, '%s.json' % testkey)
data = NestedDict()
if os.path.isfile(datafile):
data.update(json.loads(open(datafile).read()))
device_prefs = eideticker.getDevicePrefs(options)
device = eideticker.getDevice(**device_prefs)
devices = {}
devicefile = os.path.join(outputdir, 'devices.json')
if os.path.isfile(devicefile):
devices = json.loads(open(devicefile).read())['devices']
testfile = os.path.join(outputdir, '%s' % device_id, 'tests.json')
if os.path.isfile(testfile):
tests = json.loads(open(testfile).read())['tests']
else:
tests = {}
tests[testkey] = { 'shortDesc': testinfo['shortDesc'],
'defaultMeasure': testinfo['defaultMeasure'] }
if options.devicetype == "android":
devices[device_id] = { 'name': device.model,
'version': device.getprop('ro.build.version.release') }
if options.apk:
if options.app_version:
raise Exception("Should specify either --app-version or --apk, not both!")
appinfo = eideticker.get_fennec_appinfo(options.apk)
appname = appinfo['appname']
print "Using application name '%s' from apk '%s'" % (appname, options.apk)
capture_name = "%s %s" % (product['name'], appinfo['appdate'])
else:
if not options.app_version:
raise Exception("Should specify --app-version if not --apk!")
# no apk, assume it's something static on the device
appinfo = { 'appdate': time.strftime("%Y-%m-%d"), 'version': options.app_version }
appname = product['appname']
elif options.devicetype == "b2g":
if not options.sources_xml:
raise Exception("Must specify --sources-xml on b2g!")
devices[device_id] = { 'name': device.model }
appinicontents = device.pullFile('/system/b2g/application.ini')
sfh = StringIO.StringIO(appinicontents)
appinfo = eideticker.get_appinfo(sfh)
appinfo.update(get_revision_data(options.sources_xml))
appname = None
else:
print "Unknown device type '%s'!" % options.devicetype
# update the device / test list for the dashboard
with open(devicefile, 'w') as f:
f.write(json.dumps({ 'devices': devices }))
testfiledir = os.path.dirname(testfile)
if not os.path.exists(testfiledir):
os.mkdir(testfiledir)
with open(testfile, 'w') as f:
f.write(json.dumps({ 'tests': tests }))
capture_area = None
if options.capture_area:
# we validated this previously...
capture_area = json.loads(options.capture_area)
# Run the test the specified number of times
for i in range(num_runs):
# Now run the test
runtest(device, device_prefs, options.capture_device, capture_area,
product, appname, appinfo, testinfo,
capture_name + " #%s" % i, outputdir, datafile, data,
enable_profiling=options.enable_profiling,
log_http_requests=log_http_requests,
baseline=options.baseline)
if options.devicetype == "android":
# Kill app after test complete
device.killProcess(appname)
def main(args=sys.argv[1:]):
usage = "usage: %prog <test> [appname1] [appname2] ..."
parser = eideticker.TestOptionParser(usage=usage)
parser.add_option("--num-runs",
action="store",
type="int",
dest="num_runs",
default=1,
help="number of runs (default: 1)")
parser.add_option("--output-dir",
action="store",
type="string",
dest="outputdir",
help="output results to web site")
parser.add_option("--no-capture",
action="store_true",
dest="no_capture",
help="run through the test, but don't actually capture "
"anything")
parser.add_option("--enable-profiling",
action="store_true",
dest="enable_profiling",
help="Collect performance profiles using the built in "
"profiler.")
parser.add_option(
"--get-internal-checkerboard-stats",
action="store_true",
dest="get_internal_checkerboard_stats",
help="get and calculate internal checkerboard stats (Android only)")
parser.add_option("--url-params",
action="store",
dest="url_params",
default="",
help="additional url parameters for test")
parser.add_option("--use-apks",
action="store_true",
dest="use_apks",
help="use and install android APKs as part of test "
"(instead of specifying appnames)")
parser.add_option("--date",
action="store",
dest="date",
metavar="YYYY-MM-DD",
help="get and test nightly build for date")
parser.add_option("--start-date",
action="store",
dest="start_date",
metavar="YYYY-MM-DD",
help="start date for range of nightlies to test")
parser.add_option("--end-date",
action="store",
dest="end_date",
metavar="YYYY-MM-DD",
help="end date for range of nightlies to test")
options, args = parser.parse_args()
if len(args) == 0:
parser.error("Must specify at least one argument: the test")
if options.enable_profiling and not options.outputdir:
parser.error("Must specify output directory if profiling enabled")
dates = []
appnames = []
apks = []
if options.start_date and options.end_date and len(args) == 1:
testname = args[0]
start_date = eideticker.BuildRetriever.get_date(options.start_date)
end_date = eideticker.BuildRetriever.get_date(options.end_date)
days = (end_date - start_date).days
for numdays in range(days + 1):
dates.append(start_date + datetime.timedelta(days=numdays))
elif options.date and len(args) == 1:
testname = args[0]
dates = [eideticker.BuildRetriever.get_date(options.date)]
elif not options.date and len(args) >= 2:
testname = args[0]
if options.use_apks:
apks = args[1:]
else:
appnames = args[1:]
elif options.devicetype == "b2g":
testname = args[0]
elif not options.date or (not options.start_date and not options.end_date):
parser.error("On Android, must specify date, date range, a set of "
"appnames (e.g. org.mozilla.fennec) or a set of apks (if "
"--use-apks is specified)")
device_prefs = eideticker.getDevicePrefs(options)
if options.outputdir:
for dirname in [
options.outputdir,
os.path.join(options.outputdir, 'css'),
os.path.join(options.outputdir, 'fonts'),
os.path.join(options.outputdir, 'js'),
os.path.join(options.outputdir, 'videos'),
os.path.join(options.outputdir, 'metadata')
]:
if not os.path.exists(dirname):
os.makedirs(dirname)
for filename in [
'css/bootstrap.min.css',
'fonts/glyphicons-halflings-regular.eot',
'fonts/glyphicons-halflings-regular.svg',
'fonts/glyphicons-halflings-regular.ttf',
'fonts/glyphicons-halflings-regular.woff',
'framediff-view.html', 'js/ICanHaz.min.js', 'js/SS.min.js',
'js/bootstrap.min.js', 'js/common.js', 'js/framediff.js',
'js/jquery-1.7.1.min.js', 'js/jquery.flot.axislabels.js',
'js/jquery.flot.js', 'js/jquery.flot.stack.js', 'js/metric.js',
'metric.html'
]:
if filename == 'metric.html':
outfilename = 'index.html'
else:
outfilename = filename
shutil.copyfile(os.path.join(DASHBOARD_DIR, filename),
os.path.join(options.outputdir, outfilename))
if options.devicetype == "b2g":
runtest(device_prefs, testname, options)
elif appnames:
for appname in appnames:
runtest(device_prefs, testname, options, appname=appname)
elif apks:
for apk in apks:
runtest(device_prefs, testname, options, apk=apk)
else:
br = eideticker.BuildRetriever()
productname = "nightly"
product = eideticker.get_product(productname)
for date in dates:
apk = br.get_build(product, date)
runtest(device_prefs, testname, options, apk=apk, appdate=date)
def main(args=sys.argv[1:]):
usage = "usage: %prog [options] <product> <test>"
parser = eideticker.TestOptionParser(usage=usage)
parser.add_option("--enable-profiling",
action="store_true", dest="enable_profiling",
help="Create SPS profile to go along with capture")
parser.add_option("--device-id", action="store", dest="device_id",
help="id of device (used in output json)",
default=os.environ.get('DEVICE_ID'))
parser.add_option("--device-name", action="store", dest="device_name",
help="name of device to display in dashboard (if not "
"specified, display model name)",
default=os.environ.get('DEVICE_NAME'))
parser.add_option("--apk", action="store", dest="apk",
help="Product apk to get metadata from "
"(Android-specific)")
parser.add_option("--baseline", action="store_true", dest="baseline",
help="Create baseline results for dashboard")
parser.add_option("--num-runs", action="store",
type="int", dest="num_runs",
help="number of runs (default: 1)")
parser.add_option("--app-version", action="store", dest="app_version",
help="Specify app version (if not automatically "
"available; Android-specific)")
parser.add_option("--sources-xml", action="store", dest="sources_xml",
help="Path to sources XML file for getting revision "
"information (B2G-specific)")
parser.add_option("--output-dir", action="store",
type="string", dest="outputdir", default=eideticker.DASHBOARD_DIR,
help="output results to directory instead of src/dashboard")
options, args = parser.parse_args()
if len(args) != 2:
parser.print_usage()
sys.exit(1)
(productname, testkey) = args
num_runs = 1
if options.num_runs:
num_runs = options.num_runs
testinfo = eideticker.get_testinfo(testkey)
device_id = options.device_id
if not device_id:
print "ERROR: Must specify device id (either with --device-id or with "
"DEVICE_ID environment variable)"
sys.exit(1)
# we'll log http requests for webstartup tests only
log_http_requests = False
if testinfo['type'] == 'webstartup':
log_http_requests = True
# likewise, log actions only for web tests and b2g tests
log_actions = False
if testinfo['type'] == 'web' or testinfo['type'] == 'b2g':
log_actions = True
product = eideticker.get_product(productname)
current_date = time.strftime("%Y-%m-%d")
capture_name = "%s - %s (taken on %s)" % (testkey, product['name'],
current_date)
datafile = os.path.join(options.outputdir, device_id, '%s.json' % testkey)
data = NestedDict()
if os.path.isfile(datafile):
data.update(json.loads(open(datafile).read()))
device_prefs = eideticker.getDevicePrefs(options)
device = eideticker.getDevice(**device_prefs)
devices = {}
devicefile = os.path.join(options.outputdir, 'devices.json')
if os.path.isfile(devicefile):
devices = json.loads(open(devicefile).read())['devices']
testfile = os.path.join(options.outputdir, '%s' % device_id, 'tests.json')
if os.path.isfile(testfile):
tests = json.loads(open(testfile).read())['tests']
else:
tests = {}
tests[testkey] = {'shortDesc': testinfo['shortDesc'],
'defaultMeasureId': testinfo['defaultMeasure']}
device_name = options.device_name
if not device_name:
device_name = device.model
if options.devicetype == "android":
devices[device_id] = {
'name': device_name,
'version': device.getprop('ro.build.version.release')}
if options.apk:
if options.app_version:
raise Exception("Should specify either --app-version or "
"--apk, not both!")
appinfo = eideticker.get_fennec_appinfo(options.apk)
appname = appinfo['appname']
print "Using application name '%s' from apk '%s'" % (
appname, options.apk)
capture_name = "%s %s" % (product['name'], appinfo['appdate'])
else:
if not options.app_version:
raise Exception("Should specify --app-version if not --apk!")
# no apk, assume it's something static on the device
appinfo = {
'appdate': time.strftime("%Y-%m-%d"),
'version': options.app_version}
appname = product['appname']
elif options.devicetype == "b2g":
if not options.sources_xml:
raise Exception("Must specify --sources-xml on b2g!")
devices[device_id] = {'name': device_name}
appinicontents = device.pullFile('/system/b2g/application.ini')
sfh = StringIO.StringIO(appinicontents)
appinfo = eideticker.get_appinfo(sfh)
appinfo.update(get_revision_data(options.sources_xml))
appname = None
else:
print "Unknown device type '%s'!" % options.devicetype
# copy dashboard files to output directory (if applicable)
eideticker.copy_dashboard_files(options.outputdir)
# update the device / test list for the dashboard
with open(devicefile, 'w') as f:
f.write(json.dumps({'devices': devices}))
testfiledir = os.path.dirname(testfile)
if not os.path.exists(testfiledir):
os.mkdir(testfiledir)
with open(testfile, 'w') as f:
f.write(json.dumps({'tests': tests}))
if options.prepare_test:
eideticker.prepare_test(
testkey, device_prefs, options.wifi_settings_file)
# Run the test the specified number of times
for i in range(num_runs):
runtest(device, device_prefs, options,
product, appname, appinfo, testinfo,
capture_name + " #%s" % i, datafile, data,
log_http_requests=log_http_requests,
log_actions=log_actions)
def main(args=sys.argv[1:]):
usage = "usage: %prog [options] <test key>"
parser = eideticker.CaptureOptionParser(usage=usage)
parser.add_option("--url-params", action="store",
dest="url_params",
help="additional url parameters for test")
parser.add_option("--name", action="store",
type="string", dest="capture_name",
help="name to give capture")
parser.add_option("--capture-file", action="store",
type="string", dest="capture_file",
help="name to give to capture file")
parser.add_option("--no-capture", action="store_true",
dest="no_capture",
help="run through the test, but don't actually "
"capture anything")
parser.add_option("--app-name", action="store",
type="string", dest="appname",
help="Specify an application name (android only)")
parser.add_option("--test-type", action="store", type="string",
dest="test_type", help="override test type")
parser.add_option("--checkerboard-log-file", action="store",
type="string", dest="checkerboard_log_file",
help="name to give checkerboarding stats file (fennec "
"only)")
parser.add_option("--extra-prefs", action="store", dest="extra_prefs",
default="{}",
help="Extra profile preference for Firefox browsers. "
"Must be passed in as a JSON dictionary")
parser.add_option("--profile-file", action="store",
type="string", dest="profile_file",
help="Collect a performance profile using the built in "
"profiler (fennec only).")
parser.add_option("--request-log-file", action="store",
type="string", dest="request_log_file",
help="Collect a log of HTTP requests during tests")
parser.add_option("--extra-env-vars", action="store", dest="extra_env_vars",
default="",
help='Extra environment variables to set in '
'"VAR1=VAL1 VAR2=VAL2" format')
options, args = parser.parse_args()
parser.validate_options(options)
if len(args) != 1:
parser.error("You must specify (only) a test key")
sys.exit(1)
testkey = args[0]
try:
extra_prefs = json.loads(options.extra_prefs)
except ValueError:
parser.error("Error processing extra preferences: not valid JSON!")
raise
keyvals = options.extra_env_vars.split()
extra_env_vars = {}
for kv in keyvals:
(var, _, val) = kv.partition("=")
extra_env_vars[var] = val
capture_area = None
if options.capture_area:
# we validated this previously...
capture_area = json.loads(options.capture_area)
device_prefs = eideticker.getDevicePrefs(options)
eideticker.run_test(testkey, options.capture_device,
options.appname,
options.capture_name, device_prefs,
extra_prefs=extra_prefs,
extra_env_vars=extra_env_vars,
test_type=options.test_type,
profile_file=options.profile_file,
request_log_file=options.request_log_file,
checkerboard_log_file=options.checkerboard_log_file,
no_capture=options.no_capture,
capture_area=capture_area,
capture_file=options.capture_file)
def main(args=sys.argv[1:]):
usage = "usage: %prog <test> [appname1] [appname2] ..."
parser = eideticker.OptionParser(usage=usage)
parser.add_option("--num-runs", action="store",
type = "int", dest = "num_runs",
default=1,
help = "number of runs (default: 1)")
parser.add_option("--output-dir", action="store",
type="string", dest="outputdir",
help="output results to json file")
parser.add_option("--no-capture", action="store_true",
dest = "no_capture",
help = "run through the test, but don't actually "
"capture anything")
parser.add_option("--enable-profiling", action="store_true",
dest = "enable_profiling",
help="Collect performance profiles using the built in profiler.")
parser.add_option("--get-internal-checkerboard-stats",
action="store_true",
dest="get_internal_checkerboard_stats",
help="get and calculate internal checkerboard stats")
parser.add_option("--startup-test",
action="store_true",
dest="startup_test",
help="measure startup times instead of normal metrics")
parser.add_option("--url-params", action="store",
dest="url_params", default="",
help="additional url parameters for test")
parser.add_option("--extra-prefs", action="store", dest="extra_prefs",
default="{}",
help="Extra profile preference for Firefox browsers. " \
"Must be passed in as a JSON dictionary")
parser.add_option("--use-apks", action="store_true", dest="use_apks",
help="use and install android APKs as part of test (instead of specifying appnames)")
parser.add_option("--date", action="store", dest="date",
metavar="YYYY-MM-DD",
help="get and test nightly build for date")
parser.add_option("--start-date", action="store", dest="start_date",
metavar="YYYY-MM-DD",
help="start date for range of nightlies to test")
parser.add_option("--end-date", action="store", dest="end_date",
metavar="YYYY-MM-DD",
help="end date for range of nightlies to test")
options, args = parser.parse_args()
if len(args) == 0:
parser.error("Must specify at least one argument: the path to the test")
try:
# we only need to validate extra_prefs, as we'll just be passing it down
# to runtest
json.loads(options.extra_prefs)
except ValueError:
parser.error("Error processing extra preferences: not valid JSON!")
raise
dates = []
appnames = []
apks = []
if options.start_date and options.end_date and len(args) == 1:
test = args[0]
start_date = eideticker.BuildRetriever.get_date(options.start_date)
end_date = eideticker.BuildRetriever.get_date(options.end_date)
days=(end_date-start_date).days
for numdays in range(days+1):
dates.append(start_date+datetime.timedelta(days=numdays))
elif options.date and len(args) == 1:
test = args[0]
dates = [eideticker.BuildRetriever.get_date(options.date)]
elif not options.date and len(args) >= 2:
test = args[0]
if options.use_apks:
apks = args[1:]
else:
appnames = args[1:]
elif not options.date or (not options.start_date and not options.end_date):
parser.error("Must specify date, date range, a set of appnames (e.g. org.mozilla.fennec) or a set of apks (if --use-apks is specified)")
devicePrefs = eideticker.getDevicePrefs(options)
device = eideticker.getDevice(**devicePrefs)
if options.outputdir:
outputfile = os.path.join(options.outputdir, "metric-test-%s.json" % time.time())
else:
outputfile = None
if appnames:
for appname in appnames:
run_test(device, options.outputdir, outputfile, test,
options.url_params,
options.num_runs,
options.startup_test,
options.no_capture,
options.get_internal_checkerboard_stats,
appname=appname,
enable_profiling=options.enable_profiling,
extra_prefs=options.extra_prefs,
**devicePrefs)
elif apks:
for apk in apks:
run_test(device, options.outputdir,
outputfile, test,
options.url_params,
options.num_runs,
options.startup_test,
options.no_capture,
options.get_internal_checkerboard_stats, apk=apk,
enable_profiling=options.enable_profiling,
extra_prefs=options.extra_prefs,
**devicePrefs)
else:
br = eideticker.BuildRetriever()
productname = "nightly"
product = eideticker.get_product(productname)
for date in dates:
apk = br.get_build(product, date)
run_test(device, options.outputdir,
outputfile, test,
options.url_params,
options.num_runs,
options.startup_test,
options.no_capture,
options.get_internal_checkerboard_stats, apk=apk,
appdate=date,
enable_profiling=options.enable_profiling,
extra_prefs=options.extra_prefs,
**devicePrefs)
def main(args=sys.argv[1:]):
usage = "usage: %prog [options] <appname> <test path>"
parser = eideticker.OptionParser(usage=usage)
parser.add_option("--url-params",
action="store",
dest="url_params",
help="additional url parameters for test")
parser.add_option("--name",
action="store",
type="string",
dest="capture_name",
help="name to give capture")
parser.add_option("--capture-file",
action="store",
type="string",
dest="capture_file",
help="name to give to capture file")
parser.add_option("--no-capture",
action="store_true",
dest="no_capture",
help="run through the test, but don't actually "
"capture anything")
parser.add_option("--checkerboard-log-file",
action="store",
type="string",
dest="checkerboard_log_file",
help="name to give checkerboarding stats file")
parser.add_option("--startup-test",
action="store_true",
dest="startup_test",
help="do a startup test: full capture, no actions")
parser.add_option(
"--b2g",
action="store_true",
dest="b2g",
default=False,
help="Run in B2G environment. You do not need to pass an appname")
parser.add_option(
"--profile-file",
action="store",
type="string",
dest="profile_file",
help="Collect a performance profile using the built in profiler.")
options, args = parser.parse_args()
testpath, appname = None, None
if options.b2g:
if len(args) != 1:
parser.error("incorrect number of arguments")
sys.exit(1)
testpath = args[0]
else:
if len(args) != 2:
parser.error("incorrect number of arguments")
sys.exit(1)
(appname, testpath) = args
# Tests must be in src/tests/... unless it is a startup test and the
# path is about:home (indicating we want to measure startup to the
# home screen)
if options.startup_test and testpath == "about:home":
testpath_rel = testpath
capture_timeout = 5 # 5 seconds to wait for fennec to start after it claims to have started
else:
capture_timeout = None
try:
testpath_rel = os.path.abspath(testpath).split(TEST_DIR)[1][1:]
except:
print "Test must be relative to %s" % TEST_DIR
sys.exit(1)
actions = None
if not options.startup_test:
actions_path = os.path.join(os.path.dirname(testpath), "actions.json")
try:
with open(actions_path) as f:
actions = json.loads(f.read())
except EnvironmentError:
print "Couldn't open actions file '%s'" % actions_path
sys.exit(1)
if not os.path.exists(EIDETICKER_TEMP_DIR):
os.mkdir(EIDETICKER_TEMP_DIR)
if not os.path.isdir(EIDETICKER_TEMP_DIR):
print "Could not open eideticker temporary directory"
sys.exit(1)
capture_name = options.capture_name
if not capture_name:
capture_name = testpath_rel
capture_file = options.capture_file
if not capture_file and not options.no_capture:
capture_file = os.path.join(
CAPTURE_DIR,
"capture-%s.zip" % datetime.datetime.now().isoformat())
# Create a device object to interface with the phone
devicePrefs = eideticker.getDevicePrefs(options)
device = eideticker.getDevice(**devicePrefs)
if appname and device.processExist(appname):
print "An instance of %s is running. Please stop it before running Eideticker." % appname
sys.exit(1)
# If we're logging checkerboard stats, set that up here (seems like it
# takes a second or so to accept the new setting, so let's do that here --
# ideally we would detect when that's working, but I'm not sure how to do
# so trivially)
if options.checkerboard_log_file:
old_log_val = device.getprop("log.tag.GeckoLayerRendererProf")
device.setprop("log.tag.GeckoLayerRendererProf", "DEBUG")
print "Creating webserver..."
capture_metadata = {
'name': capture_name,
'testpath': testpath_rel,
'app': appname,
'device': device.model,
'startupTest': options.startup_test
}
# something of a hack. if profiling is enabled, carve off an area to
# ignore in the capture
if options.profile_file:
capture_metadata['ignoreAreas'] = [[0, 0, 3 * 64, 3]]
capture_server = CaptureServer(capture_metadata, capture_file,
options.checkerboard_log_file,
capture_controller, device, actions)
host = mozhttpd.iface.get_lan_ip()
http = mozhttpd.MozHttpd(docroot=TEST_DIR,
host=host,
port=0,
urlhandlers=[{
'method':
'GET',
'path':
'/api/captures/start/?',
'function':
capture_server.start_capture
}, {
'method':
'GET',
'path':
'/api/captures/end/?',
'function':
capture_server.end_capture
}, {
'method': 'POST',
'path': '/api/captures/input/?',
'function': capture_server.input
}])
http.start(block=False)
connected = False
tries = 0
while not connected and tries < 20:
tries += 1
import socket
s = socket.socket()
try:
s.connect((host, http.httpd.server_port))
connected = True
except Exception:
print "Can't connect to %s:%s, retrying..." % (
host, http.httpd.server_port)
if not connected:
print "Could not open webserver. Error!"
sys.exit(1)
# note: url params for startup tests currently not supported
if options.url_params:
testpath_rel += "?%s" % urllib.quote_plus(options.url_params)
if options.startup_test:
if testpath == "about:home":
url = testpath
else:
url = "http://%s:%s/%s?startup_test=1" % (
host, http.httpd.server_port, testpath_rel)
else:
url = "http://%s:%s/start.html?testpath=%s" % (
host, http.httpd.server_port, testpath_rel)
print "Test URL is: %s" % url
if options.b2g:
runner = eideticker.B2GRunner(device, url, EIDETICKER_TEMP_DIR)
else:
runner = eideticker.BrowserRunner(device, appname, url)
# FIXME: currently start capture before launching app because we wait until app is
# launched -- would be better to make waiting optional and then start capture
# after triggering app launch to reduce latency?
if options.startup_test and not options.no_capture:
capture_controller.start_capture(capture_file, device.hdmiResolution,
capture_metadata)
runner.start(profile_file=options.profile_file)
# Keep on capturing until we timeout
if capture_timeout:
timeout = capture_timeout
else:
timeout = 100
timer = 0
interval = 0.1
try:
while not capture_server.finished and timer < timeout:
time.sleep(interval)
timer += interval
except KeyboardInterrupt:
print "Aborting"
runner.stop()
capture_server.terminate_capture()
sys.exit(1)
if capture_timeout and not capture_server.finished:
capture_server.terminate_capture()
elif not capture_server.finished:
print "Did not finish test! Error!"
runner.stop()
capture_server.terminate_capture()
sys.exit(1)
runner.stop()
if capture_file:
print "Converting capture..."
try:
capture_controller.convert_capture(capture_server.start_frame,
capture_server.end_frame)
except KeyboardInterrupt:
print "Aborting"
sys.exit(1)
# Clean up checkerboard logging preferences
if options.checkerboard_log_file:
device.setprop("log.tag.GeckoLayerRendererProf", old_log_val)
def main(args=sys.argv[1:]):
usage = "usage: %prog [options] <product> <test> <output dir>"
parser = eideticker.OptionParser(usage=usage)
parser.add_option("--enable-profiling",
action="store_true", dest = "enable_profiling",
help = "Create SPS profile to go along with capture")
parser.add_option("--device-id", action="store", dest="device_id",
help="id of device (used in output json)")
parser.add_option("--apk", action="store", dest="apk",
help = "Product apk to get metadata from " \
"(Android-specific)")
parser.add_option("--num-runs", action="store",
type = "int", dest = "num_runs",
help = "number of runs (default: 1)")
options, args = parser.parse_args()
if len(args) != 3:
parser.error("incorrect number of arguments")
(productname, testkey, outputdir) = args
num_runs = 1
if options.num_runs:
num_runs = options.num_runs
manifest = manifestparser.TestManifest(manifests=[os.path.join(
os.path.dirname(__file__), '../src/tests/manifest.ini')])
# sanity check... does the test match a known test key?
testkeys = [test["key"] for test in manifest.active_tests()]
if testkey not in testkeys:
print "ERROR: No tests matching '%s' (options: %s)" % (testkey, ", ".join(testkeys))
sys.exit(1)
testinfo = [test for test in manifest.active_tests() if test['key'] == testkey][0]
device_id = options.device_id
if not device_id:
device_id = os.environ.get('DEVICE_ID')
if not device_id:
print "ERROR: Must specify device id (either with --device-id or with DEVICE_ID environment variable)"
sys.exit(1)
product = eideticker.get_product(productname)
current_date = time.strftime("%Y-%m-%d")
datafile = os.path.join(outputdir, device_id, '%s.json' % testkey)
data = NestedDict()
if os.path.isfile(datafile):
data.update(json.loads(open(datafile).read()))
devicePrefs = eideticker.getDevicePrefs(options)
device = eideticker.getDevice(**devicePrefs)
# update the device list for the dashboard
devices = {}
devicefile = os.path.join(outputdir, 'devices.json')
if os.path.isfile(devicefile):
devices = json.loads(open(devicefile).read())['devices']
testfile = os.path.join(outputdir, '%s' % device_id, 'tests.json')
if os.path.isfile(testfile):
tests = json.loads(open(testfile).read())['tests']
else:
tests = {}
tests[testkey] = { 'shortDesc': testinfo['shortDesc'],
'defaultMeasure': testinfo['defaultMeasure'] }
devices[device_id] = { 'name': device.model,
'version': device.getprop('ro.build.version.release') }
with open(devicefile, 'w') as f:
f.write(json.dumps({ 'devices': devices }))
testfiledir = os.path.dirname(testfile)
if not os.path.exists(testfiledir):
os.mkdir(testfiledir)
with open(testfile, 'w') as f:
f.write(json.dumps({ 'tests': tests }))
if options.apk:
appinfo = eideticker.get_fennec_appinfo(options.apk)
appname = appinfo['appname']
print "Using application name '%s' from apk '%s'" % (appname, options.apk)
capture_name = "%s %s" % (product['name'], appinfo['date'])
else:
# no apk, assume it's something static on the device
appinfo = { 'date': 'today' }
appname = product['appname']
capture_name = "%s (taken on %s)" % (product['name'], current_date)
# Run the test the specified number of times
for i in range(num_runs):
# Now run the test
runtest(device, product, appname, appinfo, testinfo,
capture_name + " #%s" % i, outputdir, datafile, data,
enable_profiling=options.enable_profiling, **devicePrefs)
# Kill app after test complete
device.killProcess(appname)
def main(args=sys.argv[1:]):
usage = "usage: %prog [options] TEST..."
parser = eideticker.TestOptionParser(usage=usage)
eideticker.add_dashboard_options(parser)
parser.add_option("--enable-profiling",
action="store_true",
dest="enable_profiling",
help="Create SPS profile to go along with capture")
parser.add_option("--dashboard-id",
action="store",
dest="dashboard_id",
help="id of dashboard (used in output json)",
default=os.environ.get('DASHBOARD_ID'))
parser.add_option("--dashboard-name",
action="store",
dest="dashboard_name",
help="name of dashboard to display",
default=os.environ.get('DASHBOARD_NAME'))
parser.add_option("--device-id",
action="store",
dest="device_id",
help="id of device (used in output json)",
default=os.environ.get('DEVICE_ID'))
parser.add_option("--branch",
action="store",
dest="branch_id",
help="branch under test (used in output json)",
default=os.environ.get('BRANCH'))
parser.add_option("--device-name",
action="store",
dest="device_name",
help="name of device to display in dashboard (if not "
"specified, display model name)",
default=os.environ.get('DEVICE_NAME'))
parser.add_option("--apk",
action="store",
dest="apk",
help="Product apk to get metadata from "
"(Android-specific)")
parser.add_option("--baseline",
action="store_true",
dest="baseline",
help="Create baseline results for dashboard")
parser.add_option("--num-runs",
action="store",
type="int",
dest="num_runs",
help="number of runs (default: %default)",
default=1)
parser.add_option("--app-version",
action="store",
dest="app_version",
help="Specify app version (if not automatically "
"available; Android-specific)")
parser.add_option("--sources-xml",
action="store",
dest="sources_xml",
help="Path to sources XML file for getting revision "
"information (B2G-specific)")
parser.add_option("--product",
action="store",
type="string",
dest="product_name",
default="nightly",
help="product name (android-specific, default: "
"%default)")
options, args = parser.parse_args()
if not args: # need to specify at least one test to run!
parser.print_usage()
sys.exit(1)
if not options.dashboard_id:
parser.error("Must specify dashboard id (either with --dashboard-id "
"or with DASHBOARD_ID environment variable)")
if not options.dashboard_name:
parser.error("Must specify dashboard name (either with "
"--dashboard-name or with DASHBOARD_NAME environment "
"varaiable)")
if not options.device_id:
parser.error("Must specify device id (either with --device-id or with "
"DEVICE_ID environment variable)")
if not options.branch_id:
parser.error("Must specify branch (either with --branch or with "
"BRANCH environment variable)")
# get device info
device_prefs = eideticker.getDevicePrefs(options)
device = eideticker.getDevice(**device_prefs)
device_name = options.device_name
if not device_name:
device_name = device.model
# copy dashboard files to output directory (if applicable)
eideticker.copy_dashboard_files(options.dashboard_dir)
if options.devicetype == 'android':
product = eideticker.get_product(options.product_name)
device_info = {
'name': device_name,
'version': device.getprop('ro.build.version.release')
}
elif options.devicetype == 'b2g':
product = eideticker.get_product('b2g-nightly')
device_info = {'name': device_name}
else:
print "ERROR: Unknown device type '%s'" % options.devicetype
# update dashboard / device index
eideticker.update_dashboard_list(options.dashboard_dir,
options.dashboard_id,
options.dashboard_name)
eideticker.update_dashboard_device_list(options.dashboard_dir,
options.dashboard_id,
options.device_id,
options.branch_id, device_info)
# get application/build info
if options.devicetype == "android":
if options.apk:
if options.app_version:
raise Exception("Should specify either --app-version or "
"--apk, not both!")
appinfo = eideticker.get_fennec_appinfo(options.apk)
options.appname = appinfo['appname']
print "Using application name '%s' from apk '%s'" % (
options.appname, options.apk)
options.capture_name = "%s %s" % (product['name'],
appinfo['appdate'])
else:
if not options.app_version:
raise Exception("Should specify --app-version if not --apk!")
# no apk, assume it's something static on the device
appinfo = {
'appdate': time.strftime("%Y-%m-%d"),
'version': options.app_version
}
elif options.devicetype == "b2g":
if not options.sources_xml:
raise Exception("Must specify --sources-xml on b2g!")
appinicontents = device.pullFile('/system/b2g/application.ini')
sfh = StringIO.StringIO(appinicontents)
appinfo = eideticker.get_appinfo(sfh)
appinfo.update(get_revision_data(options.sources_xml))
options.appname = None
else:
print "Unknown device type '%s'!" % options.devicetype
# run through the tests...
failed_tests = []
for testkey in args:
testinfo = eideticker.get_testinfo(testkey)
eideticker.update_dashboard_test_list(options.dashboard_dir,
options.dashboard_id,
options.device_id,
options.branch_id, testinfo)
current_date = time.strftime("%Y-%m-%d")
options.capture_name = "%s - %s (taken on %s)" % (
testkey, product['name'], current_date)
if options.prepare_test:
eideticker.prepare_test(testkey, options)
# Run the test the specified number of times
for i in range(options.num_runs):
try:
runtest(device, device_prefs, options, product, appinfo,
testinfo, options.capture_name + " #%s" % i)
except eideticker.TestException:
print "Unable to run test '%s'. Skipping and continuing." % testkey
failed_tests.append(testkey)
break
# synchronize with dashboard (if we have a server to upload to)
if options.dashboard_server:
eideticker.upload_dashboard(options)
else:
print "No dashboard server specified. Skipping upload."
if failed_tests:
print "The following tests failed: %s" % ", ".join(failed_tests)
sys.exit(1)
def main(args=sys.argv[1:]):
usage = "usage: %prog [options] TEST..."
parser = eideticker.TestOptionParser(usage=usage)
eideticker.add_dashboard_options(parser)
parser.add_option("--enable-profiling",
action="store_true", dest="enable_profiling",
help="Create SPS profile to go along with capture")
parser.add_option("--device-id", action="store", dest="device_id",
help="id of device (used in output json)",
default=os.environ.get('DEVICE_ID'))
parser.add_option("--branch", action="store", dest="branch_id",
help="branch under test (used in output json)",
default=os.environ.get('BRANCH'))
parser.add_option("--device-name", action="store", dest="device_name",
help="name of device to display in dashboard (if not "
"specified, display model name)",
default=os.environ.get('DEVICE_NAME'))
parser.add_option("--apk", action="store", dest="apk",
help="Product apk to get metadata from "
"(Android-specific)")
parser.add_option("--baseline", action="store_true", dest="baseline",
help="Create baseline results for dashboard")
parser.add_option("--num-runs", action="store",
type="int", dest="num_runs",
help="number of runs (default: %default)", default=1)
parser.add_option("--app-version", action="store", dest="app_version",
help="Specify app version (if not automatically "
"available; Android-specific)")
parser.add_option("--sources-xml", action="store", dest="sources_xml",
help="Path to sources XML file for getting revision "
"information (B2G-specific)")
parser.add_option("--product", action="store",
type="string", dest="product_name",
default="nightly",
help="product name (android-specific, default: "
"%default)")
options, args = parser.parse_args()
if not args: # need to specify at least one test to run!
parser.print_usage()
sys.exit(1)
if not options.device_id:
print "ERROR: Must specify device id (either with --device-id or with " \
"DEVICE_ID environment variable)"
sys.exit(1)
if not options.branch_id:
print "ERROR: Must specify branch (either with --branch or with " \
"BRANCH environment variable)"
sys.exit(1)
# get device info
device_prefs = eideticker.getDevicePrefs(options)
device = eideticker.getDevice(**device_prefs)
device_name = options.device_name
if not device_name:
device_name = device.model
# copy dashboard files to output directory (if applicable)
eideticker.copy_dashboard_files(options.dashboard_dir)
if options.devicetype == 'android':
product = eideticker.get_product(options.product_name)
device_info = { 'name': device_name,
'version': device.getprop('ro.build.version.release')}
elif options.devicetype == 'b2g':
product = eideticker.get_product('b2g-nightly')
device_info = { 'name': device_name }
else:
print "ERROR: Unknown device type '%s'" % options.devicetype
# update device index
eideticker.update_dashboard_device_list(options.dashboard_dir, options.device_id,
options.branch_id, device_info)
# get application/build info
if options.devicetype == "android":
if options.apk:
if options.app_version:
raise Exception("Should specify either --app-version or "
"--apk, not both!")
appinfo = eideticker.get_fennec_appinfo(options.apk)
options.appname = appinfo['appname']
print "Using application name '%s' from apk '%s'" % (
options.appname, options.apk)
options.capture_name = "%s %s" % (product['name'], appinfo['appdate'])
else:
if not options.app_version:
raise Exception("Should specify --app-version if not --apk!")
# no apk, assume it's something static on the device
appinfo = {
'appdate': time.strftime("%Y-%m-%d"),
'version': options.app_version}
elif options.devicetype == "b2g":
if not options.sources_xml:
raise Exception("Must specify --sources-xml on b2g!")
appinicontents = device.pullFile('/system/b2g/application.ini')
sfh = StringIO.StringIO(appinicontents)
appinfo = eideticker.get_appinfo(sfh)
appinfo.update(get_revision_data(options.sources_xml))
options.appname = None
else:
print "Unknown device type '%s'!" % options.devicetype
# run through the tests...
failed_tests = []
for testkey in args:
testinfo = eideticker.get_testinfo(testkey)
eideticker.update_dashboard_test_list(options.dashboard_dir, options.device_id,
options.branch_id,
testinfo)
current_date = time.strftime("%Y-%m-%d")
options.capture_name = "%s - %s (taken on %s)" % (testkey, product['name'],
current_date)
if options.prepare_test:
eideticker.prepare_test(testkey, options)
# Run the test the specified number of times
for i in range(options.num_runs):
try:
runtest(device, device_prefs, options,
product, appinfo, testinfo,
options.capture_name + " #%s" % i)
except eideticker.TestException:
print "Unable to run test '%s'. Skipping and continuing." % testkey
failed_tests.append(testkey)
break
# synchronize with dashboard (if we have a server to upload to)
if options.dashboard_server:
eideticker.upload_dashboard(options)
else:
print "No dashboard server specified. Skipping upload."
if failed_tests:
print "The following tests failed: %s" % ", ".join(failed_tests)
sys.exit(1)
def main(args=sys.argv[1:]):
usage = "usage: %prog [options] <test> <output dir>"
parser = eideticker.OptionParser(usage=usage)
parser.add_option("--enable-profiling",
action="store_true",
dest="enable_profiling",
help="Create SPS profile to go along with capture")
parser.add_option("--device-id",
action="store",
dest="device_id",
help="id of device (used in output json)")
parser.add_option(
"--product",
action="store",
dest="product",
help="Restrict testing to product (options: %s)" %
", ".join([product["name"] for product in default_products]))
parser.add_option("--num-runs",
action="store",
type="int",
dest="num_runs",
help="number of runs (default: 1)")
options, args = parser.parse_args()
if len(args) != 2:
parser.error("incorrect number of arguments")
(testname, outputdir) = args
num_runs = 1
if options.num_runs:
num_runs = options.num_runs
testnames = [test["name"] for test in default_tests]
if testname not in testnames:
print "ERROR: No tests matching '%s' (options: %s)" % (
testname, ", ".join(testnames))
sys.exit(1)
else:
test = [test for test in default_tests if test['name'] == testname][0]
device_id = options.device_id
if not device_id:
device_id = os.environ.get('DEVICE_ID')
if not device_id:
print "ERROR: Must specify device id (either with --device-id or with DEVICE_ID environment variable)"
sys.exit(1)
products = default_products
if options.product:
products = [
product for product in default_products
if product['name'] == options.product
]
if not products:
print "ERROR: No products matching '%s'" % options.product
sys.exit(1)
current_date = time.strftime("%Y-%m-%d")
datafile = os.path.join(outputdir, 'data-%s.json' % device_id)
data = NestedDict()
if os.path.isfile(datafile):
data.update(json.loads(open(datafile).read()))
devicePrefs = eideticker.getDevicePrefs(options)
device = eideticker.getDevice(**devicePrefs)
# update the device list for the dashboard
devices = {}
devicefile = os.path.join(outputdir, 'devices.json')
if os.path.isfile(devicefile):
devices = json.loads(open(devicefile).read())['devices']
devices[device_id] = {
'name': device.model,
'version': device.getprop('ro.build.version.release')
}
with open(devicefile, 'w') as f:
f.write(json.dumps({'devices': devices}))
for product in products:
if product.get('url'):
product_fname = os.path.join(DOWNLOAD_DIR,
"%s.apk" % product['name'])
appinfo = eideticker.get_fennec_appinfo(product_fname)
appname = appinfo['appname']
print "Using application name '%s' from apk '%s'" % (appname,
product_fname)
capture_name = "%s %s" % (product['name'], appinfo['date'])
else:
appinfo = {}
appname = product['appname']
capture_name = "%s (taken on %s)" % (product['name'], current_date)
if appinfo.get('appname'):
appname = appinfo['appname']
else:
appname = product['appname']
# Run the test the specified number of times
for i in range(num_runs):
# Now run the test
runtest(device,
product,
current_date,
appname,
appinfo,
test,
capture_name + " #%s" % i,
outputdir,
datafile,
data,
enable_profiling=options.enable_profiling,
**devicePrefs)
# Kill app after test complete
device.killProcess(appname)
def main(args=sys.argv[1:]):
usage = "usage: %prog <test> [appname1] [appname2] ..."
parser = eideticker.OptionParser(usage=usage)
parser.add_option("--num-runs", action="store",
type = "int", dest = "num_runs",
default=1,
help = "number of runs (default: 1)")
parser.add_option("--output-dir", action="store",
type="string", dest="outputdir",
help="output results to json file")
parser.add_option("--no-capture", action="store_true",
dest = "no_capture",
help = "run through the test, but don't actually "
"capture anything")
parser.add_option("--profile-file", action="store",
type="string", dest = "profile_file",
help="Collect a performance profile using the built in profiler.")
parser.add_option("--get-internal-checkerboard-stats",
action="store_true",
dest="get_internal_checkerboard_stats",
help="get and calculate internal checkerboard stats")
parser.add_option("--startup-test",
action="store_true",
dest="startup_test",
help="measure startup times instead of normal metrics")
parser.add_option("--url-params", action="store",
dest="url_params", default="",
help="additional url parameters for test")
parser.add_option("--use-apks", action="store_true", dest="use_apks",
help="use and install android APKs as part of test (instead of specifying appnames)")
parser.add_option("--date", action="store", dest="date",
metavar="YYYY-MM-DD",
help="get and test nightly build for date")
parser.add_option("--start-date", action="store", dest="start_date",
metavar="YYYY-MM-DD",
help="start date for range of nightlies to test")
parser.add_option("--end-date", action="store", dest="end_date",
metavar="YYYY-MM-DD",
help="end date for range of nightlies to test")
options, args = parser.parse_args()
if len(args) == 0:
parser.error("Must specify at least one argument: the path to the test")
dates = []
appnames = []
apks = []
if options.start_date and options.end_date and len(args) == 1:
test = args[0]
start_date = get_date(options.start_date)
end_date = get_date(options.end_date)
days=(end_date-start_date).days
for numdays in range(days+1):
dates.append(start_date+datetime.timedelta(days=numdays))
elif options.date and len(args) == 1:
test = args[0]
dates = [get_date(options.date)]
elif not options.date and len(args) >= 2:
test = args[0]
if options.use_apks:
apks = args[1:]
else:
appnames = args[1:]
elif not options.date or (not options.start_date and not options.end_date):
parser.error("Must specify date, date range, a set of appnames (e.g. org.mozilla.fennec) or a set of apks (if --use-apks is specified)")
devicePrefs = eideticker.getDevicePrefs(options)
device = eideticker.getDevice(**devicePrefs)
if options.outputdir:
outputfile = os.path.join(options.outputdir, "metric-test-%s.json" % time.time())
else:
outputfile = None
if appnames:
for appname in appnames:
run_test(device, options.outputdir, outputfile, test,
options.url_params,
options.num_runs,
options.startup_test,
options.no_capture,
options.get_internal_checkerboard_stats, appname=appname,
profile_file=options.profile_file, **devicePrefs)
elif apks:
for apk in apks:
run_test(device, options.outputdir,
outputfile, test,
options.url_params,
options.num_runs,
options.startup_test,
options.no_capture,
options.get_internal_checkerboard_stats, apk=apk,
profile_file=options.profile_file, **devicePrefs)
else:
for date in dates:
apk = get_build_for_date(date)
run_test(device, options.outputdir,
outputfile, test,
options.url_params,
options.num_runs,
options.startup_test,
options.no_capture,
options.get_internal_checkerboard_stats, apk=apk,
appdate=date,
profile_file=options.profile_file, **devicePrefs)
def main(args=sys.argv[1:]):
usage = "usage: %prog <test> [appname1] [appname2] ..."
parser = eideticker.OptionParser(usage=usage)
parser.add_option("--num-runs",
action="store",
type="int",
dest="num_runs",
default=1,
help="number of runs (default: 1)")
parser.add_option("--output-dir",
action="store",
type="string",
dest="outputdir",
help="output results to json file")
parser.add_option("--no-capture",
action="store_true",
dest="no_capture",
help="run through the test, but don't actually "
"capture anything")
parser.add_option(
"--profile-file",
action="store",
type="string",
dest="profile_file",
help="Collect a performance profile using the built in profiler.")
parser.add_option("--get-internal-checkerboard-stats",
action="store_true",
dest="get_internal_checkerboard_stats",
help="get and calculate internal checkerboard stats")
parser.add_option("--startup-test",
action="store_true",
dest="startup_test",
help="measure startup times instead of normal metrics")
parser.add_option("--url-params",
action="store",
dest="url_params",
default="",
help="additional url parameters for test")
parser.add_option(
"--use-apks",
action="store_true",
dest="use_apks",
help=
"use and install android APKs as part of test (instead of specifying appnames)"
)
parser.add_option("--date",
action="store",
dest="date",
metavar="YYYY-MM-DD",
help="get and test nightly build for date")
parser.add_option("--start-date",
action="store",
dest="start_date",
metavar="YYYY-MM-DD",
help="start date for range of nightlies to test")
parser.add_option("--end-date",
action="store",
dest="end_date",
metavar="YYYY-MM-DD",
help="end date for range of nightlies to test")
options, args = parser.parse_args()
if len(args) == 0:
parser.error(
"Must specify at least one argument: the path to the test")
dates = []
appnames = []
apks = []
if options.start_date and options.end_date and len(args) == 1:
test = args[0]
start_date = get_date(options.start_date)
end_date = get_date(options.end_date)
days = (end_date - start_date).days
for numdays in range(days + 1):
dates.append(start_date + datetime.timedelta(days=numdays))
elif options.date and len(args) == 1:
test = args[0]
dates = [get_date(options.date)]
elif not options.date and len(args) >= 2:
test = args[0]
if options.use_apks:
apks = args[1:]
else:
appnames = args[1:]
elif not options.date or (not options.start_date and not options.end_date):
parser.error(
"Must specify date, date range, a set of appnames (e.g. org.mozilla.fennec) or a set of apks (if --use-apks is specified)"
)
devicePrefs = eideticker.getDevicePrefs(options)
device = eideticker.getDevice(**devicePrefs)
if options.outputdir:
outputfile = os.path.join(options.outputdir,
"metric-test-%s.json" % time.time())
else:
outputfile = None
if appnames:
for appname in appnames:
run_test(device,
options.outputdir,
outputfile,
test,
options.url_params,
options.num_runs,
options.startup_test,
options.no_capture,
options.get_internal_checkerboard_stats,
appname=appname,
profile_file=options.profile_file,
**devicePrefs)
elif apks:
for apk in apks:
run_test(device,
options.outputdir,
outputfile,
test,
options.url_params,
options.num_runs,
options.startup_test,
options.no_capture,
options.get_internal_checkerboard_stats,
apk=apk,
profile_file=options.profile_file,
**devicePrefs)
else:
for date in dates:
apk = get_build_for_date(date)
run_test(device,
options.outputdir,
outputfile,
test,
options.url_params,
options.num_runs,
options.startup_test,
options.no_capture,
options.get_internal_checkerboard_stats,
apk=apk,
appdate=date,
profile_file=options.profile_file,
**devicePrefs)
def main(args=sys.argv[1:]):
usage = "usage: %prog <test> [appname1] [appname2] ..."
parser = eideticker.TestOptionParser(usage=usage)
parser.add_option(
"--num-runs", action="store", type="int", dest="num_runs", default=1, help="number of runs (default: 1)"
)
parser.add_option(
"--output-dir", action="store", type="string", dest="outputdir", help="output results to web site"
)
parser.add_option(
"--no-capture",
action="store_true",
dest="no_capture",
help="run through the test, but don't actually " "capture anything",
)
parser.add_option(
"--enable-profiling",
action="store_true",
dest="enable_profiling",
help="Collect performance profiles using the built in profiler.",
)
parser.add_option(
"--get-internal-checkerboard-stats",
action="store_true",
dest="get_internal_checkerboard_stats",
help="get and calculate internal checkerboard stats",
)
parser.add_option(
"--url-params", action="store", dest="url_params", default="", help="additional url parameters for test"
)
parser.add_option(
"--use-apks",
action="store_true",
dest="use_apks",
help="use and install android APKs as part of test (instead of specifying appnames)",
)
parser.add_option(
"--date", action="store", dest="date", metavar="YYYY-MM-DD", help="get and test nightly build for date"
)
parser.add_option(
"--start-date",
action="store",
dest="start_date",
metavar="YYYY-MM-DD",
help="start date for range of nightlies to test",
)
parser.add_option(
"--end-date",
action="store",
dest="end_date",
metavar="YYYY-MM-DD",
help="end date for range of nightlies to test",
)
options, args = parser.parse_args()
if len(args) == 0:
parser.error("Must specify at least one argument: the test")
dates = []
appnames = []
apks = []
if options.start_date and options.end_date and len(args) == 1:
testname = args[0]
start_date = eideticker.BuildRetriever.get_date(options.start_date)
end_date = eideticker.BuildRetriever.get_date(options.end_date)
days = (end_date - start_date).days
for numdays in range(days + 1):
dates.append(start_date + datetime.timedelta(days=numdays))
elif options.date and len(args) == 1:
testname = args[0]
dates = [eideticker.BuildRetriever.get_date(options.date)]
elif not options.date and len(args) >= 2:
testname = args[0]
if options.use_apks:
apks = args[1:]
else:
appnames = args[1:]
elif options.devicetype == "b2g":
testname = args[0]
elif not options.date or (not options.start_date and not options.end_date):
parser.error(
"On Android, must specify date, date range, a set of appnames (e.g. org.mozilla.fennec) or a set of apks (if --use-apks is specified)"
)
device_prefs = eideticker.getDevicePrefs(options)
if options.outputdir:
for dirname in [
options.outputdir,
os.path.join(options.outputdir, "css"),
os.path.join(options.outputdir, "fonts"),
os.path.join(options.outputdir, "js"),
os.path.join(options.outputdir, "videos"),
os.path.join(options.outputdir, "framediffs"),
]:
if not os.path.exists(dirname):
os.makedirs(dirname)
for filename in [
"css/bootstrap.min.css",
"fonts/glyphicons-halflings-regular.eot",
"fonts/glyphicons-halflings-regular.svg",
"fonts/glyphicons-halflings-regular.ttf",
"fonts/glyphicons-halflings-regular.woff",
"framediff-view.html",
"js/ICanHaz.min.js",
"js/SS.min.js",
"js/bootstrap.min.js",
"js/common.js",
"js/framediff.js",
"js/jquery-1.7.1.min.js",
"js/jquery.flot.axislabels.js",
"js/jquery.flot.js",
"js/jquery.flot.stack.js",
"js/metric.js",
"metric.html",
]:
if filename == "metric.html":
outfilename = "index.html"
else:
outfilename = filename
shutil.copyfile(os.path.join(DASHBOARD_DIR, filename), os.path.join(options.outputdir, outfilename))
if options.devicetype == "b2g":
runtest(device_prefs, testname, options)
elif appnames:
for appname in appnames:
runtest(device_prefs, testname, options, appname=appname)
elif apks:
for apk in apks:
runtest(device_prefs, testname, options, apk=apk)
else:
br = eideticker.BuildRetriever()
productname = "nightly"
product = eideticker.get_product(productname)
for date in dates:
apk = br.get_build(product, date)
runtest(device_prefs, testname, options, apk=apk, appdate=date)
def main(args=sys.argv[1:]):
usage = "usage: %prog [options] <test key>"
parser = eideticker.TestOptionParser(usage=usage)
parser.add_option("--url-params",
action="store",
dest="url_params",
help="additional url parameters for test")
parser.add_option("--name",
action="store",
type="string",
dest="capture_name",
help="name to give capture")
parser.add_option("--capture-file",
action="store",
type="string",
dest="capture_file",
help="name to give to capture file")
parser.add_option("--no-capture",
action="store_true",
dest="no_capture",
help="run through the test, but don't actually "
"capture anything")
parser.add_option("--app-name",
action="store",
type="string",
dest="appname",
help="Specify an application name (android only)")
parser.add_option("--test-type",
action="store",
type="string",
dest="test_type",
help="override test type")
parser.add_option("--profile-file",
action="store",
type="string",
dest="profile_file",
help="Collect a performance profile using the built in "
"profiler (fennec only).")
parser.add_option("--request-log-file",
action="store",
type="string",
dest="request_log_file",
help="Collect a log of HTTP requests during test")
parser.add_option("--actions-log-file",
action="store",
type="string",
dest="actions_log_file",
help="Collect a log of actions requests during test")
options, args = parser.parse_args()
if len(args) != 1:
parser.error("You must specify (only) a test key")
sys.exit(1)
testkey = args[0]
device_prefs = eideticker.getDevicePrefs(options)
if options.prepare_test:
eideticker.prepare_test(testkey, device_prefs)
testlog = eideticker.run_test(
testkey,
options.capture_device,
options.appname,
options.capture_name,
device_prefs,
extra_prefs=options.extra_prefs,
extra_env_vars=options.extra_env_vars,
test_type=options.test_type,
profile_file=options.profile_file,
no_capture=options.no_capture,
capture_area=options.capture_area,
fps=options.fps,
capture_file=options.capture_file,
wifi_settings_file=options.wifi_settings_file,
sync_time=options.sync_time)
# save logs if applicable
testlog.save_logs(http_request_log_path=options.request_log_file,
actions_log_path=options.actions_log_file)
def main(args=sys.argv[1:]):
usage = "usage: %prog [options] <test> <output dir>"
parser = eideticker.OptionParser(usage=usage)
parser.add_option("--enable-profiling",
action="store_true", dest = "enable_profiling",
help = "Create SPS profile to go along with capture")
parser.add_option("--device-id", action="store", dest="device_id",
help="id of device (used in output json)")
parser.add_option("--product",
action="store", dest="product",
help = "Restrict testing to product (options: %s)" %
", ".join([product["name"] for product in default_products]))
parser.add_option("--num-runs", action="store",
type = "int", dest = "num_runs",
help = "number of runs (default: 1)")
options, args = parser.parse_args()
if len(args) != 2:
parser.error("incorrect number of arguments")
(testname, outputdir) = args
num_runs = 1
if options.num_runs:
num_runs = options.num_runs
testnames = [test["name"] for test in default_tests]
if testname not in testnames:
print "ERROR: No tests matching '%s' (options: %s)" % (testname, ", ".join(testnames))
sys.exit(1)
else:
test = [test for test in default_tests if test['name'] == testname][0]
device_id = options.device_id
if not device_id:
device_id = os.environ.get('DEVICE_ID')
if not device_id:
print "ERROR: Must specify device id (either with --device-id or with DEVICE_ID environment variable)"
sys.exit(1)
products = default_products
if options.product:
products = [product for product in default_products if product['name'] == options.product]
if not products:
print "ERROR: No products matching '%s'" % options.product
sys.exit(1)
current_date = time.strftime("%Y-%m-%d")
datafile = os.path.join(outputdir, 'data-%s.json' % device_id)
data = NestedDict()
if os.path.isfile(datafile):
data.update(json.loads(open(datafile).read()))
devicePrefs = eideticker.getDevicePrefs(options)
device = eideticker.getDevice(**devicePrefs)
# update the device list for the dashboard
devices = {}
devicefile = os.path.join(outputdir, 'devices.json')
if os.path.isfile(devicefile):
devices = json.loads(open(devicefile).read())['devices']
devices[device_id] = { 'name': device.model,
'version': device.getprop('ro.build.version.release') }
with open(devicefile, 'w') as f:
f.write(json.dumps({ 'devices': devices }))
for product in products:
if product.get('url'):
product_fname = os.path.join(DOWNLOAD_DIR, "%s.apk" % product['name'])
appinfo = eideticker.get_fennec_appinfo(product_fname)
appname = appinfo['appname']
print "Using application name '%s' from apk '%s'" % (appname, product_fname)
capture_name = "%s %s" % (product['name'], appinfo['date'])
else:
appinfo = { }
appname = product['appname']
capture_name = "%s (taken on %s)" % (product['name'], current_date)
if appinfo.get('appname'):
appname = appinfo['appname']
else:
appname = product['appname']
# Run the test the specified number of times
for i in range(num_runs):
# Now run the test
runtest(device, product, current_date, appname, appinfo, test,
capture_name + " #%s" % i, outputdir, datafile, data,
enable_profiling=options.enable_profiling, **devicePrefs)
# Kill app after test complete
device.killProcess(appname)
