def main(args=sys.argv[1:]): usage = "usage: %prog <product> <date>" parser = eideticker.OptionParser(usage=usage) options, args = parser.parse_args() if len(args) != 2: parser.error("incorrect number of arguments") (productname, datestr) = args product = eideticker.get_product(productname) devicePrefs = eideticker.getDevicePrefs(options) device = eideticker.getDevice(**devicePrefs) if device.type != 'android': print "Device type '%s' does not currently support updates" % device.type sys.exit(0) if not product.get('reponame'): print "No download / installation needed for %s" % product['name'] else: print "Downloading %s" % product['name'] br = eideticker.BuildRetriever() if datestr == 'latest': date = None else: date = eideticker.BuildRetriever.get_date(datestr) filename = br.get_build(product, date) print "Installing %s (%s)" % (product['name'], filename) device.updateApp(filename)
def main(args=sys.argv[1:]): usage = "usage: %prog [product1] [product2]" parser = eideticker.OptionParser(usage=usage) options, args = parser.parse_args() if len(args): products = [ product for product in default_products if product['name'] in args ] else: products = default_products if not products: print "No products matching arguments!" sys.exit(1) print products devicePrefs = eideticker.getDevicePrefs(options) device = eideticker.getDevice(**devicePrefs) for product in products: if not product.get('url'): print "No download / installation needed for %s" % product['name'] else: print "Downloading %s" % product['name'] product_fname = os.path.join(DOWNLOAD_DIR, "%s.apk" % product['name']) dl = urllib2.urlopen(product['url']) with open(product_fname, 'w') as f: f.write(dl.read()) print "Installing %s" % product['name'] device.updateApp(product_fname)
def main(args=sys.argv[1:]): parser = eideticker.OptionParser(usage="usage: %prog") options, args = parser.parse_args() # Create a device object to interface with the phone devicePrefs = eideticker.getDevicePrefs(options) device = eideticker.getDevice(**devicePrefs) print "READY" sys.stdout.flush() while 1: try: line = sys.stdin.readline() except KeyboardInterrupt: break if not line: break tokens = line.rstrip().split() if len(tokens) < 1: raise Exception("No command") (cmd, args) = (tokens[0], tokens[1:]) device.executeCommand(cmd, args)
def main(args=sys.argv[1:]): usage = "usage: %prog <product> <date>" parser = eideticker.OptionParser(usage=usage) options, args = parser.parse_args() if len(args) != 2: parser.error("incorrect number of arguments") (productname, datestr) = args product = eideticker.get_product(productname) devicePrefs = eideticker.getDevicePrefs(options) device = eideticker.getDevice(**devicePrefs) if device.type != 'android': print "Device type '%s' does not currently support updates" % \ device.type sys.exit(0) if not product.get('reponame'): print "No download / installation needed for %s" % product['name'] else: print "Downloading %s (date: %s)" % (product['name'], datestr) br = eideticker.BuildRetriever() if datestr == 'latest': date = None else: date = eideticker.BuildRetriever.get_date(datestr) filename = br.get_build(product, date) print "Installing %s (%s)" % (product['name'], filename) device.updateApp(filename)
def main(args=sys.argv[1:]): usage = "usage: %prog" parser = eideticker.OptionParser(usage=usage) options, args = parser.parse_args() devicePrefs = eideticker.getDevicePrefs(options) device = eideticker.getDevice(**devicePrefs) if device.type == 'b2g': print "No cleanup presently required on b2g" return for product in eideticker.products: if product['platform'] == device.type: device.killProcess(product['appname']) # clean up any test stuff (profiles, etc.) device.removeDir(device.getDeviceRoot()) # profile files are dumped directly to the sdcard, # not the testroot so we need to clean them up # seperately files = device.listFiles('/mnt/sdcard/') for file in files: if re.match('profile_.*txt', file): device.removeFile('/mnt/sdcard/%s' % file)
def main(args=sys.argv[1:]): usage = "usage: %prog [product1] [product2]" parser = eideticker.OptionParser(usage=usage) options, args = parser.parse_args() if len(args): products = [product for product in default_products if product['name'] in args] else: products = default_products if not products: print "No products matching arguments!" sys.exit(1) print products device = eideticker.getDevice(options) for product in products: if not product.get('url'): print "No download / installation needed for %s" % product['name'] else: print "Downloading %s" % product['name'] product_fname = os.path.join(DOWNLOAD_DIR, "%s.apk" % product['name']) dl = urllib2.urlopen(product['url']) with open(product_fname, 'w') as f: f.write(dl.read()) device.updateApp(product_fname)
def main(args=sys.argv[1:]): usage = "usage: %prog" parser = eideticker.OptionParser(usage=usage) options, args = parser.parse_args() devicePrefs = eideticker.getDevicePrefs(options) device = eideticker.getDevice(**devicePrefs) device.cleanup()
def run_capture(options, capture_file): device_prefs = eideticker.getDevicePrefs(options) capture_server = CaptureServer(capture_file, options.capture_device, options.mode, capture=options.capture) host = moznetwork.get_ip() docroot = eideticker.runtest.TEST_DIR httpd = mozhttpd.MozHttpd(port=0, host=host, docroot=docroot, urlhandlers=[{ 'method': 'GET', 'path': '/api/captures/start/?', 'function': capture_server.start_capture }, { 'method': 'GET', 'path': '/api/captures/end/?', 'function': capture_server.end_capture }]) httpd.start(block=False) print "Serving '%s' at %s:%s" % (httpd.docroot, httpd.host, httpd.httpd.server_port) device = eideticker.getDevice(**device_prefs) mode = options.mode if not mode: mode = device.hdmiResolution url = "http://%s:%s/getdimensions.html" % (host, httpd.httpd.server_port) device.executeCommand("tap", [100, 100]) if device_prefs['devicetype'] == 'android': device.launchFennec(options.appname, url=url) else: if not options.wifi_settings_file: print "WIFI settings file (see --help) required for B2G!" sys.exit(1) device.restartB2G() device.connectWIFI(json.loads(open(options.wifi_settings_file).read())) device.marionette.execute_script("window.location.href='%s';" % url) while not capture_server.finished: time.sleep(0.25) capture_server.convert_capture() httpd.stop()
def run_capture(options, capture_file): device_prefs = eideticker.getDevicePrefs(options) capture_server = CaptureServer(capture_file, options.capture_device, options.mode, no_capture=options.no_capture) host = moznetwork.get_ip() docroot = eideticker.runtest.TEST_DIR httpd = mozhttpd.MozHttpd(port=0, host=host, docroot=docroot, urlhandlers=[ {'method': 'GET', 'path': '/api/captures/start/?', 'function': capture_server.start_capture}, {'method': 'GET', 'path': '/api/captures/end/?', 'function': capture_server.end_capture}]) httpd.start(block=False) print "Serving '%s' at %s:%s" % ( httpd.docroot, httpd.host, httpd.httpd.server_port) device = eideticker.getDevice(**device_prefs) mode = options.mode if not mode: mode = device.hdmiResolution url = "http://%s:%s/getdimensions.html" % (host, httpd.httpd.server_port) device.executeCommand("tap", [100, 100]) if device_prefs['devicetype'] == 'android': device.launchFennec(options.appname, url=url) else: if not options.wifi_settings_file: print "WIFI settings file (see --help) required for B2G!" sys.exit(1) device.restartB2G() device.connectWIFI(json.loads(open(options.wifi_settings_file).read())) session = device.marionette.session if 'b2g' not in session: raise Exception("bad session value %s returned by start_session" % session) device.unlock() # wait for device to become ready (yes, this is terrible, can we # detect this condition in marionette somehow?) time.sleep(5) device.marionette.execute_script("window.location.href='%s';" % url) while not capture_server.finished: time.sleep(0.25) capture_server.convert_capture() device.killProcess(options.appname) httpd.stop()
def main(args=sys.argv[1:]): usage = "usage: %prog" parser = eideticker.OptionParser(usage=usage) options, args = parser.parse_args() devicePrefs = eideticker.getDevicePrefs(options) device = eideticker.getDevice(**devicePrefs) for product in default_products: device.killProcess(product['appname']) # clean up any test stuff (profiles, etc.) device.removeDir(device.getDeviceRoot())
def main(args=sys.argv[1:]): usage = "usage: %prog" parser = eideticker.OptionParser(usage=usage) options, args = parser.parse_args() devicePrefs = eideticker.getDevicePrefs(options) device = eideticker.getDevice(**devicePrefs) if device.type == 'b2g': print "No cleanup presently required on b2g" return for product in eideticker.products: if product['platform'] == device.type: device.killProcess(product['appname']) # clean up any test stuff (profiles, etc.) device.removeDir(device.getDeviceRoot())
def run_capture(options, capture_filename): device_prefs = eideticker.getDevicePrefs(options) capture_server = CaptureServer(capture_filename, options) host = moznetwork.get_ip() docroot = eideticker.test.TEST_DIR httpd = mozhttpd.MozHttpd(port=0, host=host, docroot=docroot, urlhandlers=[ {'method': 'GET', 'path': '/api/captures/start/?', 'function': capture_server.start_capture}, {'method': 'GET', 'path': '/api/captures/end/?', 'function': capture_server.end_capture}]) httpd.start(block=False) print "Serving '%s' at %s:%s" % ( httpd.docroot, httpd.host, httpd.httpd.server_port) device = eideticker.getDevice(**device_prefs) url = "http://%s:%s/getdimensions.html" % (host, httpd.httpd.server_port) device.executeCommand("tap", [100, 100]) if device_prefs['devicetype'] == 'android': device.launchFennec(options.appname, url=url) else: if not options.wifi_settings_file: print "WIFI settings file (see --help) required for B2G!" sys.exit(1) device.restartB2G() device.connectWIFI(json.loads(open(options.wifi_settings_file).read())) device.marionette.execute_script("window.location.href='%s';" % url) while not capture_server.finished: time.sleep(0.25) capture_server.convert_capture() httpd.stop()
def main(args=sys.argv[1:]): usage = "usage: %prog [options] <product> <test> <output dir>" parser = eideticker.TestOptionParser(usage=usage) parser.add_option("--enable-profiling", action="store_true", dest="enable_profiling", help="Create SPS profile to go along with capture") parser.add_option("--device-id", action="store", dest="device_id", help="id of device (used in output json)", default=os.environ.get('DEVICE_ID')) parser.add_option("--device-name", action="store", dest="device_name", help="name of device to display in dashboard (if not " "specified, display model name)", default=os.environ.get('DEVICE_NAME')) parser.add_option("--apk", action="store", dest="apk", help="Product apk to get metadata from " "(Android-specific)") parser.add_option("--baseline", action="store_true", dest="baseline", help="Create baseline results for dashboard") parser.add_option("--num-runs", action="store", type="int", dest="num_runs", help="number of runs (default: 1)") parser.add_option("--app-version", action="store", dest="app_version", help="Specify app version (if not automatically " "available; Android-specific)") parser.add_option("--sources-xml", action="store", dest="sources_xml", help="Path to sources XML file for getting revision " "information (B2G-specific)") options, args = parser.parse_args() if len(args) != 3: parser.print_usage() sys.exit(1) (productname, testkey, outputdir) = args num_runs = 1 if options.num_runs: num_runs = options.num_runs testinfo = eideticker.get_testinfo(testkey) device_id = options.device_id if not device_id: print "ERROR: Must specify device id (either with --device-id or with " "DEVICE_ID environment variable)" sys.exit(1) # we'll log http requests for webstartup tests only log_http_requests = False if testinfo['type'] == 'webstartup': log_http_requests = True # likewise, log actions only for web tests and b2g tests log_actions = False if testinfo['type'] == 'web' or testinfo['type'] == 'b2g': log_actions = True product = eideticker.get_product(productname) current_date = time.strftime("%Y-%m-%d") capture_name = "%s - %s (taken on %s)" % (testkey, product['name'], current_date) datafile = os.path.join(outputdir, device_id, '%s.json' % testkey) data = NestedDict() if os.path.isfile(datafile): data.update(json.loads(open(datafile).read())) device_prefs = eideticker.getDevicePrefs(options) device = eideticker.getDevice(**device_prefs) devices = {} devicefile = os.path.join(outputdir, 'devices.json') if os.path.isfile(devicefile): devices = json.loads(open(devicefile).read())['devices'] testfile = os.path.join(outputdir, '%s' % device_id, 'tests.json') if os.path.isfile(testfile): tests = json.loads(open(testfile).read())['tests'] else: tests = {} tests[testkey] = { 'shortDesc': testinfo['shortDesc'], 'defaultMeasure': testinfo['defaultMeasure'] } device_name = options.device_name if not device_name: device_name = device.model if options.devicetype == "android": devices[device_id] = { 'name': device_name, 'version': device.getprop('ro.build.version.release') } if options.apk: if options.app_version: raise Exception("Should specify either --app-version or " "--apk, not both!") appinfo = eideticker.get_fennec_appinfo(options.apk) appname = appinfo['appname'] print "Using application name '%s' from apk '%s'" % (appname, options.apk) capture_name = "%s %s" % (product['name'], appinfo['appdate']) else: if not options.app_version: raise Exception("Should specify --app-version if not --apk!") # no apk, assume it's something static on the device appinfo = { 'appdate': time.strftime("%Y-%m-%d"), 'version': options.app_version } appname = product['appname'] elif options.devicetype == "b2g": if not options.sources_xml: raise Exception("Must specify --sources-xml on b2g!") devices[device_id] = {'name': device_name} appinicontents = device.pullFile('/system/b2g/application.ini') sfh = StringIO.StringIO(appinicontents) appinfo = eideticker.get_appinfo(sfh) appinfo.update(get_revision_data(options.sources_xml)) appname = None else: print "Unknown device type '%s'!" % options.devicetype # update the device / test list for the dashboard with open(devicefile, 'w') as f: f.write(json.dumps({'devices': devices})) testfiledir = os.path.dirname(testfile) if not os.path.exists(testfiledir): os.mkdir(testfiledir) with open(testfile, 'w') as f: f.write(json.dumps({'tests': tests})) if options.prepare_test: eideticker.prepare_test(testkey, device_prefs) # Run the test the specified number of times for i in range(num_runs): runtest(device, device_prefs, options.capture_device, options.capture_area, product, appname, appinfo, testinfo, capture_name + " #%s" % i, outputdir, datafile, data, enable_profiling=options.enable_profiling, log_http_requests=log_http_requests, log_actions=log_actions, baseline=options.baseline, wifi_settings_file=options.wifi_settings_file, sync_time=options.sync_time) if options.devicetype == "android": # Kill app after test complete device.killProcess(appname)
def runtest(device_prefs, testname, options, apk=None, appname=None, appdate=None): if apk: appinfo = eideticker.get_fennec_appinfo(apk) appname = appinfo['appname'] print "Installing %s (version: %s, revision %s)" % ( appinfo['appname'], appinfo['version'], appinfo['revision']) device = eideticker.getDevice(**device_prefs) device.updateApp(apk) else: appinfo = None testinfo = eideticker.get_testinfo(testname) stableframecapture = (testinfo['type'] in ('startup', 'webstartup') or testinfo['defaultMeasure'] == 'timetostableframe') capture_results = [] if options.prepare_test: eideticker.prepare_test(testname, device_prefs, options.wifi_settings_file) for i in range(options.num_runs): # Now run the test curtime = int(time.time()) capture_file = os.path.join( CAPTURE_DIR, "metric-test-%s-%s.zip" % (appname, curtime)) if options.enable_profiling and options.outputdir: profile_relpath = os.path.join('profiles', 'sps-profile-%s.zip' % time.time()) profile_file = os.path.join(options.outputdir, profile_relpath) else: profile_file = None current_date = time.strftime("%Y-%m-%d") capture_name = "%s - %s (taken on %s)" % (testname, appname, current_date) testlog = eideticker.run_test( testname, options.capture_device, appname, capture_name, device_prefs, extra_prefs=options.extra_prefs, extra_env_vars=options.extra_env_vars, log_checkerboard_stats=options.get_internal_checkerboard_stats, profile_file=profile_file, capture_area=options.capture_area, camera_settings_file=options.camera_settings_file, capture=options.capture, fps=options.fps, capture_file=capture_file, wifi_settings_file=options.wifi_settings_file, sync_time=options.sync_time, use_vpxenc=options.use_vpxenc) capture_uuid = uuid.uuid1().hex datapoint = {'uuid': capture_uuid} metadata = {} metrics = {} if options.capture: capture = videocapture.Capture(capture_file) datapoint['captureFile'] = metadata['captureFile'] = capture_file metadata['captureFPS'] = capture.fps metadata['generatedVideoFPS'] = capture.generated_video_fps if stableframecapture: metrics['timetostableframe'] = \ eideticker.get_stable_frame_time(capture) else: metrics.update( eideticker.get_standard_metrics(capture, testlog.actions)) metadata['metrics'] = metrics metadata.update(eideticker.get_standard_metric_metadata(capture)) if options.outputdir: # video video_relpath = os.path.join('videos', 'video-%s.webm' % time.time()) video_path = os.path.join(options.outputdir, video_relpath) open(video_path, 'w').write(capture.get_video().read()) metadata['video'] = video_relpath if options.get_internal_checkerboard_stats: metrics['internalcheckerboard'] = \ testlog.checkerboard_percent_totals # Want metrics data in data, so we can graph everything at once datapoint.update(metrics) if options.enable_profiling: metadata['profile'] = profile_file # dump metadata if options.outputdir: # metadata metadata_path = os.path.join(options.outputdir, 'metadata', '%s.json' % capture_uuid) open(metadata_path, 'w').write(json.dumps(metadata)) capture_results.append(datapoint) if options.devicetype == "b2g": # FIXME: get information from sources.xml and application.ini on # device, as we do in update-dashboard.py display_key = appkey = "FirefoxOS" else: appkey = appname if appdate: appkey = appdate.isoformat() else: appkey = appname if appinfo and appinfo.get('revision'): display_key = "%s (%s)" % (appkey, appinfo['revision']) else: display_key = appkey print "=== Results on %s for %s ===" % (testname, display_key) if options.capture: measures = [ ('timetostableframe', 'Times to first stable frame (seconds)'), ('uniqueframes', 'Number of unique frames'), ('fps', 'Average number of unique frames per second'), ('overallentropy', 'Overall entropy over length of capture'), ('checkerboard', 'Checkerboard area/duration (sum of percents NOT ' 'percentage)'), ('timetoresponse', 'Time to first input response (seconds)') ] for measure in measures: if capture_results[0].get(measure[0]): print " %s:" % measure[1] print " %s" % map(lambda c: c[measure[0]], capture_results) print print " Capture files:" print " Capture files: %s" % map(lambda c: c['captureFile'], capture_results) print if options.get_internal_checkerboard_stats: print " Internal Checkerboard Stats (sum of percents, not " "percentage):" print " %s" % map(lambda c: c['internalcheckerboard'], capture_results) print if options.outputdir: outputfile = os.path.join(options.outputdir, "metric.json") resultdict = {'title': testname, 'data': {}} if os.path.isfile(outputfile): resultdict.update(json.loads(open(outputfile).read())) if not resultdict['data'].get(appkey): resultdict['data'][appkey] = [] resultdict['data'][appkey].extend(capture_results) with open(outputfile, 'w') as f: f.write(json.dumps(resultdict))
def runtest(device_prefs, capture_device, outputdir, outputfile, testname, url_params, num_runs, startup_test, no_capture, get_internal_checkerboard_stats, apk=None, appname = None, appdate = None, enable_profiling=False, extra_prefs={}, extra_env_vars={}): device = None if apk: appinfo = eideticker.get_fennec_appinfo(apk) appname = appinfo['appname'] print "Installing %s (version: %s, revision %s)" % (appinfo['appname'], appinfo['version'], appinfo['revision']) device = eideticker.getDevice(**device_prefs) device.updateApp(apk) else: appinfo = None captures = [] for i in range(num_runs): # Kill any existing instances of the processes (for Android) if device: device.killProcess(appname) # Now run the test curtime = int(time.time()) capture_file = os.path.join(CAPTURE_DIR, "metric-test-%s-%s.zip" % (appname, curtime)) if enable_profiling: profile_file = os.path.join(PROFILE_DIR, "profile-%s-%s.zip" % (appname, curtime)) else: profile_file = None if get_internal_checkerboard_stats: checkerboard_log_file = tempfile.NamedTemporaryFile() else: checkerboard_log_file = None current_date = time.strftime("%Y-%m-%d") capture_name = "%s - %s (taken on %s)" % (testname, appname, current_date) eideticker.run_test(testname, capture_device, appname, capture_name, device_prefs, extra_prefs=extra_prefs, extra_env_vars=extra_env_vars, checkerboard_log_file=checkerboard_log_file, profile_file=profile_file, no_capture=no_capture, capture_file=capture_file) capture_result = {} if not no_capture: capture_result['file'] = capture_file capture = videocapture.Capture(capture_file) if startup_test: capture_result['stableframe'] = videocapture.get_stable_frame(capture) else: capture_result['uniqueframes'] = videocapture.get_num_unique_frames(capture) capture_result['fps'] = videocapture.get_fps(capture) capture_result['checkerboard'] = videocapture.get_checkerboarding_area_duration(capture) if outputdir: video_path = os.path.join('videos', 'video-%s.webm' % time.time()) video_file = os.path.join(outputdir, video_path) open(video_file, 'w').write(capture.get_video().read()) capture_result['video'] = video_path if enable_profiling: capture_result['profile'] = profile_file if get_internal_checkerboard_stats: internal_checkerboard_totals = parse_checkerboard_log(checkerboard_log_file.name) capture_result['internalcheckerboard'] = internal_checkerboard_totals captures.append(capture_result) appkey = appname if appdate: appkey = appdate.isoformat() else: appkey = appname if appinfo and appinfo.get('revision'): display_key = "%s (%s)" % (appkey, appinfo['revision']) else: display_key = appkey print "=== Results for %s ===" % display_key if not no_capture: if startup_test: print " First stable frames:" print " %s" % map(lambda c: c['stableframe'], captures) print else: print " Number of unique frames:" print " %s" % map(lambda c: c['uniqueframes'], captures) print print " Average number of unique frames per second:" print " %s" % map(lambda c: c['fps'], captures) print print " Checkerboard area/duration (sum of percents NOT percentage):" print " %s" % map(lambda c: c['checkerboard'], captures) print print " Capture files:" print " Capture files: %s" % map(lambda c: c['file'], captures) print if enable_profiling: print " Profile files:" print " Profile files: %s" % map(lambda c: c['profile'], captures) print if get_internal_checkerboard_stats: print " Internal Checkerboard Stats (sum of percents, not percentage):" print " %s" % map(lambda c: c['internalcheckerboard'], captures) print if outputfile: resultdict = { 'title': testname, 'data': {} } if os.path.isfile(outputfile): resultdict.update(json.loads(open(outputfile).read())) if not resultdict['data'].get(appkey): resultdict['data'][appkey] = [] resultdict['data'][appkey].extend(captures) with open(outputfile, 'w') as f: f.write(json.dumps(resultdict))
def main(args=sys.argv[1:]): usage = "usage: %prog [options] <product> <test> <output dir>" parser = eideticker.CaptureOptionParser(usage=usage) parser.add_option("--enable-profiling", action="store_true", dest = "enable_profiling", help = "Create SPS profile to go along with capture") parser.add_option("--device-id", action="store", dest="device_id", help="id of device (used in output json)") parser.add_option("--apk", action="store", dest="apk", help = "Product apk to get metadata from " \ "(Android-specific)") parser.add_option("--baseline", action="store_true", dest="baseline", help = "Create baseline results for dashboard") parser.add_option("--num-runs", action="store", type = "int", dest = "num_runs", help = "number of runs (default: 1)") parser.add_option("--app-version", action="store", dest="app_version", help="Specify app version (if not automatically " \ "available; Android-specific)") parser.add_option("--sources-xml", action="store", dest="sources_xml", help="Path to sources XML file for getting revision " \ "information (B2G-specific)") options, args = parser.parse_args() parser.validate_options(options) (productname, testkey, outputdir) = args num_runs = 1 if options.num_runs: num_runs = options.num_runs manifest = eideticker.get_test_manifest() # sanity check... does the test match a known test key? testkeys = [test["key"] for test in manifest.active_tests()] if testkey not in testkeys: print "ERROR: No tests matching '%s' (options: %s)" % (testkey, ", ".join(testkeys)) sys.exit(1) testinfo = [test for test in manifest.active_tests() if test['key'] == testkey][0] device_id = options.device_id if not device_id: device_id = os.environ.get('DEVICE_ID') if not device_id: print "ERROR: Must specify device id (either with --device-id or with DEVICE_ID environment variable)" sys.exit(1) # we'll log http requests for webstartup tests only log_http_requests = False if testinfo['type'] == 'webstartup': log_http_requests = True product = eideticker.get_product(productname) current_date = time.strftime("%Y-%m-%d") capture_name = "%s (taken on %s)" % (product['name'], current_date) datafile = os.path.join(outputdir, device_id, '%s.json' % testkey) data = NestedDict() if os.path.isfile(datafile): data.update(json.loads(open(datafile).read())) device_prefs = eideticker.getDevicePrefs(options) device = eideticker.getDevice(**device_prefs) devices = {} devicefile = os.path.join(outputdir, 'devices.json') if os.path.isfile(devicefile): devices = json.loads(open(devicefile).read())['devices'] testfile = os.path.join(outputdir, '%s' % device_id, 'tests.json') if os.path.isfile(testfile): tests = json.loads(open(testfile).read())['tests'] else: tests = {} tests[testkey] = { 'shortDesc': testinfo['shortDesc'], 'defaultMeasure': testinfo['defaultMeasure'] } if options.devicetype == "android": devices[device_id] = { 'name': device.model, 'version': device.getprop('ro.build.version.release') } if options.apk: if options.app_version: raise Exception("Should specify either --app-version or --apk, not both!") appinfo = eideticker.get_fennec_appinfo(options.apk) appname = appinfo['appname'] print "Using application name '%s' from apk '%s'" % (appname, options.apk) capture_name = "%s %s" % (product['name'], appinfo['appdate']) else: if not options.app_version: raise Exception("Should specify --app-version if not --apk!") # no apk, assume it's something static on the device appinfo = { 'appdate': time.strftime("%Y-%m-%d"), 'version': options.app_version } appname = product['appname'] elif options.devicetype == "b2g": if not options.sources_xml: raise Exception("Must specify --sources-xml on b2g!") devices[device_id] = { 'name': device.model } appinicontents = device.pullFile('/system/b2g/application.ini') sfh = StringIO.StringIO(appinicontents) appinfo = eideticker.get_appinfo(sfh) appinfo.update(get_revision_data(options.sources_xml)) appname = None else: print "Unknown device type '%s'!" % options.devicetype # update the device / test list for the dashboard with open(devicefile, 'w') as f: f.write(json.dumps({ 'devices': devices })) testfiledir = os.path.dirname(testfile) if not os.path.exists(testfiledir): os.mkdir(testfiledir) with open(testfile, 'w') as f: f.write(json.dumps({ 'tests': tests })) capture_area = None if options.capture_area: # we validated this previously... capture_area = json.loads(options.capture_area) # Run the test the specified number of times for i in range(num_runs): # Now run the test runtest(device, device_prefs, options.capture_device, capture_area, product, appname, appinfo, testinfo, capture_name + " #%s" % i, outputdir, datafile, data, enable_profiling=options.enable_profiling, log_http_requests=log_http_requests, baseline=options.baseline) if options.devicetype == "android": # Kill app after test complete device.killProcess(appname)
def main(args=sys.argv[1:]): usage = "usage: %prog [options] <product> <test>" parser = eideticker.TestOptionParser(usage=usage) parser.add_option("--enable-profiling", action="store_true", dest="enable_profiling", help="Create SPS profile to go along with capture") parser.add_option("--device-id", action="store", dest="device_id", help="id of device (used in output json)", default=os.environ.get('DEVICE_ID')) parser.add_option("--device-name", action="store", dest="device_name", help="name of device to display in dashboard (if not " "specified, display model name)", default=os.environ.get('DEVICE_NAME')) parser.add_option("--apk", action="store", dest="apk", help="Product apk to get metadata from " "(Android-specific)") parser.add_option("--baseline", action="store_true", dest="baseline", help="Create baseline results for dashboard") parser.add_option("--num-runs", action="store", type="int", dest="num_runs", help="number of runs (default: 1)") parser.add_option("--app-version", action="store", dest="app_version", help="Specify app version (if not automatically " "available; Android-specific)") parser.add_option("--sources-xml", action="store", dest="sources_xml", help="Path to sources XML file for getting revision " "information (B2G-specific)") parser.add_option("--output-dir", action="store", type="string", dest="outputdir", default=eideticker.DASHBOARD_DIR, help="output results to directory instead of src/dashboard") options, args = parser.parse_args() if len(args) != 2: parser.print_usage() sys.exit(1) (productname, testkey) = args num_runs = 1 if options.num_runs: num_runs = options.num_runs testinfo = eideticker.get_testinfo(testkey) device_id = options.device_id if not device_id: print "ERROR: Must specify device id (either with --device-id or with " "DEVICE_ID environment variable)" sys.exit(1) # we'll log http requests for webstartup tests only log_http_requests = False if testinfo['type'] == 'webstartup': log_http_requests = True # likewise, log actions only for web tests and b2g tests log_actions = False if testinfo['type'] == 'web' or testinfo['type'] == 'b2g': log_actions = True product = eideticker.get_product(productname) current_date = time.strftime("%Y-%m-%d") capture_name = "%s - %s (taken on %s)" % (testkey, product['name'], current_date) datafile = os.path.join(options.outputdir, device_id, '%s.json' % testkey) data = NestedDict() if os.path.isfile(datafile): data.update(json.loads(open(datafile).read())) device_prefs = eideticker.getDevicePrefs(options) device = eideticker.getDevice(**device_prefs) devices = {} devicefile = os.path.join(options.outputdir, 'devices.json') if os.path.isfile(devicefile): devices = json.loads(open(devicefile).read())['devices'] testfile = os.path.join(options.outputdir, '%s' % device_id, 'tests.json') if os.path.isfile(testfile): tests = json.loads(open(testfile).read())['tests'] else: tests = {} tests[testkey] = {'shortDesc': testinfo['shortDesc'], 'defaultMeasureId': testinfo['defaultMeasure']} device_name = options.device_name if not device_name: device_name = device.model if options.devicetype == "android": devices[device_id] = { 'name': device_name, 'version': device.getprop('ro.build.version.release')} if options.apk: if options.app_version: raise Exception("Should specify either --app-version or " "--apk, not both!") appinfo = eideticker.get_fennec_appinfo(options.apk) appname = appinfo['appname'] print "Using application name '%s' from apk '%s'" % ( appname, options.apk) capture_name = "%s %s" % (product['name'], appinfo['appdate']) else: if not options.app_version: raise Exception("Should specify --app-version if not --apk!") # no apk, assume it's something static on the device appinfo = { 'appdate': time.strftime("%Y-%m-%d"), 'version': options.app_version} appname = product['appname'] elif options.devicetype == "b2g": if not options.sources_xml: raise Exception("Must specify --sources-xml on b2g!") devices[device_id] = {'name': device_name} appinicontents = device.pullFile('/system/b2g/application.ini') sfh = StringIO.StringIO(appinicontents) appinfo = eideticker.get_appinfo(sfh) appinfo.update(get_revision_data(options.sources_xml)) appname = None else: print "Unknown device type '%s'!" % options.devicetype # copy dashboard files to output directory (if applicable) eideticker.copy_dashboard_files(options.outputdir) # update the device / test list for the dashboard with open(devicefile, 'w') as f: f.write(json.dumps({'devices': devices})) testfiledir = os.path.dirname(testfile) if not os.path.exists(testfiledir): os.mkdir(testfiledir) with open(testfile, 'w') as f: f.write(json.dumps({'tests': tests})) if options.prepare_test: eideticker.prepare_test( testkey, device_prefs, options.wifi_settings_file) # Run the test the specified number of times for i in range(num_runs): runtest(device, device_prefs, options, product, appname, appinfo, testinfo, capture_name + " #%s" % i, datafile, data, log_http_requests=log_http_requests, log_actions=log_actions)
def main(args=sys.argv[1:]): usage = "usage: %prog <test> [appname1] [appname2] ..." parser = eideticker.OptionParser(usage=usage) parser.add_option("--num-runs", action="store", type = "int", dest = "num_runs", default=1, help = "number of runs (default: 1)") parser.add_option("--output-dir", action="store", type="string", dest="outputdir", help="output results to json file") parser.add_option("--no-capture", action="store_true", dest = "no_capture", help = "run through the test, but don't actually " "capture anything") parser.add_option("--profile-file", action="store", type="string", dest = "profile_file", help="Collect a performance profile using the built in profiler.") parser.add_option("--get-internal-checkerboard-stats", action="store_true", dest="get_internal_checkerboard_stats", help="get and calculate internal checkerboard stats") parser.add_option("--startup-test", action="store_true", dest="startup_test", help="measure startup times instead of normal metrics") parser.add_option("--url-params", action="store", dest="url_params", default="", help="additional url parameters for test") parser.add_option("--use-apks", action="store_true", dest="use_apks", help="use and install android APKs as part of test (instead of specifying appnames)") parser.add_option("--date", action="store", dest="date", metavar="YYYY-MM-DD", help="get and test nightly build for date") parser.add_option("--start-date", action="store", dest="start_date", metavar="YYYY-MM-DD", help="start date for range of nightlies to test") parser.add_option("--end-date", action="store", dest="end_date", metavar="YYYY-MM-DD", help="end date for range of nightlies to test") options, args = parser.parse_args() if len(args) == 0: parser.error("Must specify at least one argument: the path to the test") dates = [] appnames = [] apks = [] if options.start_date and options.end_date and len(args) == 1: test = args[0] start_date = get_date(options.start_date) end_date = get_date(options.end_date) days=(end_date-start_date).days for numdays in range(days+1): dates.append(start_date+datetime.timedelta(days=numdays)) elif options.date and len(args) == 1: test = args[0] dates = [get_date(options.date)] elif not options.date and len(args) >= 2: test = args[0] if options.use_apks: apks = args[1:] else: appnames = args[1:] elif not options.date or (not options.start_date and not options.end_date): parser.error("Must specify date, date range, a set of appnames (e.g. org.mozilla.fennec) or a set of apks (if --use-apks is specified)") devicePrefs = eideticker.getDevicePrefs(options) device = eideticker.getDevice(**devicePrefs) if options.outputdir: outputfile = os.path.join(options.outputdir, "metric-test-%s.json" % time.time()) else: outputfile = None if appnames: for appname in appnames: run_test(device, options.outputdir, outputfile, test, options.url_params, options.num_runs, options.startup_test, options.no_capture, options.get_internal_checkerboard_stats, appname=appname, profile_file=options.profile_file, **devicePrefs) elif apks: for apk in apks: run_test(device, options.outputdir, outputfile, test, options.url_params, options.num_runs, options.startup_test, options.no_capture, options.get_internal_checkerboard_stats, apk=apk, profile_file=options.profile_file, **devicePrefs) else: for date in dates: apk = get_build_for_date(date) run_test(device, options.outputdir, outputfile, test, options.url_params, options.num_runs, options.startup_test, options.no_capture, options.get_internal_checkerboard_stats, apk=apk, appdate=date, profile_file=options.profile_file, **devicePrefs)
def main(args=sys.argv[1:]): usage = "usage: %prog [options] <appname> <test path>" parser = eideticker.OptionParser(usage=usage) parser.add_option("--url-params", action="store", dest="url_params", help="additional url parameters for test") parser.add_option("--name", action="store", type="string", dest="capture_name", help="name to give capture") parser.add_option("--capture-file", action="store", type="string", dest="capture_file", help="name to give to capture file") parser.add_option("--no-capture", action="store_true", dest="no_capture", help="run through the test, but don't actually " "capture anything") parser.add_option("--checkerboard-log-file", action="store", type="string", dest="checkerboard_log_file", help="name to give checkerboarding stats file") parser.add_option("--startup-test", action="store_true", dest="startup_test", help="do a startup test: full capture, no actions") parser.add_option( "--b2g", action="store_true", dest="b2g", default=False, help="Run in B2G environment. You do not need to pass an appname") parser.add_option( "--profile-file", action="store", type="string", dest="profile_file", help="Collect a performance profile using the built in profiler.") options, args = parser.parse_args() testpath, appname = None, None if options.b2g: if len(args) != 1: parser.error("incorrect number of arguments") sys.exit(1) testpath = args[0] else: if len(args) != 2: parser.error("incorrect number of arguments") sys.exit(1) (appname, testpath) = args # Tests must be in src/tests/... unless it is a startup test and the # path is about:home (indicating we want to measure startup to the # home screen) if options.startup_test and testpath == "about:home": testpath_rel = testpath capture_timeout = 5 # 5 seconds to wait for fennec to start after it claims to have started else: capture_timeout = None try: testpath_rel = os.path.abspath(testpath).split(TEST_DIR)[1][1:] except: print "Test must be relative to %s" % TEST_DIR sys.exit(1) actions = None if not options.startup_test: actions_path = os.path.join(os.path.dirname(testpath), "actions.json") try: with open(actions_path) as f: actions = json.loads(f.read()) except EnvironmentError: print "Couldn't open actions file '%s'" % actions_path sys.exit(1) if not os.path.exists(EIDETICKER_TEMP_DIR): os.mkdir(EIDETICKER_TEMP_DIR) if not os.path.isdir(EIDETICKER_TEMP_DIR): print "Could not open eideticker temporary directory" sys.exit(1) capture_name = options.capture_name if not capture_name: capture_name = testpath_rel capture_file = options.capture_file if not capture_file and not options.no_capture: capture_file = os.path.join( CAPTURE_DIR, "capture-%s.zip" % datetime.datetime.now().isoformat()) # Create a device object to interface with the phone devicePrefs = eideticker.getDevicePrefs(options) device = eideticker.getDevice(**devicePrefs) if appname and device.processExist(appname): print "An instance of %s is running. Please stop it before running Eideticker." % appname sys.exit(1) # If we're logging checkerboard stats, set that up here (seems like it # takes a second or so to accept the new setting, so let's do that here -- # ideally we would detect when that's working, but I'm not sure how to do # so trivially) if options.checkerboard_log_file: old_log_val = device.getprop("log.tag.GeckoLayerRendererProf") device.setprop("log.tag.GeckoLayerRendererProf", "DEBUG") print "Creating webserver..." capture_metadata = { 'name': capture_name, 'testpath': testpath_rel, 'app': appname, 'device': device.model, 'startupTest': options.startup_test } # something of a hack. if profiling is enabled, carve off an area to # ignore in the capture if options.profile_file: capture_metadata['ignoreAreas'] = [[0, 0, 3 * 64, 3]] capture_server = CaptureServer(capture_metadata, capture_file, options.checkerboard_log_file, capture_controller, device, actions) host = mozhttpd.iface.get_lan_ip() http = mozhttpd.MozHttpd(docroot=TEST_DIR, host=host, port=0, urlhandlers=[{ 'method': 'GET', 'path': '/api/captures/start/?', 'function': capture_server.start_capture }, { 'method': 'GET', 'path': '/api/captures/end/?', 'function': capture_server.end_capture }, { 'method': 'POST', 'path': '/api/captures/input/?', 'function': capture_server.input }]) http.start(block=False) connected = False tries = 0 while not connected and tries < 20: tries += 1 import socket s = socket.socket() try: s.connect((host, http.httpd.server_port)) connected = True except Exception: print "Can't connect to %s:%s, retrying..." % ( host, http.httpd.server_port) if not connected: print "Could not open webserver. Error!" sys.exit(1) # note: url params for startup tests currently not supported if options.url_params: testpath_rel += "?%s" % urllib.quote_plus(options.url_params) if options.startup_test: if testpath == "about:home": url = testpath else: url = "http://%s:%s/%s?startup_test=1" % ( host, http.httpd.server_port, testpath_rel) else: url = "http://%s:%s/start.html?testpath=%s" % ( host, http.httpd.server_port, testpath_rel) print "Test URL is: %s" % url if options.b2g: runner = eideticker.B2GRunner(device, url, EIDETICKER_TEMP_DIR) else: runner = eideticker.BrowserRunner(device, appname, url) # FIXME: currently start capture before launching app because we wait until app is # launched -- would be better to make waiting optional and then start capture # after triggering app launch to reduce latency? if options.startup_test and not options.no_capture: capture_controller.start_capture(capture_file, device.hdmiResolution, capture_metadata) runner.start(profile_file=options.profile_file) # Keep on capturing until we timeout if capture_timeout: timeout = capture_timeout else: timeout = 100 timer = 0 interval = 0.1 try: while not capture_server.finished and timer < timeout: time.sleep(interval) timer += interval except KeyboardInterrupt: print "Aborting" runner.stop() capture_server.terminate_capture() sys.exit(1) if capture_timeout and not capture_server.finished: capture_server.terminate_capture() elif not capture_server.finished: print "Did not finish test! Error!" runner.stop() capture_server.terminate_capture() sys.exit(1) runner.stop() if capture_file: print "Converting capture..." try: capture_controller.convert_capture(capture_server.start_frame, capture_server.end_frame) except KeyboardInterrupt: print "Aborting" sys.exit(1) # Clean up checkerboard logging preferences if options.checkerboard_log_file: device.setprop("log.tag.GeckoLayerRendererProf", old_log_val)
def main(args=sys.argv[1:]): usage = "usage: %prog [options] TEST..." parser = eideticker.TestOptionParser(usage=usage) eideticker.add_dashboard_options(parser) parser.add_option("--enable-profiling", action="store_true", dest="enable_profiling", help="Create SPS profile to go along with capture") parser.add_option("--dashboard-id", action="store", dest="dashboard_id", help="id of dashboard (used in output json)", default=os.environ.get('DASHBOARD_ID')) parser.add_option("--dashboard-name", action="store", dest="dashboard_name", help="name of dashboard to display", default=os.environ.get('DASHBOARD_NAME')) parser.add_option("--device-id", action="store", dest="device_id", help="id of device (used in output json)", default=os.environ.get('DEVICE_ID')) parser.add_option("--branch", action="store", dest="branch_id", help="branch under test (used in output json)", default=os.environ.get('BRANCH')) parser.add_option("--device-name", action="store", dest="device_name", help="name of device to display in dashboard (if not " "specified, display model name)", default=os.environ.get('DEVICE_NAME')) parser.add_option("--apk", action="store", dest="apk", help="Product apk to get metadata from " "(Android-specific)") parser.add_option("--baseline", action="store_true", dest="baseline", help="Create baseline results for dashboard") parser.add_option("--num-runs", action="store", type="int", dest="num_runs", help="number of runs (default: %default)", default=1) parser.add_option("--app-version", action="store", dest="app_version", help="Specify app version (if not automatically " "available; Android-specific)") parser.add_option("--sources-xml", action="store", dest="sources_xml", help="Path to sources XML file for getting revision " "information (B2G-specific)") parser.add_option("--product", action="store", type="string", dest="product_name", default="nightly", help="product name (android-specific, default: " "%default)") options, args = parser.parse_args() if not args: # need to specify at least one test to run! parser.print_usage() sys.exit(1) if not options.dashboard_id: parser.error("Must specify dashboard id (either with --dashboard-id " "or with DASHBOARD_ID environment variable)") if not options.dashboard_name: parser.error("Must specify dashboard name (either with " "--dashboard-name or with DASHBOARD_NAME environment " "varaiable)") if not options.device_id: parser.error("Must specify device id (either with --device-id or with " "DEVICE_ID environment variable)") if not options.branch_id: parser.error("Must specify branch (either with --branch or with " "BRANCH environment variable)") # get device info device_prefs = eideticker.getDevicePrefs(options) device = eideticker.getDevice(**device_prefs) device_name = options.device_name if not device_name: device_name = device.model # copy dashboard files to output directory (if applicable) eideticker.copy_dashboard_files(options.dashboard_dir) if options.devicetype == 'android': product = eideticker.get_product(options.product_name) device_info = { 'name': device_name, 'version': device.getprop('ro.build.version.release') } elif options.devicetype == 'b2g': product = eideticker.get_product('b2g-nightly') device_info = {'name': device_name} else: print "ERROR: Unknown device type '%s'" % options.devicetype # update dashboard / device index eideticker.update_dashboard_list(options.dashboard_dir, options.dashboard_id, options.dashboard_name) eideticker.update_dashboard_device_list(options.dashboard_dir, options.dashboard_id, options.device_id, options.branch_id, device_info) # get application/build info if options.devicetype == "android": if options.apk: if options.app_version: raise Exception("Should specify either --app-version or " "--apk, not both!") appinfo = eideticker.get_fennec_appinfo(options.apk) options.appname = appinfo['appname'] print "Using application name '%s' from apk '%s'" % ( options.appname, options.apk) options.capture_name = "%s %s" % (product['name'], appinfo['appdate']) else: if not options.app_version: raise Exception("Should specify --app-version if not --apk!") # no apk, assume it's something static on the device appinfo = { 'appdate': time.strftime("%Y-%m-%d"), 'version': options.app_version } elif options.devicetype == "b2g": if not options.sources_xml: raise Exception("Must specify --sources-xml on b2g!") appinicontents = device.pullFile('/system/b2g/application.ini') sfh = StringIO.StringIO(appinicontents) appinfo = eideticker.get_appinfo(sfh) appinfo.update(get_revision_data(options.sources_xml)) options.appname = None else: print "Unknown device type '%s'!" % options.devicetype # run through the tests... failed_tests = [] for testkey in args: testinfo = eideticker.get_testinfo(testkey) eideticker.update_dashboard_test_list(options.dashboard_dir, options.dashboard_id, options.device_id, options.branch_id, testinfo) current_date = time.strftime("%Y-%m-%d") options.capture_name = "%s - %s (taken on %s)" % ( testkey, product['name'], current_date) if options.prepare_test: eideticker.prepare_test(testkey, options) # Run the test the specified number of times for i in range(options.num_runs): try: runtest(device, device_prefs, options, product, appinfo, testinfo, options.capture_name + " #%s" % i) except eideticker.TestException: print "Unable to run test '%s'. Skipping and continuing." % testkey failed_tests.append(testkey) break # synchronize with dashboard (if we have a server to upload to) if options.dashboard_server: eideticker.upload_dashboard(options) else: print "No dashboard server specified. Skipping upload." if failed_tests: print "The following tests failed: %s" % ", ".join(failed_tests) sys.exit(1)
def main(args=sys.argv[1:]): usage = "usage: %prog [options] <test> <output dir>" parser = eideticker.OptionParser(usage=usage) parser.add_option("--enable-profiling", action="store_true", dest="enable_profiling", help="Create SPS profile to go along with capture") parser.add_option("--device-id", action="store", dest="device_id", help="id of device (used in output json)") parser.add_option( "--product", action="store", dest="product", help="Restrict testing to product (options: %s)" % ", ".join([product["name"] for product in default_products])) parser.add_option("--num-runs", action="store", type="int", dest="num_runs", help="number of runs (default: 1)") options, args = parser.parse_args() if len(args) != 2: parser.error("incorrect number of arguments") (testname, outputdir) = args num_runs = 1 if options.num_runs: num_runs = options.num_runs testnames = [test["name"] for test in default_tests] if testname not in testnames: print "ERROR: No tests matching '%s' (options: %s)" % ( testname, ", ".join(testnames)) sys.exit(1) else: test = [test for test in default_tests if test['name'] == testname][0] device_id = options.device_id if not device_id: device_id = os.environ.get('DEVICE_ID') if not device_id: print "ERROR: Must specify device id (either with --device-id or with DEVICE_ID environment variable)" sys.exit(1) products = default_products if options.product: products = [ product for product in default_products if product['name'] == options.product ] if not products: print "ERROR: No products matching '%s'" % options.product sys.exit(1) current_date = time.strftime("%Y-%m-%d") datafile = os.path.join(outputdir, 'data-%s.json' % device_id) data = NestedDict() if os.path.isfile(datafile): data.update(json.loads(open(datafile).read())) devicePrefs = eideticker.getDevicePrefs(options) device = eideticker.getDevice(**devicePrefs) # update the device list for the dashboard devices = {} devicefile = os.path.join(outputdir, 'devices.json') if os.path.isfile(devicefile): devices = json.loads(open(devicefile).read())['devices'] devices[device_id] = { 'name': device.model, 'version': device.getprop('ro.build.version.release') } with open(devicefile, 'w') as f: f.write(json.dumps({'devices': devices})) for product in products: if product.get('url'): product_fname = os.path.join(DOWNLOAD_DIR, "%s.apk" % product['name']) appinfo = eideticker.get_fennec_appinfo(product_fname) appname = appinfo['appname'] print "Using application name '%s' from apk '%s'" % (appname, product_fname) capture_name = "%s %s" % (product['name'], appinfo['date']) else: appinfo = {} appname = product['appname'] capture_name = "%s (taken on %s)" % (product['name'], current_date) if appinfo.get('appname'): appname = appinfo['appname'] else: appname = product['appname'] # Run the test the specified number of times for i in range(num_runs): # Now run the test runtest(device, product, current_date, appname, appinfo, test, capture_name + " #%s" % i, outputdir, datafile, data, enable_profiling=options.enable_profiling, **devicePrefs) # Kill app after test complete device.killProcess(appname)
def main(args=sys.argv[1:]): usage = "usage: %prog [options] TEST..." parser = eideticker.TestOptionParser(usage=usage) eideticker.add_dashboard_options(parser) parser.add_option("--enable-profiling", action="store_true", dest="enable_profiling", help="Create SPS profile to go along with capture") parser.add_option("--device-id", action="store", dest="device_id", help="id of device (used in output json)", default=os.environ.get('DEVICE_ID')) parser.add_option("--branch", action="store", dest="branch_id", help="branch under test (used in output json)", default=os.environ.get('BRANCH')) parser.add_option("--device-name", action="store", dest="device_name", help="name of device to display in dashboard (if not " "specified, display model name)", default=os.environ.get('DEVICE_NAME')) parser.add_option("--apk", action="store", dest="apk", help="Product apk to get metadata from " "(Android-specific)") parser.add_option("--baseline", action="store_true", dest="baseline", help="Create baseline results for dashboard") parser.add_option("--num-runs", action="store", type="int", dest="num_runs", help="number of runs (default: %default)", default=1) parser.add_option("--app-version", action="store", dest="app_version", help="Specify app version (if not automatically " "available; Android-specific)") parser.add_option("--sources-xml", action="store", dest="sources_xml", help="Path to sources XML file for getting revision " "information (B2G-specific)") parser.add_option("--product", action="store", type="string", dest="product_name", default="nightly", help="product name (android-specific, default: " "%default)") options, args = parser.parse_args() if not args: # need to specify at least one test to run! parser.print_usage() sys.exit(1) if not options.device_id: print "ERROR: Must specify device id (either with --device-id or with " \ "DEVICE_ID environment variable)" sys.exit(1) if not options.branch_id: print "ERROR: Must specify branch (either with --branch or with " \ "BRANCH environment variable)" sys.exit(1) # get device info device_prefs = eideticker.getDevicePrefs(options) device = eideticker.getDevice(**device_prefs) device_name = options.device_name if not device_name: device_name = device.model # copy dashboard files to output directory (if applicable) eideticker.copy_dashboard_files(options.dashboard_dir) if options.devicetype == 'android': product = eideticker.get_product(options.product_name) device_info = { 'name': device_name, 'version': device.getprop('ro.build.version.release')} elif options.devicetype == 'b2g': product = eideticker.get_product('b2g-nightly') device_info = { 'name': device_name } else: print "ERROR: Unknown device type '%s'" % options.devicetype # update device index eideticker.update_dashboard_device_list(options.dashboard_dir, options.device_id, options.branch_id, device_info) # get application/build info if options.devicetype == "android": if options.apk: if options.app_version: raise Exception("Should specify either --app-version or " "--apk, not both!") appinfo = eideticker.get_fennec_appinfo(options.apk) options.appname = appinfo['appname'] print "Using application name '%s' from apk '%s'" % ( options.appname, options.apk) options.capture_name = "%s %s" % (product['name'], appinfo['appdate']) else: if not options.app_version: raise Exception("Should specify --app-version if not --apk!") # no apk, assume it's something static on the device appinfo = { 'appdate': time.strftime("%Y-%m-%d"), 'version': options.app_version} elif options.devicetype == "b2g": if not options.sources_xml: raise Exception("Must specify --sources-xml on b2g!") appinicontents = device.pullFile('/system/b2g/application.ini') sfh = StringIO.StringIO(appinicontents) appinfo = eideticker.get_appinfo(sfh) appinfo.update(get_revision_data(options.sources_xml)) options.appname = None else: print "Unknown device type '%s'!" % options.devicetype # run through the tests... failed_tests = [] for testkey in args: testinfo = eideticker.get_testinfo(testkey) eideticker.update_dashboard_test_list(options.dashboard_dir, options.device_id, options.branch_id, testinfo) current_date = time.strftime("%Y-%m-%d") options.capture_name = "%s - %s (taken on %s)" % (testkey, product['name'], current_date) if options.prepare_test: eideticker.prepare_test(testkey, options) # Run the test the specified number of times for i in range(options.num_runs): try: runtest(device, device_prefs, options, product, appinfo, testinfo, options.capture_name + " #%s" % i) except eideticker.TestException: print "Unable to run test '%s'. Skipping and continuing." % testkey failed_tests.append(testkey) break # synchronize with dashboard (if we have a server to upload to) if options.dashboard_server: eideticker.upload_dashboard(options) else: print "No dashboard server specified. Skipping upload." if failed_tests: print "The following tests failed: %s" % ", ".join(failed_tests) sys.exit(1)
def main(args=sys.argv[1:]): usage = "usage: %prog [options] <appname> <test path>" parser = eideticker.OptionParser(usage=usage) parser.add_option("--url-params", action="store", dest="url_params", help="additional url parameters for test") parser.add_option("--name", action="store", type = "string", dest = "capture_name", help = "name to give capture") parser.add_option("--capture-file", action="store", type = "string", dest = "capture_file", help = "name to give to capture file") parser.add_option("--no-capture", action="store_true", dest = "no_capture", help = "run through the test, but don't actually " "capture anything") parser.add_option("--checkerboard-log-file", action="store", type = "string", dest = "checkerboard_log_file", help = "name to give checkerboarding stats file") parser.add_option("--startup-test", action="store_true", dest="startup_test", help="do a startup test: full capture, no actions") parser.add_option("--b2g", action="store_true", dest="b2g", default=False, help="Run in B2G environment. You do not need to pass an appname") parser.add_option("--gesture-file", action="store", dest="gesture_file", default=None, help="Path to gesture file on the phone. Will execute this file after test URL loads.") parser.add_option("--profile-file", action="store", type="string", dest = "profile_file", help="Collect a performance profile using the built in profiler.") options, args = parser.parse_args() testpath, appname = None, None if options.b2g: if len(args) != 1: parser.error("incorrect number of arguments") sys.exit(1) testpath = args[0] else: if len(args) != 2: parser.error("incorrect number of arguments") sys.exit(1) (appname, testpath) = args # Tests must be in src/tests/... unless it is a startup test and the # path is about:home (indicating we want to measure startup to the # home screen) if options.startup_test and testpath == "about:home": testpath_rel = testpath capture_timeout = 5 # 5 seconds to wait for fennec to start after it claims to have started else: capture_timeout = None try: testpath_rel = os.path.abspath(testpath).split(TEST_DIR)[1][1:] except: print "Test must be relative to %s" % TEST_DIR sys.exit(1) actions = None if not options.startup_test: actions_path = os.path.join(os.path.dirname(testpath), "actions.json") try: with open(actions_path) as f: actions = json.loads(f.read()) except EnvironmentError: print "Couldn't open actions file '%s'" % actions_path sys.exit(1) if not os.path.exists(EIDETICKER_TEMP_DIR): os.mkdir(EIDETICKER_TEMP_DIR) if not os.path.isdir(EIDETICKER_TEMP_DIR): print "Could not open eideticker temporary directory" sys.exit(1) capture_name = options.capture_name if not capture_name: capture_name = testpath_rel capture_file = options.capture_file if not capture_file and not options.no_capture: capture_file = os.path.join(CAPTURE_DIR, "capture-%s.zip" % datetime.datetime.now().isoformat()) # Create a device object to interface with the phone device = eideticker.getDevice(options) if appname and device.processExist(appname): print "An instance of %s is running. Please stop it before running Eideticker." % appname sys.exit(1) # If we're logging checkerboard stats, set that up here (seems like it # takes a second or so to accept the new setting, so let's do that here -- # ideally we would detect when that's working, but I'm not sure how to do # so trivially) if options.checkerboard_log_file: old_log_val = device.getprop("log.tag.GeckoLayerRendererProf") device.setprop("log.tag.GeckoLayerRendererProf", "DEBUG") print "Creating webserver..." capture_metadata = { 'name': capture_name, 'testpath': testpath_rel, 'app': appname, 'device': device.model, 'startupTest': options.startup_test } capture_server = CaptureServer(capture_metadata, capture_file, options.checkerboard_log_file, capture_controller, device, actions, options.gesture_file) host = mozhttpd.iface.get_lan_ip() http = mozhttpd.MozHttpd(docroot=TEST_DIR, host=host, port=0, urlhandlers = [ { 'method': 'GET', 'path': '/api/captures/start/?', 'function': capture_server.start_capture }, { 'method': 'GET', 'path': '/api/captures/end/?', 'function': capture_server.end_capture }, { 'method': 'POST', 'path': '/api/captures/input/?', 'function': capture_server.input } ]) http.start(block=False) connected = False tries = 0 while not connected and tries < 20: tries+=1 import socket s = socket.socket() try: s.connect((host, http.httpd.server_port)) connected = True except Exception: print "Can't connect to %s:%s, retrying..." % (host, http.httpd.server_port) if not connected: print "Could not open webserver. Error!" sys.exit(1) # note: url params for startup tests currently not supported if options.url_params: testpath_rel += "?%s" % urllib.quote_plus(options.url_params) if options.startup_test: if testpath == "about:home": url = testpath else: url = "http://%s:%s/%s?startup_test=1" % (host, http.httpd.server_port, testpath_rel) else: url = "http://%s:%s/start.html?testpath=%s" % (host, http.httpd.server_port, testpath_rel) print "Test URL is: %s" % url if options.b2g: runner = eideticker.B2GRunner(device, url, EIDETICKER_TEMP_DIR) else: runner = eideticker.BrowserRunner(device, appname, url) # FIXME: currently start capture before launching app because we wait until app is # launched -- would be better to make waiting optional and then start capture # after triggering app launch to reduce latency? if options.startup_test and not options.no_capture: capture_controller.start_capture(capture_file, device.hdmiResolution, capture_metadata) runner.start(profile_file=options.profile_file) # Keep on capturing until we timeout if capture_timeout: timeout = capture_timeout else: timeout = 100 timer = 0 interval = 0.1 try: while not capture_server.finished and timer < timeout: time.sleep(interval) timer += interval except KeyboardInterrupt: print "Aborting" runner.stop() capture_server.terminate_capture() sys.exit(1) if capture_timeout and not capture_server.finished: capture_server.terminate_capture() elif not capture_server.finished: print "Did not finish test! Error!" runner.stop() capture_server.terminate_capture() sys.exit(1) runner.stop() if capture_file: print "Converting capture..." try: capture_controller.convert_capture(capture_server.start_frame, capture_server.end_frame) except KeyboardInterrupt: print "Aborting" sys.exit(1) # Clean up checkerboard logging preferences if options.checkerboard_log_file: device.setprop("log.tag.GeckoLayerRendererProf", old_log_val)
def main(args=sys.argv[1:]): usage = "usage: %prog <test> [appname1] [appname2] ..." parser = eideticker.OptionParser(usage=usage) parser.add_option("--num-runs", action="store", type="int", dest="num_runs", default=1, help="number of runs (default: 1)") parser.add_option("--output-dir", action="store", type="string", dest="outputdir", help="output results to json file") parser.add_option("--no-capture", action="store_true", dest="no_capture", help="run through the test, but don't actually " "capture anything") parser.add_option( "--profile-file", action="store", type="string", dest="profile_file", help="Collect a performance profile using the built in profiler.") parser.add_option("--get-internal-checkerboard-stats", action="store_true", dest="get_internal_checkerboard_stats", help="get and calculate internal checkerboard stats") parser.add_option("--startup-test", action="store_true", dest="startup_test", help="measure startup times instead of normal metrics") parser.add_option("--url-params", action="store", dest="url_params", default="", help="additional url parameters for test") parser.add_option( "--use-apks", action="store_true", dest="use_apks", help= "use and install android APKs as part of test (instead of specifying appnames)" ) parser.add_option("--date", action="store", dest="date", metavar="YYYY-MM-DD", help="get and test nightly build for date") parser.add_option("--start-date", action="store", dest="start_date", metavar="YYYY-MM-DD", help="start date for range of nightlies to test") parser.add_option("--end-date", action="store", dest="end_date", metavar="YYYY-MM-DD", help="end date for range of nightlies to test") options, args = parser.parse_args() if len(args) == 0: parser.error( "Must specify at least one argument: the path to the test") dates = [] appnames = [] apks = [] if options.start_date and options.end_date and len(args) == 1: test = args[0] start_date = get_date(options.start_date) end_date = get_date(options.end_date) days = (end_date - start_date).days for numdays in range(days + 1): dates.append(start_date + datetime.timedelta(days=numdays)) elif options.date and len(args) == 1: test = args[0] dates = [get_date(options.date)] elif not options.date and len(args) >= 2: test = args[0] if options.use_apks: apks = args[1:] else: appnames = args[1:] elif not options.date or (not options.start_date and not options.end_date): parser.error( "Must specify date, date range, a set of appnames (e.g. org.mozilla.fennec) or a set of apks (if --use-apks is specified)" ) devicePrefs = eideticker.getDevicePrefs(options) device = eideticker.getDevice(**devicePrefs) if options.outputdir: outputfile = os.path.join(options.outputdir, "metric-test-%s.json" % time.time()) else: outputfile = None if appnames: for appname in appnames: run_test(device, options.outputdir, outputfile, test, options.url_params, options.num_runs, options.startup_test, options.no_capture, options.get_internal_checkerboard_stats, appname=appname, profile_file=options.profile_file, **devicePrefs) elif apks: for apk in apks: run_test(device, options.outputdir, outputfile, test, options.url_params, options.num_runs, options.startup_test, options.no_capture, options.get_internal_checkerboard_stats, apk=apk, profile_file=options.profile_file, **devicePrefs) else: for date in dates: apk = get_build_for_date(date) run_test(device, options.outputdir, outputfile, test, options.url_params, options.num_runs, options.startup_test, options.no_capture, options.get_internal_checkerboard_stats, apk=apk, appdate=date, profile_file=options.profile_file, **devicePrefs)
def main(args=sys.argv[1:]): usage = "usage: %prog [options] <test> <output dir>" parser = eideticker.OptionParser(usage=usage) parser.add_option("--enable-profiling", action="store_true", dest = "enable_profiling", help = "Create SPS profile to go along with capture") parser.add_option("--device-id", action="store", dest="device_id", help="id of device (used in output json)") parser.add_option("--product", action="store", dest="product", help = "Restrict testing to product (options: %s)" % ", ".join([product["name"] for product in default_products])) parser.add_option("--num-runs", action="store", type = "int", dest = "num_runs", help = "number of runs (default: 1)") options, args = parser.parse_args() if len(args) != 2: parser.error("incorrect number of arguments") (testname, outputdir) = args num_runs = 1 if options.num_runs: num_runs = options.num_runs testnames = [test["name"] for test in default_tests] if testname not in testnames: print "ERROR: No tests matching '%s' (options: %s)" % (testname, ", ".join(testnames)) sys.exit(1) else: test = [test for test in default_tests if test['name'] == testname][0] device_id = options.device_id if not device_id: device_id = os.environ.get('DEVICE_ID') if not device_id: print "ERROR: Must specify device id (either with --device-id or with DEVICE_ID environment variable)" sys.exit(1) products = default_products if options.product: products = [product for product in default_products if product['name'] == options.product] if not products: print "ERROR: No products matching '%s'" % options.product sys.exit(1) current_date = time.strftime("%Y-%m-%d") datafile = os.path.join(outputdir, 'data-%s.json' % device_id) data = NestedDict() if os.path.isfile(datafile): data.update(json.loads(open(datafile).read())) devicePrefs = eideticker.getDevicePrefs(options) device = eideticker.getDevice(**devicePrefs) # update the device list for the dashboard devices = {} devicefile = os.path.join(outputdir, 'devices.json') if os.path.isfile(devicefile): devices = json.loads(open(devicefile).read())['devices'] devices[device_id] = { 'name': device.model, 'version': device.getprop('ro.build.version.release') } with open(devicefile, 'w') as f: f.write(json.dumps({ 'devices': devices })) for product in products: if product.get('url'): product_fname = os.path.join(DOWNLOAD_DIR, "%s.apk" % product['name']) appinfo = eideticker.get_fennec_appinfo(product_fname) appname = appinfo['appname'] print "Using application name '%s' from apk '%s'" % (appname, product_fname) capture_name = "%s %s" % (product['name'], appinfo['date']) else: appinfo = { } appname = product['appname'] capture_name = "%s (taken on %s)" % (product['name'], current_date) if appinfo.get('appname'): appname = appinfo['appname'] else: appname = product['appname'] # Run the test the specified number of times for i in range(num_runs): # Now run the test runtest(device, product, current_date, appname, appinfo, test, capture_name + " #%s" % i, outputdir, datafile, data, enable_profiling=options.enable_profiling, **devicePrefs) # Kill app after test complete device.killProcess(appname)
def runtest(device_prefs, testname, options, apk=None, appname=None, appdate=None): if apk: appinfo = eideticker.get_fennec_appinfo(apk) appname = appinfo['appname'] print "Installing %s (version: %s, revision %s)" % ( appinfo['appname'], appinfo['version'], appinfo['revision']) device = eideticker.getDevice(**device_prefs) device.updateApp(apk) else: appinfo = None options.appname = appname testinfo = eideticker.get_testinfo(testname) stableframecapture = (testinfo['type'] in ('startup', 'webstartup') or testinfo['defaultMeasure'] == 'timetostableframe') capture_results = [] if options.prepare_test: eideticker.prepare_test(testname, options) for i in range(options.num_runs): # Now run the test curtime = int(time.time()) capture_file = os.path.join(CAPTURE_DIR, "metric-test-%s-%s.zip" % (appname, curtime)) if options.enable_profiling and options.outputdir: profile_relpath = os.path.join( 'profiles', 'sps-profile-%s.zip' % time.time()) profile_filename = os.path.join(options.outputdir, profile_relpath) else: profile_filename = None current_date = time.strftime("%Y-%m-%d") capture_name = "%s - %s (taken on %s)" % ( testname, appname, current_date) testlog = eideticker.run_test(testname, options, capture_filename=capture_file, profile_filename=profile_filename, capture_name=capture_name) capture_uuid = uuid.uuid1().hex datapoint = { 'uuid': capture_uuid } metadata = {} metrics = {} if options.capture: capture = videocapture.Capture(capture_file) datapoint['captureFile'] = metadata['captureFile'] = capture_file metadata['captureFPS'] = capture.fps metadata['generatedVideoFPS'] = capture.generated_video_fps if stableframecapture: metrics['timetostableframe'] = \ eideticker.get_stable_frame_time(capture) else: metrics.update( eideticker.get_standard_metrics(capture, testlog.actions)) metadata['metrics'] = metrics metadata.update(eideticker.get_standard_metric_metadata(capture)) if options.outputdir: # video video_relpath = os.path.join( 'videos', 'video-%s.webm' % time.time()) video_path = os.path.join(options.outputdir, video_relpath) open(video_path, 'w').write(capture.get_video().read()) metadata['video'] = video_relpath if options.log_checkerboard_stats: metrics['internalcheckerboard'] = \ testlog.checkerboard_percent_totals # Want metrics data in data, so we can graph everything at once datapoint.update(metrics) if options.enable_profiling: metadata['profile'] = profile_filename # dump metadata if options.outputdir: # metadata metadata_path = os.path.join(options.outputdir, 'metadata', '%s.json' % capture_uuid) open(metadata_path, 'w').write(json.dumps(metadata)) capture_results.append(datapoint) if options.devicetype == "b2g": # FIXME: get information from sources.xml and application.ini on # device, as we do in update-dashboard.py display_key = appkey = "FirefoxOS" else: appkey = appname if appdate: appkey = appdate.isoformat() else: appkey = appname if appinfo and appinfo.get('revision'): display_key = "%s (%s)" % (appkey, appinfo['revision']) else: display_key = appkey print "=== Results on %s for %s ===" % (testname, display_key) if options.capture: measures = [ ('timetostableframe', 'Times to first stable frame (seconds)'), ('uniqueframes', 'Number of unique frames'), ('fps', 'Average number of unique frames per second'), ('overallentropy', 'Overall entropy over length of capture'), ('checkerboard', 'Checkerboard area/duration (sum of percents NOT ' 'percentage)'), ('timetoresponse', 'Time to first input response (seconds)') ] for measure in measures: if capture_results[0].get(measure[0]): print " %s:" % measure[1] print " %s" % map(lambda c: c[measure[0]], capture_results) print print " Capture files:" print " Capture files: %s" % map(lambda c: c['captureFile'], capture_results) print if options.log_checkerboard_stats: print " Internal Checkerboard Stats (sum of percents, not " "percentage):" print " %s" % map( lambda c: c['internalcheckerboard'], capture_results) print if options.outputdir: outputfile = os.path.join(options.outputdir, "metric.json") resultdict = {'title': testname, 'data': {}} if os.path.isfile(outputfile): resultdict.update(json.loads(open(outputfile).read())) if not resultdict['data'].get(appkey): resultdict['data'][appkey] = [] resultdict['data'][appkey].extend(capture_results) with open(outputfile, 'w') as f: f.write(json.dumps(resultdict))
def main(args=sys.argv[1:]): usage = "usage: %prog <test> [appname1] [appname2] ..." parser = eideticker.OptionParser(usage=usage) parser.add_option("--num-runs", action="store", type = "int", dest = "num_runs", default=1, help = "number of runs (default: 1)") parser.add_option("--output-dir", action="store", type="string", dest="outputdir", help="output results to json file") parser.add_option("--no-capture", action="store_true", dest = "no_capture", help = "run through the test, but don't actually " "capture anything") parser.add_option("--enable-profiling", action="store_true", dest = "enable_profiling", help="Collect performance profiles using the built in profiler.") parser.add_option("--get-internal-checkerboard-stats", action="store_true", dest="get_internal_checkerboard_stats", help="get and calculate internal checkerboard stats") parser.add_option("--startup-test", action="store_true", dest="startup_test", help="measure startup times instead of normal metrics") parser.add_option("--url-params", action="store", dest="url_params", default="", help="additional url parameters for test") parser.add_option("--extra-prefs", action="store", dest="extra_prefs", default="{}", help="Extra profile preference for Firefox browsers. " \ "Must be passed in as a JSON dictionary") parser.add_option("--use-apks", action="store_true", dest="use_apks", help="use and install android APKs as part of test (instead of specifying appnames)") parser.add_option("--date", action="store", dest="date", metavar="YYYY-MM-DD", help="get and test nightly build for date") parser.add_option("--start-date", action="store", dest="start_date", metavar="YYYY-MM-DD", help="start date for range of nightlies to test") parser.add_option("--end-date", action="store", dest="end_date", metavar="YYYY-MM-DD", help="end date for range of nightlies to test") options, args = parser.parse_args() if len(args) == 0: parser.error("Must specify at least one argument: the path to the test") try: # we only need to validate extra_prefs, as we'll just be passing it down # to runtest json.loads(options.extra_prefs) except ValueError: parser.error("Error processing extra preferences: not valid JSON!") raise dates = [] appnames = [] apks = [] if options.start_date and options.end_date and len(args) == 1: test = args[0] start_date = eideticker.BuildRetriever.get_date(options.start_date) end_date = eideticker.BuildRetriever.get_date(options.end_date) days=(end_date-start_date).days for numdays in range(days+1): dates.append(start_date+datetime.timedelta(days=numdays)) elif options.date and len(args) == 1: test = args[0] dates = [eideticker.BuildRetriever.get_date(options.date)] elif not options.date and len(args) >= 2: test = args[0] if options.use_apks: apks = args[1:] else: appnames = args[1:] elif not options.date or (not options.start_date and not options.end_date): parser.error("Must specify date, date range, a set of appnames (e.g. org.mozilla.fennec) or a set of apks (if --use-apks is specified)") devicePrefs = eideticker.getDevicePrefs(options) device = eideticker.getDevice(**devicePrefs) if options.outputdir: outputfile = os.path.join(options.outputdir, "metric-test-%s.json" % time.time()) else: outputfile = None if appnames: for appname in appnames: run_test(device, options.outputdir, outputfile, test, options.url_params, options.num_runs, options.startup_test, options.no_capture, options.get_internal_checkerboard_stats, appname=appname, enable_profiling=options.enable_profiling, extra_prefs=options.extra_prefs, **devicePrefs) elif apks: for apk in apks: run_test(device, options.outputdir, outputfile, test, options.url_params, options.num_runs, options.startup_test, options.no_capture, options.get_internal_checkerboard_stats, apk=apk, enable_profiling=options.enable_profiling, extra_prefs=options.extra_prefs, **devicePrefs) else: br = eideticker.BuildRetriever() productname = "nightly" product = eideticker.get_product(productname) for date in dates: apk = br.get_build(product, date) run_test(device, options.outputdir, outputfile, test, options.url_params, options.num_runs, options.startup_test, options.no_capture, options.get_internal_checkerboard_stats, apk=apk, appdate=date, enable_profiling=options.enable_profiling, extra_prefs=options.extra_prefs, **devicePrefs)
def main(args=sys.argv[1:]): usage = "usage: %prog [options] <product> <test> <output dir>" parser = eideticker.OptionParser(usage=usage) parser.add_option("--enable-profiling", action="store_true", dest = "enable_profiling", help = "Create SPS profile to go along with capture") parser.add_option("--device-id", action="store", dest="device_id", help="id of device (used in output json)") parser.add_option("--apk", action="store", dest="apk", help = "Product apk to get metadata from " \ "(Android-specific)") parser.add_option("--num-runs", action="store", type = "int", dest = "num_runs", help = "number of runs (default: 1)") options, args = parser.parse_args() if len(args) != 3: parser.error("incorrect number of arguments") (productname, testkey, outputdir) = args num_runs = 1 if options.num_runs: num_runs = options.num_runs manifest = manifestparser.TestManifest(manifests=[os.path.join( os.path.dirname(__file__), '../src/tests/manifest.ini')]) # sanity check... does the test match a known test key? testkeys = [test["key"] for test in manifest.active_tests()] if testkey not in testkeys: print "ERROR: No tests matching '%s' (options: %s)" % (testkey, ", ".join(testkeys)) sys.exit(1) testinfo = [test for test in manifest.active_tests() if test['key'] == testkey][0] device_id = options.device_id if not device_id: device_id = os.environ.get('DEVICE_ID') if not device_id: print "ERROR: Must specify device id (either with --device-id or with DEVICE_ID environment variable)" sys.exit(1) product = eideticker.get_product(productname) current_date = time.strftime("%Y-%m-%d") datafile = os.path.join(outputdir, device_id, '%s.json' % testkey) data = NestedDict() if os.path.isfile(datafile): data.update(json.loads(open(datafile).read())) devicePrefs = eideticker.getDevicePrefs(options) device = eideticker.getDevice(**devicePrefs) # update the device list for the dashboard devices = {} devicefile = os.path.join(outputdir, 'devices.json') if os.path.isfile(devicefile): devices = json.loads(open(devicefile).read())['devices'] testfile = os.path.join(outputdir, '%s' % device_id, 'tests.json') if os.path.isfile(testfile): tests = json.loads(open(testfile).read())['tests'] else: tests = {} tests[testkey] = { 'shortDesc': testinfo['shortDesc'], 'defaultMeasure': testinfo['defaultMeasure'] } devices[device_id] = { 'name': device.model, 'version': device.getprop('ro.build.version.release') } with open(devicefile, 'w') as f: f.write(json.dumps({ 'devices': devices })) testfiledir = os.path.dirname(testfile) if not os.path.exists(testfiledir): os.mkdir(testfiledir) with open(testfile, 'w') as f: f.write(json.dumps({ 'tests': tests })) if options.apk: appinfo = eideticker.get_fennec_appinfo(options.apk) appname = appinfo['appname'] print "Using application name '%s' from apk '%s'" % (appname, options.apk) capture_name = "%s %s" % (product['name'], appinfo['date']) else: # no apk, assume it's something static on the device appinfo = { 'date': 'today' } appname = product['appname'] capture_name = "%s (taken on %s)" % (product['name'], current_date) # Run the test the specified number of times for i in range(num_runs): # Now run the test runtest(device, product, appname, appinfo, testinfo, capture_name + " #%s" % i, outputdir, datafile, data, enable_profiling=options.enable_profiling, **devicePrefs) # Kill app after test complete device.killProcess(appname)
def runtest(device_prefs, testname, options, apk=None, appname=None, appdate=None): device = None if apk: appinfo = eideticker.get_fennec_appinfo(apk) appname = appinfo["appname"] print "Installing %s (version: %s, revision %s)" % (appinfo["appname"], appinfo["version"], appinfo["revision"]) device = eideticker.getDevice(**device_prefs) device.updateApp(apk) else: appinfo = None testinfo = eideticker.get_testinfo(testname) stableframecapture = ( testinfo["type"] in ("startup", "webstartup") or testinfo["defaultMeasure"] == "timetostableframe" ) capture_results = [] for i in range(options.num_runs): # Kill any existing instances of the processes (for Android) if device: device.killProcess(appname) # Now run the test curtime = int(time.time()) capture_file = os.path.join(CAPTURE_DIR, "metric-test-%s-%s.zip" % (appname, curtime)) if options.enable_profiling: profile_file = os.path.join(PROFILE_DIR, "profile-%s-%s.zip" % (appname, curtime)) else: profile_file = None current_date = time.strftime("%Y-%m-%d") capture_name = "%s - %s (taken on %s)" % (testname, appname, current_date) if options.prepare_test: eideticker.prepare_test(testname, device_prefs) testlog = eideticker.run_test( testname, options.capture_device, appname, capture_name, device_prefs, extra_prefs=options.extra_prefs, extra_env_vars=options.extra_env_vars, log_checkerboard_stats=options.get_internal_checkerboard_stats, profile_file=profile_file, capture_area=options.capture_area, no_capture=options.no_capture, fps=options.fps, capture_file=capture_file, wifi_settings_file=options.wifi_settings_file, sync_time=options.sync_time, ) capture_result = {} if not options.no_capture: capture_result["file"] = capture_file capture = videocapture.Capture(capture_file) capture_result["captureFPS"] = capture.fps if stableframecapture: capture_result["timetostableframe"] = eideticker.get_stable_frame_time(capture) else: capture_result.update(eideticker.get_standard_metrics(capture, testlog.actions)) if options.outputdir: # video video_relpath = os.path.join("videos", "video-%s.webm" % time.time()) video_path = os.path.join(options.outputdir, video_relpath) open(video_path, "w").write(capture.get_video().read()) capture_result["video"] = video_relpath # framediff framediff_relpath = os.path.join("framediffs", "framediff-%s.json" % time.time()) framediff_path = os.path.join(options.outputdir, framediff_relpath) with open(framediff_path, "w") as f: framediff = videocapture.get_framediff_sums(capture) f.write(json.dumps({"diffsums": framediff})) capture_result["frameDiff"] = framediff_relpath if options.enable_profiling: capture_result["profile"] = profile_file if options.get_internal_checkerboard_stats: capture_result["internalcheckerboard"] = testlog.checkerboard_percent_totals capture_results.append(capture_result) if options.devicetype == "b2g": # FIXME: get information from sources.xml and application.ini on # device, as we do in update-dashboard.py display_key = appkey = "FirefoxOS" else: appkey = appname if appdate: appkey = appdate.isoformat() else: appkey = appname if appinfo and appinfo.get("revision"): display_key = "%s (%s)" % (appkey, appinfo["revision"]) else: display_key = appkey print "=== Results on %s for %s ===" % (testname, display_key) if not options.no_capture: if stableframecapture: print " Times to first stable frame (seconds):" print " %s" % map(lambda c: c["timetostableframe"], capture_results) print else: print " Number of unique frames:" print " %s" % map(lambda c: c["uniqueframes"], capture_results) print print " Average number of unique frames per second:" print " %s" % map(lambda c: c["fps"], capture_results) print print " Checkerboard area/duration (sum of percents NOT percentage):" print " %s" % map(lambda c: c["checkerboard"], capture_results) print print " Time to first input response: " print " %s" % map(lambda c: c.get("timetoresponse"), capture_results) print print " Capture files:" print " Capture files: %s" % map(lambda c: c["file"], capture_results) print if options.enable_profiling: print " Profile files:" print " Profile files: %s" % map(lambda c: c["profile"], capture_results) print if options.get_internal_checkerboard_stats: print " Internal Checkerboard Stats (sum of percents, not percentage):" print " %s" % map(lambda c: c["internalcheckerboard"], capture_results) print if options.outputdir: outputfile = os.path.join(options.outputdir, "metric.json") resultdict = {"title": testname, "data": {}} if os.path.isfile(outputfile): resultdict.update(json.loads(open(outputfile).read())) if not resultdict["data"].get(appkey): resultdict["data"][appkey] = [] resultdict["data"][appkey].extend(capture_results) with open(outputfile, "w") as f: f.write(json.dumps(resultdict))