def main(args=sys.argv[1:]): usage = "usage: %prog [options] <test key>" parser = eideticker.TestOptionParser(usage=usage) parser.add_option("--url-params", action="store", dest="url_params", help="additional url parameters for test") parser.add_option("--name", action="store", type="string", dest="capture_name", help="name to give capture") parser.add_option("--capture-file", action="store", type="string", dest="capture_file", help="name to give to capture file") parser.add_option("--no-capture", action="store_true", dest="no_capture", help="run through the test, but don't actually " "capture anything") parser.add_option("--app-name", action="store", type="string", dest="appname", help="Specify an application name (android only)") parser.add_option("--test-type", action="store", type="string", dest="test_type", help="override test type") parser.add_option("--profile-file", action="store", type="string", dest="profile_file", help="Collect a performance profile using the built in " "profiler (fennec only).") parser.add_option("--request-log-file", action="store", type="string", dest="request_log_file", help="Collect a log of HTTP requests during test") parser.add_option("--actions-log-file", action="store", type="string", dest="actions_log_file", help="Collect a log of actions requests during test") options, args = parser.parse_args() if len(args) != 1: parser.error("You must specify (only) a test key") sys.exit(1) testkey = args[0] device_prefs = eideticker.getDevicePrefs(options) if options.prepare_test: eideticker.prepare_test(testkey, device_prefs) testlog = eideticker.run_test(testkey, options.capture_device, options.appname, options.capture_name, device_prefs, extra_prefs=options.extra_prefs, extra_env_vars=options.extra_env_vars, test_type=options.test_type, profile_file=options.profile_file, no_capture=options.no_capture, capture_area=options.capture_area, capture_file=options.capture_file, wifi_settings_file=options.wifi_settings_file, sync_time=options.sync_time) # save logs if applicable testlog.save_logs(http_request_log_path=options.request_log_file, actions_log_path=options.actions_log_file)
def runtest(dm, device_prefs, options, product, appinfo, testinfo, capture_name): capture_filename = os.path.join(CAPTURE_DIR, "%s-%s-%s-%s.zip" % (testinfo['key'], options.appname, appinfo.get('appdate'), int(time.time()))) productname = product['name'] if options.enable_profiling: productname += "-profiling" profile_path = os.path.join( 'profiles', 'sps-profile-%s.zip' % time.time()) profile_filename = os.path.join(options.dashboard_dir, profile_path) else: profile_filename = None test_completed = False for i in range(3): print "Running test %s (try %s of 3)" % (testinfo['key'], (i + 1)) try: testlog = eideticker.run_test( testinfo['key'], options, capture_filename=capture_filename, profile_filename=profile_filename, capture_name=capture_name) test_completed = True break except eideticker.TestException, e: if e.can_retry: print "Test failed, but not fatally. Retrying..." print e else: print "Test failed (fatally). Aborting" print e raise
def runtest(dm, device_prefs, capture_device, capture_area, product, appname, appinfo, testinfo, capture_name, outputdir, datafile, data, enable_profiling=False, log_http_requests=False, log_actions=False, baseline=False, wifi_settings_file=None, sync_time=True): capture_file = os.path.join( CAPTURE_DIR, "%s-%s-%s-%s.zip" % (testinfo['key'], appname, appinfo.get('appdate'), int(time.time()))) productname = product['name'] profile_file = None if enable_profiling: productname += "-profiling" profile_path = os.path.join('profiles', 'sps-profile-%s.zip' % time.time()) profile_file = os.path.join(outputdir, profile_path) test_completed = False for i in range(3): print "Running test (try %s of 3)" % (i + 1) # Kill any existing instances of the processes before starting dm.killProcess(appname) try: testlog = eideticker.run_test( testinfo['key'], capture_device, appname, capture_name, device_prefs, profile_file=profile_file, capture_area=capture_area, capture_file=capture_file, wifi_settings_file=wifi_settings_file, sync_time=sync_time) test_completed = True break except eideticker.TestException, e: if e.can_retry: print "Test failed, but not fatally. Retrying..." else: raise
def runtest(dm, device_prefs, capture_device, capture_area, product, appname, appinfo, testinfo, capture_name, outputdir, datafile, data, enable_profiling=False, log_http_requests=False, baseline=False): capture_file = os.path.join(CAPTURE_DIR, "%s-%s-%s-%s.zip" % (testinfo['key'], appname, appinfo.get('appdate'), int(time.time()))) productname = product['name'] profile_file = None if enable_profiling: productname += "-profiling" profile_path = os.path.join('profiles', 'sps-profile-%s.zip' % time.time()) profile_file = os.path.join(outputdir, profile_path) request_log_file = None if log_http_requests: request_log_path = os.path.join('httplogs', 'http-log-%s.json' % time.time()) request_log_file = os.path.join(outputdir, request_log_path) test_completed = False for i in range(3): print "Running test (try %s of 3)" % (i+1) # Kill any existing instances of the processes before starting dm.killProcess(appname) try: eideticker.run_test(testinfo['key'], capture_device, appname, capture_name, device_prefs, profile_file=profile_file, request_log_file=request_log_file, capture_area=capture_area, capture_file=capture_file) test_completed = True break except eideticker.TestException, e: if e.can_retry: print "Test failed, but not fatally. Retrying..." else: raise
def main(args=sys.argv[1:]): usage = "usage: %prog [options] <test key>" parser = eideticker.TestOptionParser(usage=usage) parser.add_option("--url-params", action="store", dest="url_params", help="additional url parameters for test") parser.add_option("--name", action="store", type="string", dest="capture_name", help="name to give capture") parser.add_option("--capture-file", action="store", type="string", dest="capture_file", help="name to give to capture file") parser.add_option("--app-name", action="store", type="string", dest="appname", help="Specify an application name (android only)") parser.add_option("--test-type", action="store", type="string", dest="test_type", help="override test type") parser.add_option("--profile-file", action="store", type="string", dest="profile_file", help="Collect a performance profile using the built in " "profiler (fennec only).") parser.add_option("--request-log-file", action="store", type="string", dest="request_log_file", help="Collect a log of HTTP requests during test") parser.add_option("--actions-log-file", action="store", type="string", dest="actions_log_file", help="Collect a log of actions requests during test") options, args = parser.parse_args() if len(args) != 1: parser.error("You must specify (only) a test key") sys.exit(1) testkey = args[0] if options.prepare_test: eideticker.prepare_test(testkey, options) testlog = eideticker.run_test(testkey, options, capture_filename=options.capture_file, profile_filename=options.profile_file) # save logs if applicable if options.request_log_file: open(options.request_log_file, 'w').write(json.dumps(testlog.http_request_log)) if options.actions_log_file: open(options.actions_log_file, 'w').write(json.dumps(testlog.actions))
def runtest(dm, device_prefs, options, product, appname, appinfo, testinfo, capture_name, datafile, data, log_http_requests=False, log_actions=False): capture_file = os.path.join(CAPTURE_DIR, "%s-%s-%s-%s.zip" % (testinfo['key'], appname, appinfo.get('appdate'), int(time.time()))) productname = product['name'] profile_file = None if options.enable_profiling: productname += "-profiling" profile_path = os.path.join( 'profiles', 'sps-profile-%s.zip' % time.time()) profile_file = os.path.join(options.outputdir, profile_path) test_completed = False for i in range(3): print "Running test (try %s of 3)" % (i + 1) # Kill any existing instances of the processes before starting dm.killProcess(appname) try: testlog = eideticker.run_test( testinfo['key'], options.capture_device, appname, capture_name, device_prefs, profile_file=profile_file, capture_area=options.capture_area, camera_settings_file=options.camera_settings_file, capture=options.capture, capture_file=capture_file, wifi_settings_file=options.wifi_settings_file, sync_time=options.sync_time, use_vpxenc=options.use_vpxenc) test_completed = True break except eideticker.TestException, e: if e.can_retry: print "Test failed, but not fatally. Retrying..." else: raise
def runtest(dm, device_prefs, options, product, appinfo, testinfo, capture_name): capture_filename = os.path.join( CAPTURE_DIR, "%s-%s-%s-%s.zip" % (testinfo['key'], options.appname, appinfo.get('appdate'), int(time.time()))) productname = product['name'] test_uuid = uuid.uuid1().hex if options.enable_profiling: productname += "-profiling" profile_filename = os.path.join(options.dashboard_dir, 'profiles', '%s.zip' % test_uuid) else: profile_filename = None test_completed = False for i in range(3): print "Running test %s (try %s of 3)" % (testinfo['key'], (i + 1)) try: testlog = eideticker.run_test(testinfo['key'], options, capture_filename=capture_filename, profile_filename=profile_filename, capture_name=capture_name) test_completed = True break except eideticker.TestException, e: if e.can_retry: print "Test failed, but not fatally. Retrying..." print e else: print "Test failed (fatally). Aborting" print e raise
def main(args=sys.argv[1:]): usage = "usage: %prog [options] <test key>" parser = eideticker.TestOptionParser(usage=usage) parser.add_option("--url-params", action="store", dest="url_params", help="additional url parameters for test") parser.add_option("--name", action="store", type="string", dest="capture_name", help="name to give capture") parser.add_option("--capture-file", action="store", type="string", dest="capture_file", help="name to give to capture file") parser.add_option("--no-capture", action="store_true", dest="no_capture", help="run through the test, but don't actually " "capture anything") parser.add_option("--app-name", action="store", type="string", dest="appname", help="Specify an application name (android only)") parser.add_option("--test-type", action="store", type="string", dest="test_type", help="override test type") parser.add_option("--profile-file", action="store", type="string", dest="profile_file", help="Collect a performance profile using the built in " "profiler (fennec only).") parser.add_option("--request-log-file", action="store", type="string", dest="request_log_file", help="Collect a log of HTTP requests during test") parser.add_option("--actions-log-file", action="store", type="string", dest="actions_log_file", help="Collect a log of actions requests during test") options, args = parser.parse_args() if len(args) != 1: parser.error("You must specify (only) a test key") sys.exit(1) testkey = args[0] device_prefs = eideticker.getDevicePrefs(options) if options.prepare_test: eideticker.prepare_test(testkey, device_prefs) testlog = eideticker.run_test( testkey, options.capture_device, options.appname, options.capture_name, device_prefs, extra_prefs=options.extra_prefs, extra_env_vars=options.extra_env_vars, test_type=options.test_type, profile_file=options.profile_file, no_capture=options.no_capture, capture_area=options.capture_area, fps=options.fps, capture_file=options.capture_file, wifi_settings_file=options.wifi_settings_file, sync_time=options.sync_time) # save logs if applicable testlog.save_logs(http_request_log_path=options.request_log_file, actions_log_path=options.actions_log_file)
def runtest(device_prefs, testname, options, apk=None, appname=None, appdate=None): if apk: appinfo = eideticker.get_fennec_appinfo(apk) appname = appinfo['appname'] print "Installing %s (version: %s, revision %s)" % ( appinfo['appname'], appinfo['version'], appinfo['revision']) device = eideticker.getDevice(**device_prefs) device.updateApp(apk) else: appinfo = None options.appname = appname testinfo = eideticker.get_testinfo(testname) stableframecapture = (testinfo['type'] in ('startup', 'webstartup') or testinfo['defaultMeasure'] == 'timetostableframe') capture_results = [] if options.prepare_test: eideticker.prepare_test(testname, options) for i in range(options.num_runs): # Now run the test curtime = int(time.time()) capture_file = os.path.join(CAPTURE_DIR, "metric-test-%s-%s.zip" % (appname, curtime)) if options.enable_profiling and options.outputdir: profile_relpath = os.path.join( 'profiles', 'sps-profile-%s.zip' % time.time()) profile_filename = os.path.join(options.outputdir, profile_relpath) else: profile_filename = None current_date = time.strftime("%Y-%m-%d") capture_name = "%s - %s (taken on %s)" % ( testname, appname, current_date) testlog = eideticker.run_test(testname, options, capture_filename=capture_file, profile_filename=profile_filename, capture_name=capture_name) capture_uuid = uuid.uuid1().hex datapoint = { 'uuid': capture_uuid } metadata = {} metrics = {} if options.capture: capture = videocapture.Capture(capture_file) datapoint['captureFile'] = metadata['captureFile'] = capture_file metadata['captureFPS'] = capture.fps metadata['generatedVideoFPS'] = capture.generated_video_fps if stableframecapture: metrics['timetostableframe'] = \ eideticker.get_stable_frame_time(capture) else: metrics.update( eideticker.get_standard_metrics(capture, testlog.actions)) metadata['metrics'] = metrics metadata.update(eideticker.get_standard_metric_metadata(capture)) if options.outputdir: # video video_relpath = os.path.join( 'videos', 'video-%s.webm' % time.time()) video_path = os.path.join(options.outputdir, video_relpath) open(video_path, 'w').write(capture.get_video().read()) metadata['video'] = video_relpath if options.log_checkerboard_stats: metrics['internalcheckerboard'] = \ testlog.checkerboard_percent_totals # Want metrics data in data, so we can graph everything at once datapoint.update(metrics) if options.enable_profiling: metadata['profile'] = profile_filename # dump metadata if options.outputdir: # metadata metadata_path = os.path.join(options.outputdir, 'metadata', '%s.json' % capture_uuid) open(metadata_path, 'w').write(json.dumps(metadata)) capture_results.append(datapoint) if options.devicetype == "b2g": # FIXME: get information from sources.xml and application.ini on # device, as we do in update-dashboard.py display_key = appkey = "FirefoxOS" else: appkey = appname if appdate: appkey = appdate.isoformat() else: appkey = appname if appinfo and appinfo.get('revision'): display_key = "%s (%s)" % (appkey, appinfo['revision']) else: display_key = appkey print "=== Results on %s for %s ===" % (testname, display_key) if options.capture: measures = [ ('timetostableframe', 'Times to first stable frame (seconds)'), ('uniqueframes', 'Number of unique frames'), ('fps', 'Average number of unique frames per second'), ('overallentropy', 'Overall entropy over length of capture'), ('checkerboard', 'Checkerboard area/duration (sum of percents NOT ' 'percentage)'), ('timetoresponse', 'Time to first input response (seconds)') ] for measure in measures: if capture_results[0].get(measure[0]): print " %s:" % measure[1] print " %s" % map(lambda c: c[measure[0]], capture_results) print print " Capture files:" print " Capture files: %s" % map(lambda c: c['captureFile'], capture_results) print if options.log_checkerboard_stats: print " Internal Checkerboard Stats (sum of percents, not " "percentage):" print " %s" % map( lambda c: c['internalcheckerboard'], capture_results) print if options.outputdir: outputfile = os.path.join(options.outputdir, "metric.json") resultdict = {'title': testname, 'data': {}} if os.path.isfile(outputfile): resultdict.update(json.loads(open(outputfile).read())) if not resultdict['data'].get(appkey): resultdict['data'][appkey] = [] resultdict['data'][appkey].extend(capture_results) with open(outputfile, 'w') as f: f.write(json.dumps(resultdict))
def runtest(device_prefs, testname, options, apk=None, appname=None, appdate=None): if apk: appinfo = eideticker.get_fennec_appinfo(apk) appname = appinfo['appname'] print "Installing %s (version: %s, revision %s)" % ( appinfo['appname'], appinfo['version'], appinfo['revision']) device = eideticker.getDevice(**device_prefs) device.updateApp(apk) else: appinfo = None testinfo = eideticker.get_testinfo(testname) stableframecapture = (testinfo['type'] in ('startup', 'webstartup') or testinfo['defaultMeasure'] == 'timetostableframe') capture_results = [] if options.prepare_test: eideticker.prepare_test(testname, device_prefs, options.wifi_settings_file) for i in range(options.num_runs): # Now run the test curtime = int(time.time()) capture_file = os.path.join( CAPTURE_DIR, "metric-test-%s-%s.zip" % (appname, curtime)) if options.enable_profiling and options.outputdir: profile_relpath = os.path.join('profiles', 'sps-profile-%s.zip' % time.time()) profile_file = os.path.join(options.outputdir, profile_relpath) else: profile_file = None current_date = time.strftime("%Y-%m-%d") capture_name = "%s - %s (taken on %s)" % (testname, appname, current_date) testlog = eideticker.run_test( testname, options.capture_device, appname, capture_name, device_prefs, extra_prefs=options.extra_prefs, extra_env_vars=options.extra_env_vars, log_checkerboard_stats=options.get_internal_checkerboard_stats, profile_file=profile_file, capture_area=options.capture_area, camera_settings_file=options.camera_settings_file, capture=options.capture, fps=options.fps, capture_file=capture_file, wifi_settings_file=options.wifi_settings_file, sync_time=options.sync_time, use_vpxenc=options.use_vpxenc) capture_uuid = uuid.uuid1().hex datapoint = {'uuid': capture_uuid} metadata = {} metrics = {} if options.capture: capture = videocapture.Capture(capture_file) datapoint['captureFile'] = metadata['captureFile'] = capture_file metadata['captureFPS'] = capture.fps metadata['generatedVideoFPS'] = capture.generated_video_fps if stableframecapture: metrics['timetostableframe'] = \ eideticker.get_stable_frame_time(capture) else: metrics.update( eideticker.get_standard_metrics(capture, testlog.actions)) metadata['metrics'] = metrics metadata.update(eideticker.get_standard_metric_metadata(capture)) if options.outputdir: # video video_relpath = os.path.join('videos', 'video-%s.webm' % time.time()) video_path = os.path.join(options.outputdir, video_relpath) open(video_path, 'w').write(capture.get_video().read()) metadata['video'] = video_relpath if options.get_internal_checkerboard_stats: metrics['internalcheckerboard'] = \ testlog.checkerboard_percent_totals # Want metrics data in data, so we can graph everything at once datapoint.update(metrics) if options.enable_profiling: metadata['profile'] = profile_file # dump metadata if options.outputdir: # metadata metadata_path = os.path.join(options.outputdir, 'metadata', '%s.json' % capture_uuid) open(metadata_path, 'w').write(json.dumps(metadata)) capture_results.append(datapoint) if options.devicetype == "b2g": # FIXME: get information from sources.xml and application.ini on # device, as we do in update-dashboard.py display_key = appkey = "FirefoxOS" else: appkey = appname if appdate: appkey = appdate.isoformat() else: appkey = appname if appinfo and appinfo.get('revision'): display_key = "%s (%s)" % (appkey, appinfo['revision']) else: display_key = appkey print "=== Results on %s for %s ===" % (testname, display_key) if options.capture: measures = [ ('timetostableframe', 'Times to first stable frame (seconds)'), ('uniqueframes', 'Number of unique frames'), ('fps', 'Average number of unique frames per second'), ('overallentropy', 'Overall entropy over length of capture'), ('checkerboard', 'Checkerboard area/duration (sum of percents NOT ' 'percentage)'), ('timetoresponse', 'Time to first input response (seconds)') ] for measure in measures: if capture_results[0].get(measure[0]): print " %s:" % measure[1] print " %s" % map(lambda c: c[measure[0]], capture_results) print print " Capture files:" print " Capture files: %s" % map(lambda c: c['captureFile'], capture_results) print if options.get_internal_checkerboard_stats: print " Internal Checkerboard Stats (sum of percents, not " "percentage):" print " %s" % map(lambda c: c['internalcheckerboard'], capture_results) print if options.outputdir: outputfile = os.path.join(options.outputdir, "metric.json") resultdict = {'title': testname, 'data': {}} if os.path.isfile(outputfile): resultdict.update(json.loads(open(outputfile).read())) if not resultdict['data'].get(appkey): resultdict['data'][appkey] = [] resultdict['data'][appkey].extend(capture_results) with open(outputfile, 'w') as f: f.write(json.dumps(resultdict))
def runtest(device_prefs, testname, options, apk=None, appname=None, appdate=None): device = None if apk: appinfo = eideticker.get_fennec_appinfo(apk) appname = appinfo["appname"] print "Installing %s (version: %s, revision %s)" % (appinfo["appname"], appinfo["version"], appinfo["revision"]) device = eideticker.getDevice(**device_prefs) device.updateApp(apk) else: appinfo = None testinfo = eideticker.get_testinfo(testname) stableframecapture = ( testinfo["type"] in ("startup", "webstartup") or testinfo["defaultMeasure"] == "timetostableframe" ) capture_results = [] for i in range(options.num_runs): # Kill any existing instances of the processes (for Android) if device: device.killProcess(appname) # Now run the test curtime = int(time.time()) capture_file = os.path.join(CAPTURE_DIR, "metric-test-%s-%s.zip" % (appname, curtime)) if options.enable_profiling: profile_file = os.path.join(PROFILE_DIR, "profile-%s-%s.zip" % (appname, curtime)) else: profile_file = None current_date = time.strftime("%Y-%m-%d") capture_name = "%s - %s (taken on %s)" % (testname, appname, current_date) if options.prepare_test: eideticker.prepare_test(testname, device_prefs) testlog = eideticker.run_test( testname, options.capture_device, appname, capture_name, device_prefs, extra_prefs=options.extra_prefs, extra_env_vars=options.extra_env_vars, log_checkerboard_stats=options.get_internal_checkerboard_stats, profile_file=profile_file, capture_area=options.capture_area, no_capture=options.no_capture, fps=options.fps, capture_file=capture_file, wifi_settings_file=options.wifi_settings_file, sync_time=options.sync_time, ) capture_result = {} if not options.no_capture: capture_result["file"] = capture_file capture = videocapture.Capture(capture_file) capture_result["captureFPS"] = capture.fps if stableframecapture: capture_result["timetostableframe"] = eideticker.get_stable_frame_time(capture) else: capture_result.update(eideticker.get_standard_metrics(capture, testlog.actions)) if options.outputdir: # video video_relpath = os.path.join("videos", "video-%s.webm" % time.time()) video_path = os.path.join(options.outputdir, video_relpath) open(video_path, "w").write(capture.get_video().read()) capture_result["video"] = video_relpath # framediff framediff_relpath = os.path.join("framediffs", "framediff-%s.json" % time.time()) framediff_path = os.path.join(options.outputdir, framediff_relpath) with open(framediff_path, "w") as f: framediff = videocapture.get_framediff_sums(capture) f.write(json.dumps({"diffsums": framediff})) capture_result["frameDiff"] = framediff_relpath if options.enable_profiling: capture_result["profile"] = profile_file if options.get_internal_checkerboard_stats: capture_result["internalcheckerboard"] = testlog.checkerboard_percent_totals capture_results.append(capture_result) if options.devicetype == "b2g": # FIXME: get information from sources.xml and application.ini on # device, as we do in update-dashboard.py display_key = appkey = "FirefoxOS" else: appkey = appname if appdate: appkey = appdate.isoformat() else: appkey = appname if appinfo and appinfo.get("revision"): display_key = "%s (%s)" % (appkey, appinfo["revision"]) else: display_key = appkey print "=== Results on %s for %s ===" % (testname, display_key) if not options.no_capture: if stableframecapture: print " Times to first stable frame (seconds):" print " %s" % map(lambda c: c["timetostableframe"], capture_results) print else: print " Number of unique frames:" print " %s" % map(lambda c: c["uniqueframes"], capture_results) print print " Average number of unique frames per second:" print " %s" % map(lambda c: c["fps"], capture_results) print print " Checkerboard area/duration (sum of percents NOT percentage):" print " %s" % map(lambda c: c["checkerboard"], capture_results) print print " Time to first input response: " print " %s" % map(lambda c: c.get("timetoresponse"), capture_results) print print " Capture files:" print " Capture files: %s" % map(lambda c: c["file"], capture_results) print if options.enable_profiling: print " Profile files:" print " Profile files: %s" % map(lambda c: c["profile"], capture_results) print if options.get_internal_checkerboard_stats: print " Internal Checkerboard Stats (sum of percents, not percentage):" print " %s" % map(lambda c: c["internalcheckerboard"], capture_results) print if options.outputdir: outputfile = os.path.join(options.outputdir, "metric.json") resultdict = {"title": testname, "data": {}} if os.path.isfile(outputfile): resultdict.update(json.loads(open(outputfile).read())) if not resultdict["data"].get(appkey): resultdict["data"][appkey] = [] resultdict["data"][appkey].extend(capture_results) with open(outputfile, "w") as f: f.write(json.dumps(resultdict))
def main(args=sys.argv[1:]): usage = "usage: %prog [options] <test key>" parser = eideticker.CaptureOptionParser(usage=usage) parser.add_option("--url-params", action="store", dest="url_params", help="additional url parameters for test") parser.add_option("--name", action="store", type="string", dest="capture_name", help="name to give capture") parser.add_option("--capture-file", action="store", type="string", dest="capture_file", help="name to give to capture file") parser.add_option("--no-capture", action="store_true", dest="no_capture", help="run through the test, but don't actually " "capture anything") parser.add_option("--app-name", action="store", type="string", dest="appname", help="Specify an application name (android only)") parser.add_option("--test-type", action="store", type="string", dest="test_type", help="override test type") parser.add_option("--checkerboard-log-file", action="store", type="string", dest="checkerboard_log_file", help="name to give checkerboarding stats file (fennec " "only)") parser.add_option("--extra-prefs", action="store", dest="extra_prefs", default="{}", help="Extra profile preference for Firefox browsers. " "Must be passed in as a JSON dictionary") parser.add_option("--profile-file", action="store", type="string", dest="profile_file", help="Collect a performance profile using the built in " "profiler (fennec only).") parser.add_option("--request-log-file", action="store", type="string", dest="request_log_file", help="Collect a log of HTTP requests during tests") parser.add_option("--extra-env-vars", action="store", dest="extra_env_vars", default="", help='Extra environment variables to set in ' '"VAR1=VAL1 VAR2=VAL2" format') options, args = parser.parse_args() parser.validate_options(options) if len(args) != 1: parser.error("You must specify (only) a test key") sys.exit(1) testkey = args[0] try: extra_prefs = json.loads(options.extra_prefs) except ValueError: parser.error("Error processing extra preferences: not valid JSON!") raise keyvals = options.extra_env_vars.split() extra_env_vars = {} for kv in keyvals: (var, _, val) = kv.partition("=") extra_env_vars[var] = val capture_area = None if options.capture_area: # we validated this previously... capture_area = json.loads(options.capture_area) device_prefs = eideticker.getDevicePrefs(options) eideticker.run_test(testkey, options.capture_device, options.appname, options.capture_name, device_prefs, extra_prefs=extra_prefs, extra_env_vars=extra_env_vars, test_type=options.test_type, profile_file=options.profile_file, request_log_file=options.request_log_file, checkerboard_log_file=options.checkerboard_log_file, no_capture=options.no_capture, capture_area=capture_area, capture_file=options.capture_file)
def runtest(device_prefs, capture_device, outputdir, outputfile, testname, url_params, num_runs, startup_test, no_capture, get_internal_checkerboard_stats, apk=None, appname = None, appdate = None, enable_profiling=False, extra_prefs={}, extra_env_vars={}): device = None if apk: appinfo = eideticker.get_fennec_appinfo(apk) appname = appinfo['appname'] print "Installing %s (version: %s, revision %s)" % (appinfo['appname'], appinfo['version'], appinfo['revision']) device = eideticker.getDevice(**device_prefs) device.updateApp(apk) else: appinfo = None captures = [] for i in range(num_runs): # Kill any existing instances of the processes (for Android) if device: device.killProcess(appname) # Now run the test curtime = int(time.time()) capture_file = os.path.join(CAPTURE_DIR, "metric-test-%s-%s.zip" % (appname, curtime)) if enable_profiling: profile_file = os.path.join(PROFILE_DIR, "profile-%s-%s.zip" % (appname, curtime)) else: profile_file = None if get_internal_checkerboard_stats: checkerboard_log_file = tempfile.NamedTemporaryFile() else: checkerboard_log_file = None current_date = time.strftime("%Y-%m-%d") capture_name = "%s - %s (taken on %s)" % (testname, appname, current_date) eideticker.run_test(testname, capture_device, appname, capture_name, device_prefs, extra_prefs=extra_prefs, extra_env_vars=extra_env_vars, checkerboard_log_file=checkerboard_log_file, profile_file=profile_file, no_capture=no_capture, capture_file=capture_file) capture_result = {} if not no_capture: capture_result['file'] = capture_file capture = videocapture.Capture(capture_file) if startup_test: capture_result['stableframe'] = videocapture.get_stable_frame(capture) else: capture_result['uniqueframes'] = videocapture.get_num_unique_frames(capture) capture_result['fps'] = videocapture.get_fps(capture) capture_result['checkerboard'] = videocapture.get_checkerboarding_area_duration(capture) if outputdir: video_path = os.path.join('videos', 'video-%s.webm' % time.time()) video_file = os.path.join(outputdir, video_path) open(video_file, 'w').write(capture.get_video().read()) capture_result['video'] = video_path if enable_profiling: capture_result['profile'] = profile_file if get_internal_checkerboard_stats: internal_checkerboard_totals = parse_checkerboard_log(checkerboard_log_file.name) capture_result['internalcheckerboard'] = internal_checkerboard_totals captures.append(capture_result) appkey = appname if appdate: appkey = appdate.isoformat() else: appkey = appname if appinfo and appinfo.get('revision'): display_key = "%s (%s)" % (appkey, appinfo['revision']) else: display_key = appkey print "=== Results for %s ===" % display_key if not no_capture: if startup_test: print " First stable frames:" print " %s" % map(lambda c: c['stableframe'], captures) print else: print " Number of unique frames:" print " %s" % map(lambda c: c['uniqueframes'], captures) print print " Average number of unique frames per second:" print " %s" % map(lambda c: c['fps'], captures) print print " Checkerboard area/duration (sum of percents NOT percentage):" print " %s" % map(lambda c: c['checkerboard'], captures) print print " Capture files:" print " Capture files: %s" % map(lambda c: c['file'], captures) print if enable_profiling: print " Profile files:" print " Profile files: %s" % map(lambda c: c['profile'], captures) print if get_internal_checkerboard_stats: print " Internal Checkerboard Stats (sum of percents, not percentage):" print " %s" % map(lambda c: c['internalcheckerboard'], captures) print if outputfile: resultdict = { 'title': testname, 'data': {} } if os.path.isfile(outputfile): resultdict.update(json.loads(open(outputfile).read())) if not resultdict['data'].get(appkey): resultdict['data'][appkey] = [] resultdict['data'][appkey].extend(captures) with open(outputfile, 'w') as f: f.write(json.dumps(resultdict))