def runtest(device_prefs, testname, options, apk=None, appname=None, appdate=None): if apk: appinfo = eideticker.get_fennec_appinfo(apk) appname = appinfo['appname'] print "Installing %s (version: %s, revision %s)" % ( appinfo['appname'], appinfo['version'], appinfo['revision']) device = eideticker.getDevice(**device_prefs) device.updateApp(apk) else: appinfo = None options.appname = appname testinfo = eideticker.get_testinfo(testname) stableframecapture = (testinfo['type'] in ('startup', 'webstartup') or testinfo['defaultMeasure'] == 'timetostableframe') capture_results = [] if options.prepare_test: eideticker.prepare_test(testname, options) for i in range(options.num_runs): # Now run the test curtime = int(time.time()) capture_file = os.path.join(CAPTURE_DIR, "metric-test-%s-%s.zip" % (appname, curtime)) if options.enable_profiling and options.outputdir: profile_relpath = os.path.join( 'profiles', 'sps-profile-%s.zip' % time.time()) profile_filename = os.path.join(options.outputdir, profile_relpath) else: profile_filename = None current_date = time.strftime("%Y-%m-%d") capture_name = "%s - %s (taken on %s)" % ( testname, appname, current_date) testlog = eideticker.run_test(testname, options, capture_filename=capture_file, profile_filename=profile_filename, capture_name=capture_name) capture_uuid = uuid.uuid1().hex datapoint = { 'uuid': capture_uuid } metadata = {} metrics = {} if options.capture: capture = videocapture.Capture(capture_file) datapoint['captureFile'] = metadata['captureFile'] = capture_file metadata['captureFPS'] = capture.fps metadata['generatedVideoFPS'] = capture.generated_video_fps if stableframecapture: metrics['timetostableframe'] = \ eideticker.get_stable_frame_time(capture) else: metrics.update( eideticker.get_standard_metrics(capture, testlog.actions)) metadata['metrics'] = metrics metadata.update(eideticker.get_standard_metric_metadata(capture)) if options.outputdir: # video video_relpath = os.path.join( 'videos', 'video-%s.webm' % time.time()) video_path = os.path.join(options.outputdir, video_relpath) open(video_path, 'w').write(capture.get_video().read()) metadata['video'] = video_relpath if options.log_checkerboard_stats: metrics['internalcheckerboard'] = \ testlog.checkerboard_percent_totals # Want metrics data in data, so we can graph everything at once datapoint.update(metrics) if options.enable_profiling: metadata['profile'] = profile_filename # dump metadata if options.outputdir: # metadata metadata_path = os.path.join(options.outputdir, 'metadata', '%s.json' % capture_uuid) open(metadata_path, 'w').write(json.dumps(metadata)) capture_results.append(datapoint) if options.devicetype == "b2g": # FIXME: get information from sources.xml and application.ini on # device, as we do in update-dashboard.py display_key = appkey = "FirefoxOS" else: appkey = appname if appdate: appkey = appdate.isoformat() else: appkey = appname if appinfo and appinfo.get('revision'): display_key = "%s (%s)" % (appkey, appinfo['revision']) else: display_key = appkey print "=== Results on %s for %s ===" % (testname, display_key) if options.capture: measures = [ ('timetostableframe', 'Times to first stable frame (seconds)'), ('uniqueframes', 'Number of unique frames'), ('fps', 'Average number of unique frames per second'), ('overallentropy', 'Overall entropy over length of capture'), ('checkerboard', 'Checkerboard area/duration (sum of percents NOT ' 'percentage)'), ('timetoresponse', 'Time to first input response (seconds)') ] for measure in measures: if capture_results[0].get(measure[0]): print " %s:" % measure[1] print " %s" % map(lambda c: c[measure[0]], capture_results) print print " Capture files:" print " Capture files: %s" % map(lambda c: c['captureFile'], capture_results) print if options.log_checkerboard_stats: print " Internal Checkerboard Stats (sum of percents, not " "percentage):" print " %s" % map( lambda c: c['internalcheckerboard'], capture_results) print if options.outputdir: outputfile = os.path.join(options.outputdir, "metric.json") resultdict = {'title': testname, 'data': {}} if os.path.isfile(outputfile): resultdict.update(json.loads(open(outputfile).read())) if not resultdict['data'].get(appkey): resultdict['data'][appkey] = [] resultdict['data'][appkey].extend(capture_results) with open(outputfile, 'w') as f: f.write(json.dumps(resultdict))
metadata.update({ 'version': appinfo['version'] }) if options.baseline: datapoint.update({'baseline': True}) metrics = {} if options.capture: if testinfo['type'] == 'startup' or testinfo['type'] == 'webstartup' or \ testinfo['defaultMeasure'] == 'timetostableframe': metrics['timetostableframe'] = eideticker.get_stable_frame_time( capture) else: # standard test metrics metrics = eideticker.get_standard_metrics(capture, testlog.actions) metadata.update(eideticker.get_standard_metric_metadata(capture)) datapoint.update(metrics) metadata['metrics'] = metrics if options.enable_profiling: metadata['profile'] = profile_path # add logs (if any) to test metadata. we log http requests for webstartup # tests only. likewise, we log actions only for web tests and b2g tests testtype = testinfo['type'] metadata.update(testlog.getdict( log_http_requests=(testtype == 'webstartup'), log_actions=(testtype == 'web' or testtype == 'b2g'))) # Write testdata
def runtest(device_prefs, testname, options, apk=None, appname=None, appdate=None): if apk: appinfo = eideticker.get_fennec_appinfo(apk) appname = appinfo['appname'] print "Installing %s (version: %s, revision %s)" % ( appinfo['appname'], appinfo['version'], appinfo['revision']) device = eideticker.getDevice(**device_prefs) device.updateApp(apk) else: appinfo = None testinfo = eideticker.get_testinfo(testname) stableframecapture = (testinfo['type'] in ('startup', 'webstartup') or testinfo['defaultMeasure'] == 'timetostableframe') capture_results = [] if options.prepare_test: eideticker.prepare_test(testname, device_prefs, options.wifi_settings_file) for i in range(options.num_runs): # Now run the test curtime = int(time.time()) capture_file = os.path.join( CAPTURE_DIR, "metric-test-%s-%s.zip" % (appname, curtime)) if options.enable_profiling and options.outputdir: profile_relpath = os.path.join('profiles', 'sps-profile-%s.zip' % time.time()) profile_file = os.path.join(options.outputdir, profile_relpath) else: profile_file = None current_date = time.strftime("%Y-%m-%d") capture_name = "%s - %s (taken on %s)" % (testname, appname, current_date) testlog = eideticker.run_test( testname, options.capture_device, appname, capture_name, device_prefs, extra_prefs=options.extra_prefs, extra_env_vars=options.extra_env_vars, log_checkerboard_stats=options.get_internal_checkerboard_stats, profile_file=profile_file, capture_area=options.capture_area, camera_settings_file=options.camera_settings_file, capture=options.capture, fps=options.fps, capture_file=capture_file, wifi_settings_file=options.wifi_settings_file, sync_time=options.sync_time, use_vpxenc=options.use_vpxenc) capture_uuid = uuid.uuid1().hex datapoint = {'uuid': capture_uuid} metadata = {} metrics = {} if options.capture: capture = videocapture.Capture(capture_file) datapoint['captureFile'] = metadata['captureFile'] = capture_file metadata['captureFPS'] = capture.fps metadata['generatedVideoFPS'] = capture.generated_video_fps if stableframecapture: metrics['timetostableframe'] = \ eideticker.get_stable_frame_time(capture) else: metrics.update( eideticker.get_standard_metrics(capture, testlog.actions)) metadata['metrics'] = metrics metadata.update(eideticker.get_standard_metric_metadata(capture)) if options.outputdir: # video video_relpath = os.path.join('videos', 'video-%s.webm' % time.time()) video_path = os.path.join(options.outputdir, video_relpath) open(video_path, 'w').write(capture.get_video().read()) metadata['video'] = video_relpath if options.get_internal_checkerboard_stats: metrics['internalcheckerboard'] = \ testlog.checkerboard_percent_totals # Want metrics data in data, so we can graph everything at once datapoint.update(metrics) if options.enable_profiling: metadata['profile'] = profile_file # dump metadata if options.outputdir: # metadata metadata_path = os.path.join(options.outputdir, 'metadata', '%s.json' % capture_uuid) open(metadata_path, 'w').write(json.dumps(metadata)) capture_results.append(datapoint) if options.devicetype == "b2g": # FIXME: get information from sources.xml and application.ini on # device, as we do in update-dashboard.py display_key = appkey = "FirefoxOS" else: appkey = appname if appdate: appkey = appdate.isoformat() else: appkey = appname if appinfo and appinfo.get('revision'): display_key = "%s (%s)" % (appkey, appinfo['revision']) else: display_key = appkey print "=== Results on %s for %s ===" % (testname, display_key) if options.capture: measures = [ ('timetostableframe', 'Times to first stable frame (seconds)'), ('uniqueframes', 'Number of unique frames'), ('fps', 'Average number of unique frames per second'), ('overallentropy', 'Overall entropy over length of capture'), ('checkerboard', 'Checkerboard area/duration (sum of percents NOT ' 'percentage)'), ('timetoresponse', 'Time to first input response (seconds)') ] for measure in measures: if capture_results[0].get(measure[0]): print " %s:" % measure[1] print " %s" % map(lambda c: c[measure[0]], capture_results) print print " Capture files:" print " Capture files: %s" % map(lambda c: c['captureFile'], capture_results) print if options.get_internal_checkerboard_stats: print " Internal Checkerboard Stats (sum of percents, not " "percentage):" print " %s" % map(lambda c: c['internalcheckerboard'], capture_results) print if options.outputdir: outputfile = os.path.join(options.outputdir, "metric.json") resultdict = {'title': testname, 'data': {}} if os.path.isfile(outputfile): resultdict.update(json.loads(open(outputfile).read())) if not resultdict['data'].get(appkey): resultdict['data'][appkey] = [] resultdict['data'][appkey].extend(capture_results) with open(outputfile, 'w') as f: f.write(json.dumps(resultdict))
metadata.update({'version': appinfo['version']}) if options.baseline: datapoint.update({'baseline': True}) metrics = {} if options.capture: if testinfo['type'] == 'startup' or testinfo['type'] == 'webstartup' or \ testinfo['defaultMeasure'] == 'timetostableframe': metrics['timetostableframe'] = eideticker.get_stable_frame_time( capture) else: # standard test metrics metrics = eideticker.get_standard_metrics(capture, testlog.actions) metadata.update(eideticker.get_standard_metric_metadata(capture)) datapoint.update(metrics) metadata['metrics'] = metrics if options.enable_profiling: metadata['profile'] = profile_path # add logs (if any) to test metadata metadata.update(testlog.getdict()) # Add datapoint data['testdata'][productname][appdate].append(datapoint) # Dump metadata open(