def run_test(device,
outputdir,
outputfile,
test,
url_params,
num_runs,
startup_test,
no_capture,
get_internal_checkerboard_stats,
apk=None,
appname=None,
appdate=None,
profile_file=None,
dmtype="adb",
host=None,
port=None):
if apk:
appinfo = eideticker.get_fennec_appinfo(apk)
appname = appinfo['appname']
print "Installing %s (version: %s, revision %s)" % (
appinfo['appname'], appinfo['version'], appinfo['revision'])
device.updateApp(apk)
else:
appinfo = None
captures = []
for i in range(num_runs):
# Kill any existing instances of the processes
device.killProcess(appname)
# Now run the test
capture_file = os.path.join(
CAPTURE_DIR, "metric-test-%s-%s.zip" % (appname, int(time.time())))
args = ["runtest.py", "--url-params", url_params, appname, test]
if get_internal_checkerboard_stats:
checkerboard_logfile = tempfile.NamedTemporaryFile()
args.extend(["--checkerboard-log-file", checkerboard_logfile.name])
if startup_test:
args.extend(["--startup-test"])
if no_capture:
args.extend(["--no-capture"])
else:
args.extend(["--capture-file", capture_file])
if profile_file:
args.extend(["--profile-file", profile_file])
if dmtype:
args.extend(["-m", dmtype])
if host:
args.extend(["--host", host])
if port:
args.extend(["--port", port])
print args
retval = subprocess.call(args)
if retval != 0:
raise Exception("Failed to run test %s for %s" % (test, appname))
capture_result = {}
if not no_capture:
capture_result['file'] = capture_file
capture = videocapture.Capture(capture_file)
framediff_sums = videocapture.get_framediff_sums(capture)
if startup_test:
capture_result['stableframe'] = videocapture.get_stable_frame(
capture)
else:
capture_result[
'uniqueframes'] = videocapture.get_num_unique_frames(
capture)
capture_result['fps'] = videocapture.get_fps(capture)
capture_result[
'checkerboard'] = videocapture.get_checkerboarding_area_duration(
capture)
if outputdir:
video_path = os.path.join('videos',
'video-%s.webm' % time.time())
video_file = os.path.join(outputdir, video_path)
open(video_file, 'w').write(capture.get_video().read())
capture_result['video'] = video_path
if get_internal_checkerboard_stats:
internal_checkerboard_totals = parse_checkerboard_log(
checkerboard_logfile.name)
capture_result[
'internalcheckerboard'] = internal_checkerboard_totals
captures.append(capture_result)
appkey = appname
if appdate:
appkey = appdate.isoformat()
else:
appkey = appname
if appinfo and appinfo.get('revision'):
display_key = "%s (%s)" % (appkey, appinfo['revision'])
else:
display_key = appkey
print "=== Results for %s ===" % display_key
if not no_capture:
if startup_test:
print " First stable frames:"
print " %s" % map(lambda c: c['stableframe'], captures)
print
else:
print " Number of unique frames:"
print " %s" % map(lambda c: c['uniqueframes'], captures)
print
print " Average number of unique frames per second:"
print " %s" % map(lambda c: c['fps'], captures)
print
print " Checkerboard area/duration (sum of percents NOT percentage):"
print " %s" % map(lambda c: c['checkerboard'], captures)
print
print " Capture files (for further reference):"
print " Capture files: %s" % map(lambda c: c['file'], captures)
print
if get_internal_checkerboard_stats:
print " Internal Checkerboard Stats (sum of percents, not percentage):"
print " %s" % map(lambda c: c['internalcheckerboard'], captures)
print
if outputfile:
resultdict = {'title': test, 'data': {}}
if os.path.isfile(outputfile):
resultdict.update(json.loads(open(outputfile).read()))
if not resultdict['data'].get(appkey):
resultdict['data'][appkey] = []
resultdict['data'][appkey].extend(captures)
with open(outputfile, 'w') as f:
f.write(json.dumps(resultdict))
def run_test(device, outputdir, outputfile, test, url_params, num_runs,
startup_test, no_capture, get_internal_checkerboard_stats,
apk=None, appname = None, appdate = None, enable_profiling=False,
extra_prefs="{}", dmtype="adb", devicetype="android", host=None,
port=None):
if apk:
appinfo = eideticker.get_fennec_appinfo(apk)
appname = appinfo['appname']
print "Installing %s (version: %s, revision %s)" % (appinfo['appname'],
appinfo['version'],
appinfo['revision'])
device.updateApp(apk)
else:
appinfo = None
captures = []
for i in range(num_runs):
# Kill any existing instances of the processes
device.killProcess(appname)
# Now run the test
curtime = int(time.time())
capture_file = os.path.join(CAPTURE_DIR,
"metric-test-%s-%s.zip" % (appname,
curtime))
if enable_profiling:
profile_file = os.path.join(PROFILE_DIR,
"profile-%s-%s.zip" % (appname, curtime))
args = ["runtest.py", "--url-params", url_params,
"--extra-prefs", extra_prefs, appname, test]
if get_internal_checkerboard_stats:
checkerboard_logfile = tempfile.NamedTemporaryFile()
args.extend(["--checkerboard-log-file", checkerboard_logfile.name])
if startup_test:
args.extend(["--startup-test"])
if no_capture:
args.extend(["--no-capture"])
else:
args.extend(["--capture-file", capture_file])
if enable_profiling:
args.extend(["--profile-file", profile_file])
if dmtype:
args.extend(["-m", dmtype])
if devicetype:
args.extend(["-d", devicetype])
if host:
args.extend(["--host", host])
if port:
args.extend(["--port", port])
print args
retval = subprocess.call(args)
if retval != 0:
raise Exception("Failed to run test %s for %s" % (test, appname))
capture_result = {}
if not no_capture:
capture_result['file'] = capture_file
capture = videocapture.Capture(capture_file)
if startup_test:
capture_result['stableframe'] = videocapture.get_stable_frame(capture)
else:
capture_result['uniqueframes'] = videocapture.get_num_unique_frames(capture)
capture_result['fps'] = videocapture.get_fps(capture)
capture_result['checkerboard'] = videocapture.get_checkerboarding_area_duration(capture)
if outputdir:
video_path = os.path.join('videos', 'video-%s.webm' % time.time())
video_file = os.path.join(outputdir, video_path)
open(video_file, 'w').write(capture.get_video().read())
capture_result['video'] = video_path
if enable_profiling:
capture_result['profile'] = profile_file
if get_internal_checkerboard_stats:
internal_checkerboard_totals = parse_checkerboard_log(checkerboard_logfile.name)
capture_result['internalcheckerboard'] = internal_checkerboard_totals
captures.append(capture_result)
appkey = appname
if appdate:
appkey = appdate.isoformat()
else:
appkey = appname
if appinfo and appinfo.get('revision'):
display_key = "%s (%s)" % (appkey, appinfo['revision'])
else:
display_key = appkey
print "=== Results for %s ===" % display_key
if not no_capture:
if startup_test:
print " First stable frames:"
print " %s" % map(lambda c: c['stableframe'], captures)
print
else:
print " Number of unique frames:"
print " %s" % map(lambda c: c['uniqueframes'], captures)
print
print " Average number of unique frames per second:"
print " %s" % map(lambda c: c['fps'], captures)
print
print " Checkerboard area/duration (sum of percents NOT percentage):"
print " %s" % map(lambda c: c['checkerboard'], captures)
print
print " Capture files:"
print " Capture files: %s" % map(lambda c: c['file'], captures)
print
if enable_profiling:
print " Profile files:"
print " Profile files: %s" % map(lambda c: c['profile'], captures)
print
if get_internal_checkerboard_stats:
print " Internal Checkerboard Stats (sum of percents, not percentage):"
print " %s" % map(lambda c: c['internalcheckerboard'], captures)
print
if outputfile:
resultdict = { 'title': test, 'data': {} }
if os.path.isfile(outputfile):
resultdict.update(json.loads(open(outputfile).read()))
if not resultdict['data'].get(appkey):
resultdict['data'][appkey] = []
resultdict['data'][appkey].extend(captures)
with open(outputfile, 'w') as f:
f.write(json.dumps(resultdict))
def runtest(device_prefs, capture_device, outputdir, outputfile, testname, url_params, num_runs,
startup_test, no_capture, get_internal_checkerboard_stats,
apk=None, appname = None, appdate = None, enable_profiling=False,
extra_prefs={}, extra_env_vars={}):
device = None
if apk:
appinfo = eideticker.get_fennec_appinfo(apk)
appname = appinfo['appname']
print "Installing %s (version: %s, revision %s)" % (appinfo['appname'],
appinfo['version'],
appinfo['revision'])
device = eideticker.getDevice(**device_prefs)
device.updateApp(apk)
else:
appinfo = None
captures = []
for i in range(num_runs):
# Kill any existing instances of the processes (for Android)
if device:
device.killProcess(appname)
# Now run the test
curtime = int(time.time())
capture_file = os.path.join(CAPTURE_DIR,
"metric-test-%s-%s.zip" % (appname,
curtime))
if enable_profiling:
profile_file = os.path.join(PROFILE_DIR,
"profile-%s-%s.zip" % (appname, curtime))
else:
profile_file = None
if get_internal_checkerboard_stats:
checkerboard_log_file = tempfile.NamedTemporaryFile()
else:
checkerboard_log_file = None
current_date = time.strftime("%Y-%m-%d")
capture_name = "%s - %s (taken on %s)" % (testname, appname, current_date)
eideticker.run_test(testname, capture_device,
appname, capture_name, device_prefs,
extra_prefs=extra_prefs,
extra_env_vars=extra_env_vars,
checkerboard_log_file=checkerboard_log_file,
profile_file=profile_file,
no_capture=no_capture,
capture_file=capture_file)
capture_result = {}
if not no_capture:
capture_result['file'] = capture_file
capture = videocapture.Capture(capture_file)
if startup_test:
capture_result['stableframe'] = videocapture.get_stable_frame(capture)
else:
capture_result['uniqueframes'] = videocapture.get_num_unique_frames(capture)
capture_result['fps'] = videocapture.get_fps(capture)
capture_result['checkerboard'] = videocapture.get_checkerboarding_area_duration(capture)
if outputdir:
video_path = os.path.join('videos', 'video-%s.webm' % time.time())
video_file = os.path.join(outputdir, video_path)
open(video_file, 'w').write(capture.get_video().read())
capture_result['video'] = video_path
if enable_profiling:
capture_result['profile'] = profile_file
if get_internal_checkerboard_stats:
internal_checkerboard_totals = parse_checkerboard_log(checkerboard_log_file.name)
capture_result['internalcheckerboard'] = internal_checkerboard_totals
captures.append(capture_result)
appkey = appname
if appdate:
appkey = appdate.isoformat()
else:
appkey = appname
if appinfo and appinfo.get('revision'):
display_key = "%s (%s)" % (appkey, appinfo['revision'])
else:
display_key = appkey
print "=== Results for %s ===" % display_key
if not no_capture:
if startup_test:
print " First stable frames:"
print " %s" % map(lambda c: c['stableframe'], captures)
print
else:
print " Number of unique frames:"
print " %s" % map(lambda c: c['uniqueframes'], captures)
print
print " Average number of unique frames per second:"
print " %s" % map(lambda c: c['fps'], captures)
print
print " Checkerboard area/duration (sum of percents NOT percentage):"
print " %s" % map(lambda c: c['checkerboard'], captures)
print
print " Capture files:"
print " Capture files: %s" % map(lambda c: c['file'], captures)
print
if enable_profiling:
print " Profile files:"
print " Profile files: %s" % map(lambda c: c['profile'], captures)
print
if get_internal_checkerboard_stats:
print " Internal Checkerboard Stats (sum of percents, not percentage):"
print " %s" % map(lambda c: c['internalcheckerboard'], captures)
print
if outputfile:
resultdict = { 'title': testname, 'data': {} }
if os.path.isfile(outputfile):
resultdict.update(json.loads(open(outputfile).read()))
if not resultdict['data'].get(appkey):
resultdict['data'][appkey] = []
resultdict['data'][appkey].extend(captures)
with open(outputfile, 'w') as f:
f.write(json.dumps(resultdict))
