def get_stable_frame_time(capture):
analysis_props = _get_analysis_props(capture.metadata['captureDevice'])
return videocapture.get_stable_frame_time(
capture,
method=analysis_props['stable_frame_analysis_method'],
threshold=analysis_props['stable_frame_threshold'],
sobelized=analysis_props['sobelize'])
def get_stable_frame_time(capture):
analysis_props = _get_analysis_props(capture.metadata["captureDevice"])
return videocapture.get_stable_frame_time(
capture,
method=analysis_props["stable_frame_analysis_method"],
threshold=analysis_props["stable_frame_threshold"],
edge_detection=analysis_props["edge_detection"],
)
def runtest(dm, product, appname, appinfo, testinfo, capture_name,
outputdir, datafile, data, enable_profiling=False,
dmtype="adb", host=None, port=None, devicetype="android"):
capture_file = os.path.join(CAPTURE_DIR,
"%s-%s-%s-%s.zip" % (testinfo['key'],
appname,
appinfo.get('date'),
int(time.time())))
if enable_profiling:
profile_path = os.path.join('profiles', 'sps-profile-%s.zip' % time.time())
profile_file = os.path.join(outputdir, profile_path)
test_completed = False
for i in range(3):
print "Running test (try %s of 3)" % (i+1)
# Kill any existing instances of the processes before starting
dm.killProcess(appname)
args = [ "runtest.py", "--name", capture_name, "--capture-file",
capture_file ]
if dmtype:
args.extend(["-m", dmtype])
if devicetype:
args.extend(["-d", devicetype])
if host:
args.extend(["--host", host])
if port:
args.extend(["--port", port])
if enable_profiling:
args.extend(["--profile-file", profile_file])
if appname:
args.extend(["--app-name", appname])
retval = subprocess.call(args + [ testinfo['key'] ])
if retval == 0:
test_completed = True
break
else:
print "Test failed, retrying..."
if not test_completed:
raise Exception("Failed to run test %s for %s (after 3 tries). "
"Aborting." % (testinfo['key'], product['name']))
capture = videocapture.Capture(capture_file)
# video file
video_path = os.path.join('videos', 'video-%s.webm' % time.time())
video_file = os.path.join(outputdir, video_path)
open(video_file, 'w').write(capture.get_video().read())
# need to initialize dict for product if not there already
if not data['testdata'].get(product['name']):
data['testdata'][product['name']] = {}
# app date
appdate = appinfo.get('date')
if not data['testdata'][product['name']].get(appdate):
data['testdata'][product['name']][appdate] = []
datapoint = { 'uuid': uuid.uuid1().hex,
'video': video_path,
'appdate': appinfo.get('date'),
'buildid': appinfo.get('buildid'),
'revision': appinfo.get('revision') }
if testinfo['type'] == 'startup':
datapoint['timetostableframe'] = videocapture.get_stable_frame_time(capture)
else:
# standard test metrics
datapoint['uniqueframes'] = videocapture.get_num_unique_frames(capture)
datapoint['fps'] = videocapture.get_fps(capture)
datapoint['checkerboard'] = videocapture.get_checkerboarding_area_duration(capture)
if enable_profiling:
datapoint['profile'] = profile_path
data['testdata'][product['name']][appdate].append(datapoint)
# Write the data to disk immediately (so we don't lose it if we fail later)
datafile_dir = os.path.dirname(datafile)
if not os.path.exists(datafile_dir):
os.mkdir(datafile_dir)
with open(datafile, 'w') as f:
f.write(json.dumps(data))
'video': video_path }
for key in ['appdate', 'buildid', 'revision', 'geckoRevision',
'gaiaRevision', 'buildRevision', 'sourceRepo']:
if appinfo.get(key):
datapoint.update({key: appinfo[key]})
# only interested in version if we don't have revision
if not appinfo.get('revision') and appinfo.get('version'):
datapoint.update({ 'version': appinfo['version'] })
if baseline:
datapoint.update({ 'baseline': True })
if testinfo['type'] == 'startup' or testinfo['type'] == 'webstartup' or \
testinfo['defaultMeasure'] == 'timetostableframe':
datapoint['timetostableframe'] = videocapture.get_stable_frame_time(capture)
else:
# standard test metrics
threshold = 0
if capture_device == "pointgrey":
# even with median filtering, pointgrey captures tend to have a
# bunch of visual noise -- try to compensate for this by setting
# a higher threshold for frames to be considered different
threshold = 2000
datapoint['uniqueframes'] = videocapture.get_num_unique_frames(capture, threshold=threshold)
datapoint['fps'] = videocapture.get_fps(capture, threshold=threshold)
datapoint['checkerboard'] = videocapture.get_checkerboarding_area_duration(capture)
if enable_profiling:
datapoint['profile'] = profile_path
def get_stable_frame_time(capture):
analysis_props = _get_analysis_props(capture.metadata['captureDevice'])
return videocapture.get_stable_frame_time(
capture, method=analysis_props['stable_frame_analysis_method'],
threshold=analysis_props['stable_frame_threshold'])
def runtest(dm,
product,
current_date,
appname,
appinfo,
test,
capture_name,
outputdir,
datafile,
data,
enable_profiling=False,
dmtype="adb",
host=None,
port=None):
capture_file = os.path.join(
CAPTURE_DIR, "%s-%s-%s-%s.zip" %
(test['name'], appname, appinfo.get('date'), int(time.time())))
if enable_profiling:
profile_package = os.path.join(
CAPTURE_DIR, "profile-package-%s-%s-%s-%s.zip" %
(test['name'], appname, appinfo.get('date'), int(time.time())))
urlparams = test.get('urlparams', '')
test_completed = False
for i in range(3):
print "Running test (try %s of 3)" % (i + 1)
# Kill any existing instances of the processes before starting
dm.killProcess(appname)
args = [
"runtest.py", "--url-params", urlparams, "--name", capture_name,
"--capture-file", capture_file
]
if test.get('startup_test'):
args.append("--startup-test")
if dmtype:
args.extend(["-m", dmtype])
if host:
args.extend(["--host", host])
if port:
args.extend(["--port", port])
if enable_profiling:
args.extend(["--profile-file", profile_package])
retval = subprocess.call(args + [appname, test['path']])
if retval == 0:
test_completed = True
break
else:
print "Test failed, retrying..."
if not test_completed:
raise Exception("Failed to run test %s for %s (after 3 tries). "
"Aborting." % (test['name'], product['name']))
capture = videocapture.Capture(capture_file)
# video file
video_path = os.path.join('videos', 'video-%s.webm' % time.time())
video_file = os.path.join(outputdir, video_path)
open(video_file, 'w').write(capture.get_video().read())
# profile file
if enable_profiling:
profile_path = os.path.join('profiles',
'sps-profile-%s.zip' % time.time())
profile_file = os.path.join(outputdir, profile_path)
symbolicated_profile_path = symbolicate_profile_package(
profile_package, profile_path, profile_file)
os.remove(profile_package)
# need to initialize dict for product if not there already
if not data[test['name']].get(product['name']):
data[test['name']][product['name']] = {}
if not data[test['name']][product['name']].get(current_date):
data[test['name']][product['name']][current_date] = []
datapoint = {
'uuid': uuid.uuid1().hex,
'video': video_path,
'appdate': appinfo.get('date'),
'buildid': appinfo.get('buildid'),
'revision': appinfo.get('revision')
}
if test.get('startup_test'):
datapoint['timetostableframe'] = videocapture.get_stable_frame_time(
capture)
else:
# standard test metrics
datapoint['uniqueframes'] = videocapture.get_num_unique_frames(capture)
datapoint['fps'] = videocapture.get_fps(capture)
datapoint[
'checkerboard'] = videocapture.get_checkerboarding_area_duration(
capture)
if enable_profiling:
datapoint['profile'] = symbolicated_profile_path
data[test['name']][product['name']][current_date].append(datapoint)
# Write the data to disk immediately (so we don't lose it if we fail later)
with open(datafile, 'w') as f:
f.write(json.dumps(data))
def get_stable_frame_time(capture):
analysis_props = _get_analysis_props(capture.metadata['captureDevice'])
return videocapture.get_stable_frame_time(
capture, method=analysis_props['stable_frame_analysis_method'],
threshold=analysis_props['stable_frame_threshold'],
edge_detection=analysis_props['edge_detection'])
