def run_groups(groups=[], version=None, host='localhost', port=2828, mode='phone'):
hasadb = mode == 'phone'
logger = get_default_logger()
if groups is None or len(groups) == 0: # run all groups
logger.debug('running securitysuite tests for all groups %s' % str(ExtraTest.group_list(mode=mode)))
groups = ExtraTest.group_list(mode=mode)
else:
logger.debug('running securitysuite tests for groups %s' % str(groups))
logger.suite_start(tests=groups)
# setup marionette before any test
marionette = DeviceHelper.getMarionette(host=host, port=port)
# setup device before any test
device = DeviceHelper.getDevice()
for g in groups:
logger.debug("running securitysuite test group %s" % g)
logger.test_start(g)
try:
ExtraTest.run(g, version=version)
logger.test_end(g, 'OK')
except:
logger.critical(traceback.format_exc())
logger.test_end(g, 'FAIL')
raise
logger.suite_end()
def run_groups(groups=[], version=None, host='localhost', port=2828, mode='phone'):
hasadb = mode == 'phone'
logger = get_default_logger()
if groups is None or len(groups) == 0: # run all groups
logger.debug('running securitysuite tests for all groups %s' % str(ExtraTest.group_list(mode=mode)))
groups = ExtraTest.group_list(mode=mode)
else:
logger.debug('running securitysuite tests for groups %s' % str(groups))
logger.suite_start(tests=groups)
# setup marionette before any test
marionette = DeviceHelper.getMarionette(host=host, port=port)
# setup device before any test
device = DeviceHelper.getDevice(runAdbAsRoot=True)
for g in groups:
logger.debug("running securitysuite test group %s" % g)
logger.test_start(g)
try:
ExtraTest.run(g, version=version)
logger.test_end(g, 'OK')
except:
logger.critical(traceback.format_exc())
logger.test_end(g, 'FAIL')
raise
logger.suite_end()
def create_marionette():
"""Returns current Marionette session, or creates one if
one does not exist.
"""
m = TestCase.stored.marionette
if m is None:
m = DeviceHelper.getMarionette(host=_host, port=_port)
m.wait_for_port()
m.start_session()
TestCase.stored.marionette = m
return TestCase.stored.marionette
def create_marionette():
"""Returns current Marionette session, or creates one if
one does not exist.
"""
m = TestCase.stored.marionette
if m is None:
m = DeviceHelper.getMarionette(host=_host, port=_port)
m.wait_for_port()
m.start_session()
TestCase.stored.marionette = m
return TestCase.stored.marionette
def _run(args, logger):
# This function is to simply make the cli() function easier to handle
test_groups = [
'omni-analyzer',
'permissions',
'webapi',
'user-agent',
'crash-reporter'
]
if args.list_test_groups:
for t in test_groups:
print t
return 0
skip_tests = []
test_groups = set(args.include if args.include else test_groups)
if args.device_profile:
skiplist = []
with open(args.device_profile, 'r') as device_profile_file:
skiplist = json.load(device_profile_file)['result']['cert']
skip_tests = [x for x in test_groups if x in skiplist]
test_groups = [x for x in test_groups if x not in skiplist]
report = {'buildprops': {}}
logging.basicConfig()
# Step 1: Get device information
try:
dm = DeviceHelper.getDevice()
except mozdevice.DMError as e:
print "Error connecting to device via adb (error: %s). Please be " \
"sure device is connected and 'remote debugging' is enabled." % \
e.msg
raise
# wait here to make sure marionette is running
logger.debug('Attempting to set up port forwarding for marionette')
retries = 0
while retries < 5:
try:
m = DeviceHelper.getMarionette()
m.start_session()
m.delete_session()
break
except (IOError, TypeError):
time.sleep(5)
retries += 1
else:
raise Exception("Couldn't connect to marionette after %d attempts. " \
"Is the marionette extension installed?" % retries)
# if args.version not in supported_versions:
# print "%s is not a valid version. Please enter one of %s" % \
# (args.version, supported_versions)
# raise Exception("%s is not a valid version" % args.version)
result_file_path = args.result_file
if not result_file_path:
result_file_path = "results.json"
# Make sure we can write to the results file before running tests.
# This will also ensure this file exists in case we error out later on.
try:
result_file = open(result_file_path, "w")
result_file.close()
except IOError as e:
print 'Could not open result file for writing: %s errno: %d' % (result_file_path, e.errno)
raise
report['buildprops'] = get_buildprop(dm)
report['processes_running'] = get_processes_running(dm)
report['kernel_version'] = get_kernel_version(dm)
report['application_ini'] = get_application_ini(dm)
logger.suite_start(tests=[])
# record skipped test to report
for test in skip_tests:
logger.test_start(test)
logger.test_end(test, 'SKIP', message='Skipped by device profile')
# run the omni.ja analyzer
if 'omni-analyzer' in test_groups:
test_omni_analyzer(logger, report, args)
# start webserver
if 'webapi' in test_groups or 'permissions' in test_groups:
httpd = wptserve.server.WebTestHttpd(
host=moznetwork.get_ip(), port=8000, routes=routes, doc_root=static_path)
httpd.start()
addr = (httpd.host, httpd.port)
# run webapi and webidl tests
if 'webapi' in test_groups:
test_webapi(logger, report, args, addr)
if 'permissions' in test_groups:
test_permissions(logger, report, args, addr)
if 'user-agent' in test_groups:
test_user_agent(logger, report)
if 'crash-reporter' in test_groups:
test_crash_reporter(logger, report)
logger.suite_end()
with open(result_file_path, "w") as result_file:
result_file.write(json.dumps(report, indent=2))
logger.debug('Results have been stored in: %s' % result_file_path)
if args.html_result_file is not None:
make_html_report(args.html_result_file, report)
logger.debug('HTML Results have been stored in: %s' % args.html_result_file)
log_results(added_webidl_results, logger, report, 'webapi', prefix + 'added-webidl-results')
log_results(missing_webidl_results, logger, report, 'webapi', prefix + 'missing-webidl-results')
def parse_permissions_results(expected_results_path, results, prefix, logger, report):
with open(expected_results_path) as f:
expected_results = json.load(f)
# compute differences in permissions results
unexpected_results = diff_results(expected_results, results)
log_results(unexpected_results, logger, report, 'permissions', prefix + 'unexpected-permissions-results')
return not unexpected_results
def run_marionette_script(script, chrome=False, async=False, host='localhost', port=2828):
"""Create a Marionette instance and run the provided script"""
m = DeviceHelper.getMarionette(host, port)
m.start_session()
if chrome:
m.set_context(marionette.Marionette.CONTEXT_CHROME)
if not async:
result = m.execute_script(script)
else:
result = m.execute_async_script(script)
m.delete_session()
return result
def kill(name):
"""Kill the specified app"""
script = """
let manager = window.wrappedJSObject.appWindowManager || new window.wrappedJSObject.AppWindowManager();
def _run(args, logger):
# This function is to simply make the cli() function easier to handle
test_groups = [
'omni-analyzer', 'permissions', 'webapi', 'user-agent',
'crash-reporter'
]
if args.list_test_groups:
for t in test_groups:
print t
return 0
skip_tests = []
test_groups = set(args.include if args.include else test_groups)
if args.device_profile:
skiplist = []
with open(args.device_profile, 'r') as device_profile_file:
skiplist = json.load(device_profile_file)['result']['cert']
skip_tests = [x for x in test_groups if x in skiplist]
test_groups = [x for x in test_groups if x not in skiplist]
report = {'buildprops': {}}
logging.basicConfig()
# Step 1: Get device information
try:
dm = DeviceHelper.getDevice()
except mozdevice.DMError as e:
print "Error connecting to device via adb (error: %s). Please be " \
"sure device is connected and 'remote debugging' is enabled." % \
e.msg
raise
# wait here to make sure marionette is running
logger.debug('Attempting to set up port forwarding for marionette')
retries = 0
while retries < 5:
try:
m = DeviceHelper.getMarionette()
m.start_session()
m.delete_session()
break
except (IOError, TypeError):
time.sleep(5)
retries += 1
else:
raise Exception("Couldn't connect to marionette after %d attempts. " \
"Is the marionette extension installed?" % retries)
# if args.version not in supported_versions:
# print "%s is not a valid version. Please enter one of %s" % \
# (args.version, supported_versions)
# raise Exception("%s is not a valid version" % args.version)
result_file_path = args.result_file
if not result_file_path:
result_file_path = "results.json"
# Make sure we can write to the results file before running tests.
# This will also ensure this file exists in case we error out later on.
try:
result_file = open(result_file_path, "w")
result_file.close()
except IOError as e:
print 'Could not open result file for writing: %s errno: %d' % (
result_file_path, e.errno)
raise
report['buildprops'] = get_buildprop(dm)
report['processes_running'] = get_processes_running(dm)
report['kernel_version'] = get_kernel_version(dm)
report['application_ini'] = get_application_ini(dm)
logger.suite_start(tests=[])
# record skipped test to report
for test in skip_tests:
logger.test_start(test)
logger.test_end(test, 'SKIP', message='Skipped by device profile')
# run the omni.ja analyzer
if 'omni-analyzer' in test_groups:
test_omni_analyzer(logger, report, args)
# start webserver
if 'webapi' in test_groups or 'permissions' in test_groups:
httpd = wptserve.server.WebTestHttpd(host=moznetwork.get_ip(),
port=8000,
routes=routes,
doc_root=static_path)
httpd.start()
addr = (httpd.host, httpd.port)
# run webapi and webidl tests
if 'webapi' in test_groups:
test_webapi(logger, report, args, addr)
if 'permissions' in test_groups:
test_permissions(logger, report, args, addr)
if 'user-agent' in test_groups:
test_user_agent(logger, report)
if 'crash-reporter' in test_groups:
test_crash_reporter(logger, report)
logger.suite_end()
with open(result_file_path, "w") as result_file:
result_file.write(json.dumps(report, indent=2))
logger.debug('Results have been stored in: %s' % result_file_path)
if args.html_result_file is not None:
make_html_report(args.html_result_file, report)
logger.debug('HTML Results have been stored in: %s' %
args.html_result_file)
expected_results = json.load(f)
# compute differences in permissions results
unexpected_results = diff_results(expected_results, results)
log_results(unexpected_results, logger, report, 'permissions',
prefix + 'unexpected-permissions-results')
return not unexpected_results
def run_marionette_script(script,
chrome=False,
async=False,
host='localhost',
port=2828):
"""Create a Marionette instance and run the provided script"""
m = DeviceHelper.getMarionette(host, port)
m.start_session()
if chrome:
m.set_context(marionette.Marionette.CONTEXT_CHROME)
if not async:
result = m.execute_script(script)
else:
result = m.execute_async_script(script)
m.delete_session()
return result
def kill(name):
"""Kill the specified app"""
script = """
let manager = window.wrappedJSObject.appWindowManager || new window.wrappedJSObject.AppWindowManager();