def run_groups(groups=[], version=None, host='localhost', port=2828, mode='phone'):
hasadb = mode == 'phone'
logger = get_default_logger()
if groups is None or len(groups) == 0: # run all groups
logger.debug('running securitysuite tests for all groups %s' % str(ExtraTest.group_list(mode=mode)))
groups = ExtraTest.group_list(mode=mode)
else:
logger.debug('running securitysuite tests for groups %s' % str(groups))
logger.suite_start(tests=groups)
# setup marionette before any test
marionette = DeviceHelper.getMarionette(host=host, port=port)
# setup device before any test
device = DeviceHelper.getDevice(runAdbAsRoot=True)
for g in groups:
logger.debug("running securitysuite test group %s" % g)
logger.test_start(g)
try:
ExtraTest.run(g, version=version)
logger.test_end(g, 'OK')
except:
logger.critical(traceback.format_exc())
logger.test_end(g, 'FAIL')
raise
logger.suite_end()
def run_groups(groups=[], version=None, host='localhost', port=2828, mode='phone'):
hasadb = mode == 'phone'
logger = get_default_logger()
if groups is None or len(groups) == 0: # run all groups
logger.debug('running securitysuite tests for all groups %s' % str(ExtraTest.group_list(mode=mode)))
groups = ExtraTest.group_list(mode=mode)
else:
logger.debug('running securitysuite tests for groups %s' % str(groups))
logger.suite_start(tests=groups)
# setup marionette before any test
marionette = DeviceHelper.getMarionette(host=host, port=port)
# setup device before any test
device = DeviceHelper.getDevice()
for g in groups:
logger.debug("running securitysuite test group %s" % g)
logger.test_start(g)
try:
ExtraTest.run(g, version=version)
logger.test_end(g, 'OK')
except:
logger.critical(traceback.format_exc())
logger.test_end(g, 'FAIL')
raise
logger.suite_end()
def wait_for_adb_device():
try:
adb = DeviceHelper.getDevice(runAdbAsRoot=True)
except DMError:
adb = None
print "Waiting for adb connection..."
while adb is None:
try:
adb = DeviceHelper.getDevice(runAdbAsRoot=True)
except DMError:
sleep(0.2)
if len(adb.devices()) < 1:
print "Waiting for adb device..."
while len(adb.devices()) < 1:
sleep(0.2)
def wait_for_adb_device():
try:
device = DeviceHelper.getDevice()
except ADBError:
device = None
print "Waiting for adb connection..."
while device is None:
try:
device = DeviceHelper.getDevice()
except ADBError:
sleep(0.2)
if len(device.devices()) < 1:
print "Waiting for adb device..."
while len(device.devices()) < 1:
sleep(0.2)
def wait_for_adb_device():
try:
device = DeviceHelper.getDevice()
except ADBError:
device = None
print "Waiting for adb connection..."
while device is None:
try:
device = DeviceHelper.getDevice()
except ADBError:
sleep(0.2)
if len(device.devices()) < 1:
print "Waiting for adb device..."
while len(device.devices()) < 1:
sleep(0.2)
def run(cls, version=None):
logger = get_default_logger()
try:
dm = DeviceHelper.getDevice(runAdbAsRoot=True)
except mozdevice.DMError as e:
logger.error("Error connecting to device via adb (error: %s). Please be " \
"sure device is connected and 'remote debugging' is enabled." % \
e.msg)
raise
try:
out = dm.shellCheckOutput(['ls', '-alR', '/'], root=True)
except mozdevice.DMError as e:
cls.log_status('FAIL', 'Failed to gather filesystem information from device via adb: %s' % e.msg)
return False
found = []
for f in parse_ls(out):
if f['perm'][7] == 'w' and f['mode'] != 'l':
if not cls.whitelist_check(f['name']):
found.append(f['name'])
if len(found) > 0:
cls.log_status('PASS',
'Please ensure that the following world-writable files will not pose a security risk:\n%s' % '\n'.join(
found))
else:
cls.log_status('PASS', 'No unexpected suidroot executables found.')
return True
def run(cls, version=None):
logger = get_default_logger()
try:
device = DeviceHelper.getDevice()
except ADBError as e:
logger.error("Error connecting to device via adb (error: %s). Please be " \
"sure device is connected and 'remote debugging' is enabled." % \
e.msg)
raise
try:
out = device.shell_output('ls -alR /', root=True)
except ADBError as e:
cls.log_status(
'FAIL',
'Failed to gather filesystem information from device via adb: %s'
% e.msg)
return False
found = []
for f in parse_ls(out):
if f['perm'][2] == 's' and f['uid'] == 'root':
if not cls.whitelist_check(f['name']):
found.append(f['name'])
if len(found) > 0:
cls.log_status(
'PASS',
'Please ensure that the following suid root files are no security risk:\n%s'
% '\n'.join(found))
else:
cls.log_status('PASS', 'No unexpected suidroot executables found.')
return True
def run(cls, version=None):
logger = get_default_logger()
try:
device = DeviceHelper.getDevice()
except ADBError as e:
logger.error("Error connecting to device via adb (error: %s). Please be " \
"sure device is connected and 'remote debugging' is enabled." % \
e.msg)
raise
try:
out = device.shell_output('ls -alR /', root=True)
except ADBError as e:
cls.log_status('FAIL', 'Failed to gather filesystem information from device via adb: %s' % e.msg)
return False
found = []
for f in parse_ls(out):
if f['perm'][2] == 's' and f['uid'] == 'root':
if not cls.whitelist_check(f['name']):
found.append(f['name'])
if len(found) > 0:
cls.log_status('PASS',
'Please ensure that the following suid root files are no security risk:\n%s' % '\n'.join(
found))
else:
cls.log_status('PASS', 'No unexpected suidroot executables found.')
return True
def close_app_manually(self):
success = self.instruct("Could not close %s automatically. "
"Please close the app manually by holding down the Home button "
"and pressing the X above the %s card." % (certapp.name, certapp.name))
if not success:
device = DeviceHelper.getDevice()
device.reboot(wait=True)
self.instruct("Please unlock the lockscreen (if present) after device reboots")
self.fail("Failed attempts at closing certapp")
def __init__(self):
self.logger = get_default_logger()
try:
self.device = DeviceHelper.getDevice()
except ADBError as e:
self.logger.error("Error connecting to device via adb (error: %s). Please be "
"sure device is connected and 'remote debugging' is enabled." %
e.msg)
raise
def __init__(self):
self.logger = get_default_logger()
try:
self.device = DeviceHelper.getDevice()
except ADBError as e:
self.logger.error(
"Error connecting to device via adb (error: %s). Please be "
"sure device is connected and 'remote debugging' is enabled." %
e.msg)
raise
def __init__(self, *args, **kwargs):
self.version = kwargs.pop('version')
super(TestCase, self).__init__(*args, **kwargs)
self.stored.handler = None
self.stored.marionette = None
self.marionette, self.server, self.handler, self.app = None, None, None, None
device = DeviceHelper.getDevice()
# Cleanups are run irrespective of whether setUp fails
self.addCleanup(self.cleanup)
def __init__(self, *args, **kwargs):
self.version = kwargs.pop('version')
super(TestCase, self).__init__(*args, **kwargs)
self.stored.handler = None
self.stored.marionette = None
self.marionette, self.server, self.handler, self.app = None, None, None, None
device = DeviceHelper.getDevice()
# Cleanups are run irrespective of whether setUp fails
self.addCleanup(self.cleanup)
def close_app_manually(self):
success = self.instruct(
"Could not close %s automatically. "
"Please close the app manually by holding down the Home button "
"and pressing the X above the %s card." %
(certapp.name, certapp.name))
if not success:
device = DeviceHelper.getDevice()
device.reboot(wait=True)
self.instruct(
"Please unlock the lockscreen (if present) after device reboots"
)
self.fail("Failed attempts at closing certapp")
def create_marionette():
"""Returns current Marionette session, or creates one if
one does not exist.
"""
m = TestCase.stored.marionette
if m is None:
m = DeviceHelper.getMarionette(host=_host, port=_port)
m.wait_for_port()
m.start_session()
TestCase.stored.marionette = m
return TestCase.stored.marionette
def create_marionette():
"""Returns current Marionette session, or creates one if
one does not exist.
"""
m = TestCase.stored.marionette
if m is None:
m = DeviceHelper.getMarionette(host=_host, port=_port)
m.wait_for_port()
m.start_session()
TestCase.stored.marionette = m
return TestCase.stored.marionette
def __init__(self):
self.logger = get_default_logger()
try:
self.device = DeviceHelper.getDevice()
except ADBError as e:
self.logger.error("Error connecting to device via adb (error: %s). Please be "
"sure device is connected and 'remote debugging' is enabled." %
e.msg)
raise
try:
self.ps = self.device.shell_output('b2g-ps', root=True).split('\r\n')
except ADBError as e:
self.logger.error("Error reading b2g-ps result from device: %s" % e.msg)
raise
def __init__(self):
self.logger = get_default_logger()
try:
self.device = DeviceHelper.getDevice()
except ADBError as e:
self.logger.error(
"Error connecting to device via adb (error: %s). Please be "
"sure device is connected and 'remote debugging' is enabled." %
e.msg)
raise
try:
self.ps = self.device.shell_output('b2g-ps',
root=True).split('\r\n')
except ADBError as e:
self.logger.error("Error reading b2g-ps result from device: %s" %
e.msg)
raise
def _run(args, logger):
# This function is to simply make the cli() function easier to handle
test_groups = [
'omni-analyzer',
'permissions',
'webapi',
'user-agent',
'crash-reporter'
]
if args.list_test_groups:
for t in test_groups:
print t
return 0
skip_tests = []
test_groups = set(args.include if args.include else test_groups)
if args.device_profile:
skiplist = []
with open(args.device_profile, 'r') as device_profile_file:
skiplist = json.load(device_profile_file)['result']['cert']
skip_tests = [x for x in test_groups if x in skiplist]
test_groups = [x for x in test_groups if x not in skiplist]
report = {'buildprops': {}}
logging.basicConfig()
# Step 1: Get device information
try:
dm = DeviceHelper.getDevice()
except mozdevice.DMError as e:
print "Error connecting to device via adb (error: %s). Please be " \
"sure device is connected and 'remote debugging' is enabled." % \
e.msg
raise
# wait here to make sure marionette is running
logger.debug('Attempting to set up port forwarding for marionette')
retries = 0
while retries < 5:
try:
m = DeviceHelper.getMarionette()
m.start_session()
m.delete_session()
break
except (IOError, TypeError):
time.sleep(5)
retries += 1
else:
raise Exception("Couldn't connect to marionette after %d attempts. " \
"Is the marionette extension installed?" % retries)
# if args.version not in supported_versions:
# print "%s is not a valid version. Please enter one of %s" % \
# (args.version, supported_versions)
# raise Exception("%s is not a valid version" % args.version)
result_file_path = args.result_file
if not result_file_path:
result_file_path = "results.json"
# Make sure we can write to the results file before running tests.
# This will also ensure this file exists in case we error out later on.
try:
result_file = open(result_file_path, "w")
result_file.close()
except IOError as e:
print 'Could not open result file for writing: %s errno: %d' % (result_file_path, e.errno)
raise
report['buildprops'] = get_buildprop(dm)
report['processes_running'] = get_processes_running(dm)
report['kernel_version'] = get_kernel_version(dm)
report['application_ini'] = get_application_ini(dm)
logger.suite_start(tests=[])
# record skipped test to report
for test in skip_tests:
logger.test_start(test)
logger.test_end(test, 'SKIP', message='Skipped by device profile')
# run the omni.ja analyzer
if 'omni-analyzer' in test_groups:
test_omni_analyzer(logger, report, args)
# start webserver
if 'webapi' in test_groups or 'permissions' in test_groups:
httpd = wptserve.server.WebTestHttpd(
host=moznetwork.get_ip(), port=8000, routes=routes, doc_root=static_path)
httpd.start()
addr = (httpd.host, httpd.port)
# run webapi and webidl tests
if 'webapi' in test_groups:
test_webapi(logger, report, args, addr)
if 'permissions' in test_groups:
test_permissions(logger, report, args, addr)
if 'user-agent' in test_groups:
test_user_agent(logger, report)
if 'crash-reporter' in test_groups:
test_crash_reporter(logger, report)
logger.suite_end()
with open(result_file_path, "w") as result_file:
result_file.write(json.dumps(report, indent=2))
logger.debug('Results have been stored in: %s' % result_file_path)
if args.html_result_file is not None:
make_html_report(args.html_result_file, report)
logger.debug('HTML Results have been stored in: %s' % args.html_result_file)
expected_results = json.load(f)
# compute differences in permissions results
unexpected_results = diff_results(expected_results, results)
log_results(unexpected_results, logger, report, 'permissions',
prefix + 'unexpected-permissions-results')
return not unexpected_results
def run_marionette_script(script,
chrome=False,
async=False,
host='localhost',
port=2828):
"""Create a Marionette instance and run the provided script"""
m = DeviceHelper.getMarionette(host, port)
m.start_session()
if chrome:
m.set_context(marionette.Marionette.CONTEXT_CHROME)
if not async:
result = m.execute_script(script)
else:
result = m.execute_async_script(script)
m.delete_session()
return result
def kill(name):
"""Kill the specified app"""
script = """
let manager = window.wrappedJSObject.appWindowManager || new window.wrappedJSObject.AppWindowManager();
def _run(args, logger):
# This function is to simply make the cli() function easier to handle
test_groups = [
'omni-analyzer', 'permissions', 'webapi', 'user-agent',
'crash-reporter'
]
if args.list_test_groups:
for t in test_groups:
print t
return 0
skip_tests = []
test_groups = set(args.include if args.include else test_groups)
if args.device_profile:
skiplist = []
with open(args.device_profile, 'r') as device_profile_file:
skiplist = json.load(device_profile_file)['result']['cert']
skip_tests = [x for x in test_groups if x in skiplist]
test_groups = [x for x in test_groups if x not in skiplist]
report = {'buildprops': {}}
logging.basicConfig()
# Step 1: Get device information
try:
dm = DeviceHelper.getDevice()
except mozdevice.DMError as e:
print "Error connecting to device via adb (error: %s). Please be " \
"sure device is connected and 'remote debugging' is enabled." % \
e.msg
raise
# wait here to make sure marionette is running
logger.debug('Attempting to set up port forwarding for marionette')
retries = 0
while retries < 5:
try:
m = DeviceHelper.getMarionette()
m.start_session()
m.delete_session()
break
except (IOError, TypeError):
time.sleep(5)
retries += 1
else:
raise Exception("Couldn't connect to marionette after %d attempts. " \
"Is the marionette extension installed?" % retries)
# if args.version not in supported_versions:
# print "%s is not a valid version. Please enter one of %s" % \
# (args.version, supported_versions)
# raise Exception("%s is not a valid version" % args.version)
result_file_path = args.result_file
if not result_file_path:
result_file_path = "results.json"
# Make sure we can write to the results file before running tests.
# This will also ensure this file exists in case we error out later on.
try:
result_file = open(result_file_path, "w")
result_file.close()
except IOError as e:
print 'Could not open result file for writing: %s errno: %d' % (
result_file_path, e.errno)
raise
report['buildprops'] = get_buildprop(dm)
report['processes_running'] = get_processes_running(dm)
report['kernel_version'] = get_kernel_version(dm)
report['application_ini'] = get_application_ini(dm)
logger.suite_start(tests=[])
# record skipped test to report
for test in skip_tests:
logger.test_start(test)
logger.test_end(test, 'SKIP', message='Skipped by device profile')
# run the omni.ja analyzer
if 'omni-analyzer' in test_groups:
test_omni_analyzer(logger, report, args)
# start webserver
if 'webapi' in test_groups or 'permissions' in test_groups:
httpd = wptserve.server.WebTestHttpd(host=moznetwork.get_ip(),
port=8000,
routes=routes,
doc_root=static_path)
httpd.start()
addr = (httpd.host, httpd.port)
# run webapi and webidl tests
if 'webapi' in test_groups:
test_webapi(logger, report, args, addr)
if 'permissions' in test_groups:
test_permissions(logger, report, args, addr)
if 'user-agent' in test_groups:
test_user_agent(logger, report)
if 'crash-reporter' in test_groups:
test_crash_reporter(logger, report)
logger.suite_end()
with open(result_file_path, "w") as result_file:
result_file.write(json.dumps(report, indent=2))
logger.debug('Results have been stored in: %s' % result_file_path)
if args.html_result_file is not None:
make_html_report(args.html_result_file, report)
logger.debug('HTML Results have been stored in: %s' %
args.html_result_file)
def adb_has_root():
# normally this should check via root=True to .shell_output, but doesn't work
device = DeviceHelper.getDevice()
return device.shell_output("id").startswith("uid=0(root)")
def adb_has_root():
# normally this should check via root=True to .shellCheckOutput, but doesn't work
adb = DeviceHelper.getDevice(runAdbAsRoot=True)
return adb.shellCheckOutput(["id"]).startswith("uid=0(root)")
log_results(added_webidl_results, logger, report, 'webapi', prefix + 'added-webidl-results')
log_results(missing_webidl_results, logger, report, 'webapi', prefix + 'missing-webidl-results')
def parse_permissions_results(expected_results_path, results, prefix, logger, report):
with open(expected_results_path) as f:
expected_results = json.load(f)
# compute differences in permissions results
unexpected_results = diff_results(expected_results, results)
log_results(unexpected_results, logger, report, 'permissions', prefix + 'unexpected-permissions-results')
return not unexpected_results
def run_marionette_script(script, chrome=False, async=False, host='localhost', port=2828):
"""Create a Marionette instance and run the provided script"""
m = DeviceHelper.getMarionette(host, port)
m.start_session()
if chrome:
m.set_context(marionette.Marionette.CONTEXT_CHROME)
if not async:
result = m.execute_script(script)
else:
result = m.execute_async_script(script)
m.delete_session()
return result
def kill(name):
"""Kill the specified app"""
script = """
let manager = window.wrappedJSObject.appWindowManager || new window.wrappedJSObject.AppWindowManager();
def adb_has_root():
# normally this should check via root=True to .shell_output, but doesn't work
device = DeviceHelper.getDevice()
return device.shell_output("id").startswith("uid=0(root)")
