def run_groups(groups=[], version=None, host='localhost', port=2828, mode='phone'): hasadb = mode == 'phone' logger = get_default_logger() if groups is None or len(groups) == 0: # run all groups logger.debug('running securitysuite tests for all groups %s' % str(ExtraTest.group_list(mode=mode))) groups = ExtraTest.group_list(mode=mode) else: logger.debug('running securitysuite tests for groups %s' % str(groups)) logger.suite_start(tests=groups) # setup marionette before any test marionette = DeviceHelper.getMarionette(host=host, port=port) # setup device before any test device = DeviceHelper.getDevice() for g in groups: logger.debug("running securitysuite test group %s" % g) logger.test_start(g) try: ExtraTest.run(g, version=version) logger.test_end(g, 'OK') except: logger.critical(traceback.format_exc()) logger.test_end(g, 'FAIL') raise logger.suite_end()
def run_groups(groups=[], version=None, host='localhost', port=2828, mode='phone'): hasadb = mode == 'phone' logger = get_default_logger() if groups is None or len(groups) == 0: # run all groups logger.debug('running securitysuite tests for all groups %s' % str(ExtraTest.group_list(mode=mode))) groups = ExtraTest.group_list(mode=mode) else: logger.debug('running securitysuite tests for groups %s' % str(groups)) logger.suite_start(tests=groups) # setup marionette before any test marionette = DeviceHelper.getMarionette(host=host, port=port) # setup device before any test device = DeviceHelper.getDevice(runAdbAsRoot=True) for g in groups: logger.debug("running securitysuite test group %s" % g) logger.test_start(g) try: ExtraTest.run(g, version=version) logger.test_end(g, 'OK') except: logger.critical(traceback.format_exc()) logger.test_end(g, 'FAIL') raise logger.suite_end()
def create_marionette(): """Returns current Marionette session, or creates one if one does not exist. """ m = TestCase.stored.marionette if m is None: m = DeviceHelper.getMarionette(host=_host, port=_port) m.wait_for_port() m.start_session() TestCase.stored.marionette = m return TestCase.stored.marionette
def create_marionette(): """Returns current Marionette session, or creates one if one does not exist. """ m = TestCase.stored.marionette if m is None: m = DeviceHelper.getMarionette(host=_host, port=_port) m.wait_for_port() m.start_session() TestCase.stored.marionette = m return TestCase.stored.marionette
def _run(args, logger): # This function is to simply make the cli() function easier to handle test_groups = [ 'omni-analyzer', 'permissions', 'webapi', 'user-agent', 'crash-reporter' ] if args.list_test_groups: for t in test_groups: print t return 0 skip_tests = [] test_groups = set(args.include if args.include else test_groups) if args.device_profile: skiplist = [] with open(args.device_profile, 'r') as device_profile_file: skiplist = json.load(device_profile_file)['result']['cert'] skip_tests = [x for x in test_groups if x in skiplist] test_groups = [x for x in test_groups if x not in skiplist] report = {'buildprops': {}} logging.basicConfig() # Step 1: Get device information try: dm = DeviceHelper.getDevice() except mozdevice.DMError as e: print "Error connecting to device via adb (error: %s). Please be " \ "sure device is connected and 'remote debugging' is enabled." % \ e.msg raise # wait here to make sure marionette is running logger.debug('Attempting to set up port forwarding for marionette') retries = 0 while retries < 5: try: m = DeviceHelper.getMarionette() m.start_session() m.delete_session() break except (IOError, TypeError): time.sleep(5) retries += 1 else: raise Exception("Couldn't connect to marionette after %d attempts. " \ "Is the marionette extension installed?" % retries) # if args.version not in supported_versions: # print "%s is not a valid version. Please enter one of %s" % \ # (args.version, supported_versions) # raise Exception("%s is not a valid version" % args.version) result_file_path = args.result_file if not result_file_path: result_file_path = "results.json" # Make sure we can write to the results file before running tests. # This will also ensure this file exists in case we error out later on. try: result_file = open(result_file_path, "w") result_file.close() except IOError as e: print 'Could not open result file for writing: %s errno: %d' % (result_file_path, e.errno) raise report['buildprops'] = get_buildprop(dm) report['processes_running'] = get_processes_running(dm) report['kernel_version'] = get_kernel_version(dm) report['application_ini'] = get_application_ini(dm) logger.suite_start(tests=[]) # record skipped test to report for test in skip_tests: logger.test_start(test) logger.test_end(test, 'SKIP', message='Skipped by device profile') # run the omni.ja analyzer if 'omni-analyzer' in test_groups: test_omni_analyzer(logger, report, args) # start webserver if 'webapi' in test_groups or 'permissions' in test_groups: httpd = wptserve.server.WebTestHttpd( host=moznetwork.get_ip(), port=8000, routes=routes, doc_root=static_path) httpd.start() addr = (httpd.host, httpd.port) # run webapi and webidl tests if 'webapi' in test_groups: test_webapi(logger, report, args, addr) if 'permissions' in test_groups: test_permissions(logger, report, args, addr) if 'user-agent' in test_groups: test_user_agent(logger, report) if 'crash-reporter' in test_groups: test_crash_reporter(logger, report) logger.suite_end() with open(result_file_path, "w") as result_file: result_file.write(json.dumps(report, indent=2)) logger.debug('Results have been stored in: %s' % result_file_path) if args.html_result_file is not None: make_html_report(args.html_result_file, report) logger.debug('HTML Results have been stored in: %s' % args.html_result_file)
log_results(added_webidl_results, logger, report, 'webapi', prefix + 'added-webidl-results') log_results(missing_webidl_results, logger, report, 'webapi', prefix + 'missing-webidl-results') def parse_permissions_results(expected_results_path, results, prefix, logger, report): with open(expected_results_path) as f: expected_results = json.load(f) # compute differences in permissions results unexpected_results = diff_results(expected_results, results) log_results(unexpected_results, logger, report, 'permissions', prefix + 'unexpected-permissions-results') return not unexpected_results def run_marionette_script(script, chrome=False, async=False, host='localhost', port=2828): """Create a Marionette instance and run the provided script""" m = DeviceHelper.getMarionette(host, port) m.start_session() if chrome: m.set_context(marionette.Marionette.CONTEXT_CHROME) if not async: result = m.execute_script(script) else: result = m.execute_async_script(script) m.delete_session() return result def kill(name): """Kill the specified app""" script = """ let manager = window.wrappedJSObject.appWindowManager || new window.wrappedJSObject.AppWindowManager();
def _run(args, logger): # This function is to simply make the cli() function easier to handle test_groups = [ 'omni-analyzer', 'permissions', 'webapi', 'user-agent', 'crash-reporter' ] if args.list_test_groups: for t in test_groups: print t return 0 skip_tests = [] test_groups = set(args.include if args.include else test_groups) if args.device_profile: skiplist = [] with open(args.device_profile, 'r') as device_profile_file: skiplist = json.load(device_profile_file)['result']['cert'] skip_tests = [x for x in test_groups if x in skiplist] test_groups = [x for x in test_groups if x not in skiplist] report = {'buildprops': {}} logging.basicConfig() # Step 1: Get device information try: dm = DeviceHelper.getDevice() except mozdevice.DMError as e: print "Error connecting to device via adb (error: %s). Please be " \ "sure device is connected and 'remote debugging' is enabled." % \ e.msg raise # wait here to make sure marionette is running logger.debug('Attempting to set up port forwarding for marionette') retries = 0 while retries < 5: try: m = DeviceHelper.getMarionette() m.start_session() m.delete_session() break except (IOError, TypeError): time.sleep(5) retries += 1 else: raise Exception("Couldn't connect to marionette after %d attempts. " \ "Is the marionette extension installed?" % retries) # if args.version not in supported_versions: # print "%s is not a valid version. Please enter one of %s" % \ # (args.version, supported_versions) # raise Exception("%s is not a valid version" % args.version) result_file_path = args.result_file if not result_file_path: result_file_path = "results.json" # Make sure we can write to the results file before running tests. # This will also ensure this file exists in case we error out later on. try: result_file = open(result_file_path, "w") result_file.close() except IOError as e: print 'Could not open result file for writing: %s errno: %d' % ( result_file_path, e.errno) raise report['buildprops'] = get_buildprop(dm) report['processes_running'] = get_processes_running(dm) report['kernel_version'] = get_kernel_version(dm) report['application_ini'] = get_application_ini(dm) logger.suite_start(tests=[]) # record skipped test to report for test in skip_tests: logger.test_start(test) logger.test_end(test, 'SKIP', message='Skipped by device profile') # run the omni.ja analyzer if 'omni-analyzer' in test_groups: test_omni_analyzer(logger, report, args) # start webserver if 'webapi' in test_groups or 'permissions' in test_groups: httpd = wptserve.server.WebTestHttpd(host=moznetwork.get_ip(), port=8000, routes=routes, doc_root=static_path) httpd.start() addr = (httpd.host, httpd.port) # run webapi and webidl tests if 'webapi' in test_groups: test_webapi(logger, report, args, addr) if 'permissions' in test_groups: test_permissions(logger, report, args, addr) if 'user-agent' in test_groups: test_user_agent(logger, report) if 'crash-reporter' in test_groups: test_crash_reporter(logger, report) logger.suite_end() with open(result_file_path, "w") as result_file: result_file.write(json.dumps(report, indent=2)) logger.debug('Results have been stored in: %s' % result_file_path) if args.html_result_file is not None: make_html_report(args.html_result_file, report) logger.debug('HTML Results have been stored in: %s' % args.html_result_file)
expected_results = json.load(f) # compute differences in permissions results unexpected_results = diff_results(expected_results, results) log_results(unexpected_results, logger, report, 'permissions', prefix + 'unexpected-permissions-results') return not unexpected_results def run_marionette_script(script, chrome=False, async=False, host='localhost', port=2828): """Create a Marionette instance and run the provided script""" m = DeviceHelper.getMarionette(host, port) m.start_session() if chrome: m.set_context(marionette.Marionette.CONTEXT_CHROME) if not async: result = m.execute_script(script) else: result = m.execute_async_script(script) m.delete_session() return result def kill(name): """Kill the specified app""" script = """ let manager = window.wrappedJSObject.appWindowManager || new window.wrappedJSObject.AppWindowManager();