def run_tests(self): dirs = self.query_abs_dirs() cmd = self._query_cmd() parser = StructuredOutputParser(config=self.config, log_obj=self.log_obj, log_compact=True) env = {'MINIDUMP_SAVE_PATH': dirs['abs_blob_upload_dir']} env['RUST_BACKTRACE'] = '1' if self.config['allow_software_gl_layers']: env['MOZ_LAYERS_ALLOW_SOFTWARE_GL'] = '1' if self.config['enable_webrender']: env['MOZ_WEBRENDER'] = '1' env = self.query_env(partial_env=env, log_level=INFO) return_code = self.run_command(cmd, cwd=dirs['abs_work_dir'], output_timeout=1000, output_parser=parser, env=env) tbpl_status, log_level = parser.evaluate_parser(return_code) self.buildbot_status(tbpl_status, level=log_level)
def run_test(self, binary_path, env=None, marionette_port=2828): """All required steps for running the tests against an installer.""" dirs = self.query_abs_dirs() # Import the harness to retrieve the location of the cli scripts import firefox_ui_harness cmd = [ self.query_python_path(), os.path.join(os.path.dirname(firefox_ui_harness.__file__), self.cli_script), '--binary', binary_path, '--address', 'localhost:{}'.format(marionette_port), # Resource files to serve via local webserver '--server-root', os.path.join(dirs['abs_fxui_dir'], 'resources'), # Use the work dir to get temporary data stored '--workspace', dirs['abs_work_dir'], # logging options '--gecko-log=-', # output from the gecko process redirected to stdout '--log-raw=-', # structured log for output parser redirected to stdout # additional reports helpful for Jenkins and inpection via Treeherder '--log-html', os.path.join(dirs["abs_reports_dir"], self.reports['html']), '--log-xunit', os.path.join(dirs["abs_reports_dir"], self.reports['xunit']), ] # Collect all pass-through harness options to the script cmd.extend(self.query_harness_args()) # Set further environment settings env = env or self.query_env() if self.symbols_url: cmd.extend(['--symbols-path', self.symbols_url]) if self.query_minidump_stackwalk(): env['MINIDUMP_STACKWALK'] = self.minidump_stackwalk_path parser = StructuredOutputParser(config=self.config, log_obj=self.log_obj, strict=False) # Add the default tests to run tests = [os.path.join(dirs['abs_fxui_dir'], 'tests', test) for test in self.default_tests] cmd.extend(tests) return_code = self.run_command(cmd, cwd=dirs['abs_work_dir'], output_timeout=300, output_parser=parser, env=env) tbpl_status, log_level = parser.evaluate_parser(return_code) self.buildbot_status(tbpl_status, level=log_level) return return_code
def run_test(self, binary_path, env=None, marionette_port=2828): """All required steps for running the tests against an installer.""" dirs = self.query_abs_dirs() # Import the harness to retrieve the location of the cli scripts import telemetry_harness cmd = [ self.query_python_path(), os.path.join(os.path.dirname(telemetry_harness.__file__), self.cli_script), '--binary', binary_path, '--address', 'localhost:{}'.format(marionette_port), # Resource files to serve via local webserver '--server-root', os.path.join(dirs['abs_telemetry_dir'], 'harness', 'www'), # Use the work dir to get temporary data stored '--workspace', dirs['abs_work_dir'], # logging options '--gecko-log=-', # output from the gecko process redirected to stdout '--log-raw=-', # structured log for output parser redirected to stdout # additional reports helpful for Jenkins and inpection via Treeherder '--log-html', os.path.join(dirs['abs_blob_upload_dir'], 'report.html'), '--log-xunit', os.path.join(dirs['abs_blob_upload_dir'], 'report.xml'), # Enable tracing output to log transmission protocol '-vv', ] parser = StructuredOutputParser(config=self.config, log_obj=self.log_obj, strict=False) # Add the default tests to run tests = [os.path.join(dirs['abs_telemetry_dir'], 'tests', test) for test in self.default_tests] cmd.extend(tests) # Set further environment settings env = env or self.query_env() env.update({'MINIDUMP_SAVE_PATH': dirs['abs_blob_upload_dir']}) if self.query_minidump_stackwalk(): env.update({'MINIDUMP_STACKWALK': self.minidump_stackwalk_path}) env['RUST_BACKTRACE'] = '1' # If code coverage is enabled, set GCOV_PREFIX env variable if self.config.get('code_coverage'): env['GCOV_PREFIX'] = self.gcov_dir return_code = self.run_command(cmd, cwd=dirs['abs_work_dir'], output_timeout=300, output_parser=parser, env=env) tbpl_status, log_level = parser.evaluate_parser(return_code) self.buildbot_status(tbpl_status, level=log_level) return return_code
def run_test(self, installer_path, env=None, cleanup=True, marionette_port=2828): """All required steps for running the tests against an installer.""" dirs = self.query_abs_dirs() cmd = [ self.query_python_path(), os.path.join(dirs['fx_ui_dir'], 'firefox_ui_harness', self.cli_script), '--installer', installer_path, '--address', 'localhost:{}'.format(marionette_port), # Use the work dir to get temporary data stored '--workspace', dirs['abs_work_dir'], # logging options '--gecko-log=-', # output from the gecko process redirected to stdout '--log-raw=-', # structured log for output parser redirected to stdout # additional reports helpful for Jenkins and inpection via Treeherder '--log-html', os.path.join(dirs["abs_reports_dir"], self.reports['html']), '--log-xunit', os.path.join(dirs["abs_reports_dir"], self.reports['xunit']), ] # Collect all pass-through harness options to the script cmd.extend(self.query_extra_cmd_args()) # Set further environment settings env = env or self.query_env() if self.minidump_stackwalk_path: env['MINIDUMP_STACKWALK'] = self.minidump_stackwalk_path if self.query_symbols_url(): cmd += ['--symbols-path', self.symbols_url] parser = StructuredOutputParser(config=self.config, log_obj=self.log_obj, strict=False) return_code = self.run_command(cmd, cwd=dirs['abs_work_dir'], output_timeout=300, output_parser=parser, env=env) tbpl_status, log_level = parser.evaluate_parser(return_code) self.buildbot_status(tbpl_status, level=log_level) if cleanup: for filepath in (installer_path,): if os.path.exists(filepath): self.debug('Removing {}'.format(filepath)) os.remove(filepath) return return_code
def run_tests(self, args=None, **kw): ''' AWSY test should be implemented here ''' dirs = self.abs_dirs env = {} error_summary_file = os.path.join(dirs['abs_blob_upload_dir'], 'marionette_errorsummary.log') runtime_testvars = {'webRootDir': self.webroot_dir, 'resultsDir': self.results_dir} runtime_testvars_path = os.path.join(self.awsy_path, 'runtime-testvars.json') runtime_testvars_file = open(runtime_testvars_path, 'wb') runtime_testvars_file.write(json.dumps(runtime_testvars, indent=2)) runtime_testvars_file.close() cmd = ['marionette'] cmd.append("--preferences=%s" % os.path.join(self.awsy_path, "conf", "prefs.json")) cmd.append("--testvars=%s" % os.path.join(self.awsy_path, "conf", "testvars.json")) cmd.append("--testvars=%s" % runtime_testvars_path) cmd.append("--log-raw=-") cmd.append("--log-errorsummary=%s" % error_summary_file) cmd.append("--binary=%s" % self.binary_path) cmd.append("--profile=%s" % (os.path.join(dirs['abs_work_dir'], 'profile'))) if not self.config['e10s']: cmd.append('--disable-e10s') cmd.append('--gecko-log=%s' % os.path.join(dirs["abs_blob_upload_dir"], 'gecko.log')) test_file = os.path.join(self.awsy_libdir, 'test_memory_usage.py') cmd.append(test_file) env['MOZ_UPLOAD_DIR'] = dirs['abs_blob_upload_dir'] if not os.path.isdir(env['MOZ_UPLOAD_DIR']): self.mkdir_p(env['MOZ_UPLOAD_DIR']) env = self.query_env(partial_env=env) parser = StructuredOutputParser(config=self.config, log_obj=self.log_obj, strict=False) return_code = self.run_command(command=cmd, cwd=self.awsy_path, output_timeout=self.config.get("cmd_timeout"), env=env, output_parser=parser) level = INFO tbpl_status, log_level = parser.evaluate_parser( return_code=return_code) self.log("AWSY exited with return code %s: %s" % (return_code, tbpl_status), level=level) self.buildbot_status(tbpl_status)
def run_test(self, binary_path, env=None, marionette_port=2828): """All required steps for running the tests against an installer.""" dirs = self.query_abs_dirs() # Import the harness to retrieve the location of the cli scripts import firefox_ui_harness cmd = [ self.query_python_path(), os.path.join(os.path.dirname(firefox_ui_harness.__file__), self.cli_script), "--binary", binary_path, "--address", "localhost:{}".format(marionette_port), # Use the work dir to get temporary data stored "--workspace", dirs["abs_work_dir"], # logging options "--gecko-log=-", # output from the gecko process redirected to stdout "--log-raw=-", # structured log for output parser redirected to stdout # additional reports helpful for Jenkins and inpection via Treeherder "--log-html", os.path.join(dirs["abs_reports_dir"], self.reports["html"]), "--log-xunit", os.path.join(dirs["abs_reports_dir"], self.reports["xunit"]), ] # Collect all pass-through harness options to the script cmd.extend(self.query_harness_args()) # Set further environment settings env = env or self.query_env() if self.symbols_url: cmd.extend(["--symbols-path", self.symbols_url]) if self.query_minidump_stackwalk(): env["MINIDUMP_STACKWALK"] = self.minidump_stackwalk_path parser = StructuredOutputParser(config=self.config, log_obj=self.log_obj, strict=False) return_code = self.run_command(cmd, cwd=dirs["abs_work_dir"], output_timeout=300, output_parser=parser, env=env) tbpl_status, log_level = parser.evaluate_parser(return_code) self.buildbot_status(tbpl_status, level=log_level) return return_code
def run_tests(self): dirs = self.query_abs_dirs() cmd = self._query_cmd() parser = StructuredOutputParser(config=self.config, log_obj=self.log_obj) env = {} env = self.query_env(partial_env=env, log_level=INFO) return_code = self.run_command(cmd, cwd=dirs['abs_work_dir'], output_timeout=1000, output_parser=parser, env=env) tbpl_status, log_level = parser.evaluate_parser(return_code) self.buildbot_status(tbpl_status, level=log_level)
def run_tests(self): dirs = self.query_abs_dirs() cmd = self._query_cmd() cmd = self.append_harness_extra_args(cmd) parser = StructuredOutputParser(config=self.config, log_obj=self.log_obj) env = {'MINIDUMP_SAVE_PATH': dirs['abs_blob_upload_dir']} env = self.query_env(partial_env=env, log_level=INFO) return_code = self.run_command(cmd, cwd=dirs['abs_work_dir'], output_timeout=1000, output_parser=parser, env=env) tbpl_status, log_level = parser.evaluate_parser(return_code) self.buildbot_status(tbpl_status, level=log_level)
def get_mozharness_status(suite, lines, status, formatter=None, buf=None): """Given list of log lines, determine what the mozharness status would be.""" parser = StructuredOutputParser( config={'log_level': INFO}, error_list=BaseErrorList+HarnessErrorList, strict=False, suite_category=suite, ) if formatter: parser.formatter = formatter # Processing the log with mozharness will re-print all the output to stdout # Since this exact same output has already been printed by the actual test # run, temporarily redirect stdout to devnull. buf = buf or open(os.devnull, 'w') orig = sys.stdout sys.stdout = buf for line in lines: parser.parse_single_line(json.dumps(line)) sys.stdout = orig return parser.evaluate_parser(status)
def run_tests(self): dirs = self.query_abs_dirs() self._install_fonts() parser = StructuredOutputParser(config=self.config, log_obj=self.log_obj, log_compact=True, error_list=BaseErrorList + HarnessErrorList) env = {'MINIDUMP_SAVE_PATH': dirs['abs_blob_upload_dir']} env['RUST_BACKTRACE'] = 'full' if self.config['allow_software_gl_layers']: env['MOZ_LAYERS_ALLOW_SOFTWARE_GL'] = '1' if self.config['enable_webrender']: env['MOZ_WEBRENDER'] = '1' env['MOZ_ACCELERATED'] = '1' if self.config['headless']: env['MOZ_HEADLESS'] = '1' env['MOZ_HEADLESS_WIDTH'] = self.config['headless_width'] env['MOZ_HEADLESS_HEIGHT'] = self.config['headless_height'] if self.config['disable_stylo']: if self.config['single_stylo_traversal']: self.fatal("--disable-stylo conflicts with --single-stylo-traversal") if self.config['enable_stylo']: self.fatal("--disable-stylo conflicts with --enable-stylo") if self.config['single_stylo_traversal']: env['STYLO_THREADS'] = '1' else: env['STYLO_THREADS'] = '4' if self.config['enable_stylo']: env['STYLO_FORCE_ENABLED'] = '1' if self.config['disable_stylo']: env['STYLO_FORCE_DISABLED'] = '1' env = self.query_env(partial_env=env, log_level=INFO) start_time = datetime.now() max_verify_time = timedelta(minutes=60) max_verify_tests = 10 verified_tests = 0 if self.config.get("verify") is True: verify_suites = self.query_verify_category_suites(None, None) else: test_types = self.config.get("test_type", []) verify_suites = [None] for verify_suite in verify_suites: if verify_suite: test_types = [verify_suite] for verify_args in self.query_verify_args(verify_suite): if (datetime.now() - start_time) > max_verify_time: # Verification has run out of time. That is okay! Stop running # tests so that a task timeout is not triggered, and so that # (partial) results are made available in a timely manner. self.info("TinderboxPrint: Verification too long: Not all tests " "were verified.<br/>") return if verified_tests >= max_verify_tests: # When changesets are merged between trees or many tests are # otherwise updated at once, there probably is not enough time # to verify all tests, and attempting to do so may cause other # problems, such as generating too much log output. self.info("TinderboxPrint: Too many modified tests: Not all tests " "were verified.<br/>") return verified_tests = verified_tests + 1 cmd = self._query_cmd(test_types) cmd.extend(verify_args) return_code = self.run_command(cmd, cwd=dirs['abs_work_dir'], output_timeout=1000, output_parser=parser, env=env) tbpl_status, log_level = parser.evaluate_parser(return_code) self.buildbot_status(tbpl_status, level=log_level) if len(verify_args) > 0: self.log_verify_status(verify_args[-1], tbpl_status, log_level)
def run_tests(self, args=None, **kw): ''' AWSY test should be implemented here ''' dirs = self.abs_dirs env = {} error_summary_file = os.path.join(dirs['abs_blob_upload_dir'], 'marionette_errorsummary.log') runtime_testvars = { 'webRootDir': self.webroot_dir, 'resultsDir': self.results_dir } # Check if this is a DMD build and if so enable it. dmd_enabled = False dmd_py_lib_dir = os.path.dirname(self.binary_path) if mozinfo.os == 'mac': # On mac binary is in MacOS and dmd.py is in Resources, ie: # Name.app/Contents/MacOS/libdmd.dylib # Name.app/Contents/Resources/dmd.py dmd_py_lib_dir = os.path.join(dmd_py_lib_dir, "../Resources/") dmd_path = os.path.join(dmd_py_lib_dir, "dmd.py") if self.config['dmd'] and os.path.isfile(dmd_path): dmd_enabled = True runtime_testvars['dmd'] = True # Allow the child process to import dmd.py python_path = os.environ.get('PYTHONPATH') if python_path: os.environ['PYTHONPATH'] = "%s%s%s" % (python_path, os.pathsep, dmd_py_lib_dir) else: os.environ['PYTHONPATH'] = dmd_py_lib_dir env['DMD'] = "--mode=dark-matter --stacks=full" runtime_testvars_path = os.path.join(self.awsy_path, 'runtime-testvars.json') runtime_testvars_file = open(runtime_testvars_path, 'wb') runtime_testvars_file.write(json.dumps(runtime_testvars, indent=2)) runtime_testvars_file.close() cmd = ['marionette'] if self.config['test_about_blank']: cmd.append( "--testvars=%s" % os.path.join(self.awsy_path, "conf", "base-testvars.json")) else: cmd.append("--testvars=%s" % os.path.join(self.awsy_path, "conf", "testvars.json")) cmd.append("--testvars=%s" % runtime_testvars_path) cmd.append("--log-raw=-") cmd.append("--log-errorsummary=%s" % error_summary_file) cmd.append("--binary=%s" % self.binary_path) cmd.append("--profile=%s" % (os.path.join(dirs['abs_work_dir'], 'profile'))) if not self.config['e10s']: cmd.append('--disable-e10s') cmd.append('--gecko-log=%s' % os.path.join(dirs["abs_blob_upload_dir"], 'gecko.log')) # TestingMixin._download_and_extract_symbols() should set # self.symbols_path cmd.append('--symbols-path=%s' % self.symbols_path) if self.config['test_about_blank']: test_file = os.path.join(self.awsy_libdir, 'test_base_memory_usage.py') prefs_file = "base-prefs.json" else: test_file = os.path.join(self.awsy_libdir, 'test_memory_usage.py') prefs_file = "prefs.json" cmd.append("--preferences=%s" % os.path.join(self.awsy_path, "conf", prefs_file)) if dmd_enabled: cmd.append("--pref=security.sandbox.content.level:0") cmd.append(test_file) if self.config['single_stylo_traversal']: env['STYLO_THREADS'] = '1' else: env['STYLO_THREADS'] = '4' # TODO: consider getting rid of this as stylo is enabled by default env['STYLO_FORCE_ENABLED'] = '1' if self.config['enable_webrender']: env['MOZ_WEBRENDER'] = '1' env['MOZ_ACCELERATED'] = '1' env['MOZ_UPLOAD_DIR'] = dirs['abs_blob_upload_dir'] if not os.path.isdir(env['MOZ_UPLOAD_DIR']): self.mkdir_p(env['MOZ_UPLOAD_DIR']) if self.query_minidump_stackwalk(): env['MINIDUMP_STACKWALK'] = self.minidump_stackwalk_path env['MINIDUMP_SAVE_PATH'] = dirs['abs_blob_upload_dir'] env['RUST_BACKTRACE'] = '1' env = self.query_env(partial_env=env) parser = StructuredOutputParser(config=self.config, log_obj=self.log_obj, error_list=self.error_list, strict=False) return_code = self.run_command( command=cmd, cwd=self.awsy_path, output_timeout=self.config.get("cmd_timeout"), env=env, output_parser=parser) level = INFO tbpl_status, log_level, summary = parser.evaluate_parser( return_code=return_code) self.log("AWSY exited with return code %s: %s" % (return_code, tbpl_status), level=level) self.record_status(tbpl_status)
def run_tests(self): dirs = self.query_abs_dirs() parser = StructuredOutputParser(config=self.config, log_obj=self.log_obj, log_compact=True, error_list=BaseErrorList + WptHarnessErrorList, allow_crashes=True) env = {'MINIDUMP_SAVE_PATH': dirs['abs_blob_upload_dir']} env['RUST_BACKTRACE'] = 'full' if self.config['allow_software_gl_layers']: env['MOZ_LAYERS_ALLOW_SOFTWARE_GL'] = '1' if self.config['headless']: env['MOZ_HEADLESS'] = '1' env['MOZ_HEADLESS_WIDTH'] = self.config['headless_width'] env['MOZ_HEADLESS_HEIGHT'] = self.config['headless_height'] env['STYLO_THREADS'] = '4' if self.is_android: env['ADB_PATH'] = self.adb_path env = self.query_env(partial_env=env, log_level=INFO) start_time = datetime.now() max_per_test_time = timedelta(minutes=60) max_per_test_tests = 10 if self.per_test_coverage: max_per_test_tests = 30 executed_tests = 0 executed_too_many_tests = False if self.per_test_coverage or self.verify_enabled: suites = self.query_per_test_category_suites(None, None) if "wdspec" in suites: # geckodriver is required for wdspec, but not always available geckodriver_path = self._query_geckodriver() if not geckodriver_path or not os.path.isfile( geckodriver_path): suites.remove("wdspec") self.info("Skipping 'wdspec' tests - no geckodriver") else: test_types = self.config.get("test_type", []) suites = [None] for suite in suites: if executed_too_many_tests and not self.per_test_coverage: continue if suite: test_types = [suite] summary = {} for per_test_args in self.query_args(suite): # Make sure baseline code coverage tests are never # skipped and that having them run has no influence # on the max number of actual tests that are to be run. is_baseline_test = 'baselinecoverage' in per_test_args[-1] \ if self.per_test_coverage else False if executed_too_many_tests and not is_baseline_test: continue if not is_baseline_test: if (datetime.now() - start_time) > max_per_test_time: # Running tests has run out of time. That is okay! Stop running # them so that a task timeout is not triggered, and so that # (partial) results are made available in a timely manner. self.info( "TinderboxPrint: Running tests took too long: Not all tests " "were executed.<br/>") return if executed_tests >= max_per_test_tests: # When changesets are merged between trees or many tests are # otherwise updated at once, there probably is not enough time # to run all tests, and attempting to do so may cause other # problems, such as generating too much log output. self.info( "TinderboxPrint: Too many modified tests: Not all tests " "were executed.<br/>") executed_too_many_tests = True executed_tests = executed_tests + 1 cmd = self._query_cmd(test_types) cmd.extend(per_test_args) final_env = copy.copy(env) if self.per_test_coverage: self.set_coverage_env(final_env, is_baseline_test) return_code = self.run_command(cmd, cwd=dirs['abs_work_dir'], output_timeout=1000, output_parser=parser, env=final_env) if self.per_test_coverage: self.add_per_test_coverage_report(final_env, suite, per_test_args[-1]) tbpl_status, log_level, summary = parser.evaluate_parser( return_code, previous_summary=summary) self.record_status(tbpl_status, level=log_level) if len(per_test_args) > 0: self.log_per_test_status(per_test_args[-1], tbpl_status, log_level) if tbpl_status == TBPL_RETRY: self.info("Per-test run abandoned due to RETRY status") return
def run_test(self, binary_path, env=None, marionette_port=2828): """All required steps for running the tests against an installer.""" dirs = self.query_abs_dirs() # Import the harness to retrieve the location of the cli scripts import telemetry_harness cmd = [ self.query_python_path(), os.path.join(os.path.dirname(telemetry_harness.__file__), self.cli_script), '--binary', binary_path, '--address', 'localhost:{}'.format(marionette_port), # Resource files to serve via local webserver '--server-root', os.path.join(dirs['abs_telemetry_dir'], 'harness', 'www'), # Use the work dir to get temporary data stored '--workspace', dirs['abs_work_dir'], # logging options '--gecko-log=-', # output from the gecko process redirected to stdout '--log-raw=-', # structured log for output parser redirected to stdout # additional reports helpful for Jenkins and inpection via Treeherder '--log-html', os.path.join(dirs['abs_blob_upload_dir'], 'report.html'), '--log-xunit', os.path.join(dirs['abs_blob_upload_dir'], 'report.xml'), # Enable tracing output to log transmission protocol '-vv', ] parser = StructuredOutputParser(config=self.config, log_obj=self.log_obj, strict=False) # Add the default tests to run tests = [ os.path.join(dirs['abs_telemetry_dir'], 'tests', test) for test in self.default_tests ] cmd.extend(tests) # Set further environment settings env = env or self.query_env() env.update({'MINIDUMP_SAVE_PATH': dirs['abs_blob_upload_dir']}) if self.query_minidump_stackwalk(): env.update({'MINIDUMP_STACKWALK': self.minidump_stackwalk_path}) env['RUST_BACKTRACE'] = '1' # If code coverage is enabled, set GCOV_PREFIX env variable if self.config.get('code_coverage'): env['GCOV_PREFIX'] = self.gcov_dir return_code = self.run_command(cmd, cwd=dirs['abs_work_dir'], output_timeout=300, output_parser=parser, env=env) tbpl_status, log_level, _ = parser.evaluate_parser(return_code) self.record_status(tbpl_status, level=log_level) return return_code
def run_tests(self, args=None, **kw): """ AWSY test should be implemented here """ dirs = self.abs_dirs env = {} error_summary_file = os.path.join(dirs["abs_blob_upload_dir"], "marionette_errorsummary.log") runtime_testvars = { "webRootDir": self.webroot_dir, "resultsDir": self.results_dir, "bin": self.binary_path, } # Check if this is a DMD build and if so enable it. dmd_enabled = False dmd_py_lib_dir = os.path.dirname(self.binary_path) if mozinfo.os == "mac": # On mac binary is in MacOS and dmd.py is in Resources, ie: # Name.app/Contents/MacOS/libdmd.dylib # Name.app/Contents/Resources/dmd.py dmd_py_lib_dir = os.path.join(dmd_py_lib_dir, "../Resources/") dmd_path = os.path.join(dmd_py_lib_dir, "dmd.py") if self.config["dmd"] and os.path.isfile(dmd_path): dmd_enabled = True runtime_testvars["dmd"] = True # Allow the child process to import dmd.py python_path = os.environ.get("PYTHONPATH") if python_path: os.environ["PYTHONPATH"] = "%s%s%s" % ( python_path, os.pathsep, dmd_py_lib_dir, ) else: os.environ["PYTHONPATH"] = dmd_py_lib_dir env["DMD"] = "--mode=dark-matter --stacks=full" runtime_testvars["tp6"] = self.config["tp6"] if self.config["tp6"]: # mitmproxy needs path to mozharness when installing the cert, and tooltool env["SCRIPTSPATH"] = scripts_path env["EXTERNALTOOLSPATH"] = external_tools_path runtime_testvars_path = os.path.join(self.awsy_path, "runtime-testvars.json") runtime_testvars_file = open(runtime_testvars_path, "wb") runtime_testvars_file.write(json.dumps(runtime_testvars, indent=2)) runtime_testvars_file.close() cmd = ["marionette"] test_vars_file = None if self.config["test_about_blank"]: test_vars_file = "base-testvars.json" else: if self.config["tp6"]: test_vars_file = "tp6-testvars.json" else: test_vars_file = "testvars.json" cmd.append("--testvars=%s" % os.path.join(self.awsy_path, "conf", test_vars_file)) cmd.append("--testvars=%s" % runtime_testvars_path) cmd.append("--log-raw=-") cmd.append("--log-errorsummary=%s" % error_summary_file) cmd.append("--binary=%s" % self.binary_path) cmd.append("--profile=%s" % (os.path.join(dirs["abs_work_dir"], "profile"))) if not self.config["e10s"]: cmd.append("--disable-e10s") cmd.extend( ["--setpref={}".format(p) for p in self.config["extra_prefs"]]) cmd.append("--gecko-log=%s" % os.path.join(dirs["abs_blob_upload_dir"], "gecko.log")) # TestingMixin._download_and_extract_symbols() should set # self.symbols_path cmd.append("--symbols-path=%s" % self.symbols_path) if self.config["test_about_blank"]: test_file = os.path.join(self.awsy_libdir, "test_base_memory_usage.py") prefs_file = "base-prefs.json" else: test_file = os.path.join(self.awsy_libdir, "test_memory_usage.py") if self.config["tp6"]: prefs_file = "tp6-prefs.json" else: prefs_file = "prefs.json" cmd.append("--preferences=%s" % os.path.join(self.awsy_path, "conf", prefs_file)) if dmd_enabled: cmd.append("--setpref=security.sandbox.content.level=0") cmd.append(test_file) if self.config["enable_webrender"]: cmd.append("--enable-webrender") env["STYLO_THREADS"] = "4" env["MOZ_UPLOAD_DIR"] = dirs["abs_blob_upload_dir"] if not os.path.isdir(env["MOZ_UPLOAD_DIR"]): self.mkdir_p(env["MOZ_UPLOAD_DIR"]) if self.query_minidump_stackwalk(): env["MINIDUMP_STACKWALK"] = self.minidump_stackwalk_path env["MINIDUMP_SAVE_PATH"] = dirs["abs_blob_upload_dir"] env["RUST_BACKTRACE"] = "1" env = self.query_env(partial_env=env) parser = StructuredOutputParser( config=self.config, log_obj=self.log_obj, error_list=self.error_list, strict=False, ) return_code = self.run_command( command=cmd, cwd=self.awsy_path, output_timeout=self.config.get("cmd_timeout"), env=env, output_parser=parser, ) level = INFO tbpl_status, log_level, summary = parser.evaluate_parser( return_code=return_code) self.log( "AWSY exited with return code %s: %s" % (return_code, tbpl_status), level=level, ) self.record_status(tbpl_status)
def run_tests(self, args=None, **kw): ''' AWSY test should be implemented here ''' dirs = self.abs_dirs env = {} error_summary_file = os.path.join(dirs['abs_blob_upload_dir'], 'marionette_errorsummary.log') runtime_testvars = {'webRootDir': self.webroot_dir, 'resultsDir': self.results_dir} runtime_testvars_path = os.path.join(self.awsy_path, 'runtime-testvars.json') runtime_testvars_file = open(runtime_testvars_path, 'wb') runtime_testvars_file.write(json.dumps(runtime_testvars, indent=2)) runtime_testvars_file.close() cmd = ['marionette'] cmd.append("--preferences=%s" % os.path.join(self.awsy_path, "conf", "prefs.json")) cmd.append("--testvars=%s" % os.path.join(self.awsy_path, "conf", "testvars.json")) cmd.append("--testvars=%s" % runtime_testvars_path) cmd.append("--log-raw=-") cmd.append("--log-errorsummary=%s" % error_summary_file) cmd.append("--binary=%s" % self.binary_path) cmd.append("--profile=%s" % (os.path.join(dirs['abs_work_dir'], 'profile'))) if not self.config['e10s']: cmd.append('--disable-e10s') cmd.append('--gecko-log=%s' % os.path.join(dirs["abs_blob_upload_dir"], 'gecko.log')) # TestingMixin._download_and_extract_symbols() should set # self.symbols_path cmd.append('--symbols-path=%s' % self.symbols_path) test_file = os.path.join(self.awsy_libdir, 'test_memory_usage.py') cmd.append(test_file) if self.config['disable_stylo']: if self.config['single_stylo_traversal']: self.fatal("--disable-stylo conflicts with --single-stylo-traversal") if self.config['enable_stylo']: self.fatal("--disable-stylo conflicts with --enable-stylo") if self.config['single_stylo_traversal']: env['STYLO_THREADS'] = '1' else: env['STYLO_THREADS'] = '4' if self.config['enable_stylo']: env['STYLO_FORCE_ENABLED'] = '1' if self.config['disable_stylo']: env['STYLO_FORCE_DISABLED'] = '1' env['MOZ_UPLOAD_DIR'] = dirs['abs_blob_upload_dir'] if not os.path.isdir(env['MOZ_UPLOAD_DIR']): self.mkdir_p(env['MOZ_UPLOAD_DIR']) if self.query_minidump_stackwalk(): env['MINIDUMP_STACKWALK'] = self.minidump_stackwalk_path env['MINIDUMP_SAVE_PATH'] = dirs['abs_blob_upload_dir'] env['RUST_BACKTRACE'] = '1' env = self.query_env(partial_env=env) parser = StructuredOutputParser(config=self.config, log_obj=self.log_obj, error_list=self.error_list, strict=False) return_code = self.run_command(command=cmd, cwd=self.awsy_path, output_timeout=self.config.get("cmd_timeout"), env=env, output_parser=parser) level = INFO tbpl_status, log_level = parser.evaluate_parser( return_code=return_code) self.log("AWSY exited with return code %s: %s" % (return_code, tbpl_status), level=level) self.buildbot_status(tbpl_status)
def run_test(self, binary_path, env=None, marionette_port=2828): """All required steps for running the tests against an installer.""" dirs = self.query_abs_dirs() # Import the harness to retrieve the location of the cli scripts import firefox_ui_harness cmd = [ self.query_python_path(), os.path.join(os.path.dirname(firefox_ui_harness.__file__), self.cli_script), "--binary", binary_path, "--address", "localhost:{}".format(marionette_port), # Resource files to serve via local webserver "--server-root", os.path.join(dirs["abs_fxui_dir"], "resources"), # Use the work dir to get temporary data stored "--workspace", dirs["abs_work_dir"], # logging options "--gecko-log=-", # output from the gecko process redirected to stdout "--log-raw=-", # structured log for output parser redirected to stdout # additional reports helpful for Jenkins and inpection via Treeherder "--log-html", os.path.join(dirs["abs_blob_upload_dir"], "report.html"), "--log-xunit", os.path.join(dirs["abs_blob_upload_dir"], "report.xml"), # Enable tracing output to log transmission protocol "-vv", ] # Collect all pass-through harness options to the script cmd.extend(self.query_harness_args()) # Translate deprecated --e10s flag if not self.config.get("e10s"): cmd.append("--disable-e10s") if self.symbols_url: cmd.extend(["--symbols-path", self.symbols_url]) if self.config.get("tag"): cmd.extend(["--tag", self.config["tag"]]) parser = StructuredOutputParser(config=self.config, log_obj=self.log_obj, strict=False) # Add the default tests to run tests = [os.path.join(dirs["abs_fxui_dir"], "tests", test) for test in self.default_tests] cmd.extend(tests) # Set further environment settings env = env or self.query_env() env.update({"MINIDUMP_SAVE_PATH": dirs["abs_blob_upload_dir"]}) if self.query_minidump_stackwalk(): env.update({"MINIDUMP_STACKWALK": self.minidump_stackwalk_path}) if self.config["allow_software_gl_layers"]: env["MOZ_LAYERS_ALLOW_SOFTWARE_GL"] = "1" return_code = self.run_command(cmd, cwd=dirs["abs_work_dir"], output_timeout=300, output_parser=parser, env=env) tbpl_status, log_level = parser.evaluate_parser(return_code) self.buildbot_status(tbpl_status, level=log_level) return return_code
def run_test(self, binary_path, env=None, marionette_port=2828): """All required steps for running the tests against an installer.""" dirs = self.query_abs_dirs() # Import the harness to retrieve the location of the cli scripts import telemetry_harness cmd = [ self.query_python_path(), os.path.join(os.path.dirname(telemetry_harness.__file__), self.cli_script), "--binary", binary_path, "--address", "localhost:{}".format(marionette_port), # Resource files to serve via local webserver "--server-root", os.path.join(dirs["abs_telemetry_dir"], "harness", "www"), # Use the work dir to get temporary data stored "--workspace", dirs["abs_work_dir"], # logging options "--gecko-log=-", # output from the gecko process redirected to stdout "--log-raw=-", # structured log for output parser redirected to stdout # additional reports helpful for Jenkins and inpection via Treeherder "--log-html", os.path.join(dirs["abs_blob_upload_dir"], "report.html"), "--log-xunit", os.path.join(dirs["abs_blob_upload_dir"], "report.xml"), # Enable tracing output to log transmission protocol "-vv", ] if self.config["enable_webrender"]: cmd.extend(["--enable-webrender"]) cmd.extend( ["--setpref={}".format(p) for p in self.config["extra_prefs"]]) if not self.config["e10s"]: cmd.append("--disable-e10s") parser = StructuredOutputParser(config=self.config, log_obj=self.log_obj, strict=False) # Add the default tests to run tests = [ os.path.join(dirs["abs_telemetry_dir"], "tests", test) for test in self.default_tests ] cmd.extend(tests) # Set further environment settings env = env or self.query_env() env.update({"MINIDUMP_SAVE_PATH": dirs["abs_blob_upload_dir"]}) if self.query_minidump_stackwalk(): env.update({"MINIDUMP_STACKWALK": self.minidump_stackwalk_path}) env["RUST_BACKTRACE"] = "1" env["MOZ_IGNORE_NSS_SHUTDOWN_LEAKS"] = "1" # If code coverage is enabled, set GCOV_PREFIX env variable if self.config.get("code_coverage"): env["GCOV_PREFIX"] = self.gcov_dir return_code = self.run_command( cmd, cwd=dirs["abs_work_dir"], output_timeout=300, output_parser=parser, env=env, ) tbpl_status, log_level, _ = parser.evaluate_parser(return_code) self.record_status(tbpl_status, level=log_level) return return_code
def run_test(self, binary_path, env=None, marionette_port=2828): """All required steps for running the tests against an installer.""" dirs = self.query_abs_dirs() # Import the harness to retrieve the location of the cli scripts import firefox_ui_harness cmd = [ self.query_python_path(), os.path.join(os.path.dirname(firefox_ui_harness.__file__), self.cli_script), '--binary', binary_path, '--address', 'localhost:{}'.format(marionette_port), # Resource files to serve via local webserver '--server-root', os.path.join(dirs['abs_fxui_dir'], 'resources'), # Use the work dir to get temporary data stored '--workspace', dirs['abs_work_dir'], # logging options '--gecko-log=-', # output from the gecko process redirected to stdout '--log-raw=-', # structured log for output parser redirected to stdout # additional reports helpful for Jenkins and inpection via Treeherder '--log-html', os.path.join(dirs['abs_blob_upload_dir'], 'report.html'), '--log-xunit', os.path.join(dirs['abs_blob_upload_dir'], 'report.xml'), # Enable tracing output to log transmission protocol '-vv', ] # Collect all pass-through harness options to the script cmd.extend(self.query_harness_args()) # Translate deprecated --e10s flag if not self.config.get('e10s'): cmd.append('--disable-e10s') if self.symbols_url: cmd.extend(['--symbols-path', self.symbols_url]) if self.config.get('tag'): cmd.extend(['--tag', self.config['tag']]) parser = StructuredOutputParser(config=self.config, log_obj=self.log_obj, strict=False) # Add the default tests to run tests = [ os.path.join(dirs['abs_fxui_dir'], 'tests', test) for test in self.default_tests ] cmd.extend(tests) # Set further environment settings env = env or self.query_env() env.update({'MINIDUMP_SAVE_PATH': dirs['abs_blob_upload_dir']}) if self.query_minidump_stackwalk(): env.update({'MINIDUMP_STACKWALK': self.minidump_stackwalk_path}) env['RUST_BACKTRACE'] = '1' if self.config['allow_software_gl_layers']: env['MOZ_LAYERS_ALLOW_SOFTWARE_GL'] = '1' return_code = self.run_command(cmd, cwd=dirs['abs_work_dir'], output_timeout=300, output_parser=parser, env=env) tbpl_status, log_level = parser.evaluate_parser(return_code) self.buildbot_status(tbpl_status, level=log_level) return return_code
def run_test(self, binary_path, env=None, marionette_port=2828): """All required steps for running the tests against an installer.""" dirs = self.query_abs_dirs() # Import the harness to retrieve the location of the cli scripts import firefox_ui_harness cmd = [ self.query_python_path(), os.path.join(os.path.dirname(firefox_ui_harness.__file__), self.cli_script), '--binary', binary_path, '--address', 'localhost:{}'.format(marionette_port), # Resource files to serve via local webserver '--server-root', os.path.join(dirs['abs_fxui_dir'], 'resources'), # Use the work dir to get temporary data stored '--workspace', dirs['abs_work_dir'], # logging options '--gecko-log=-', # output from the gecko process redirected to stdout '--log-raw=-', # structured log for output parser redirected to stdout # additional reports helpful for Jenkins and inpection via Treeherder '--log-html', os.path.join(dirs['abs_blob_upload_dir'], 'report.html'), '--log-xunit', os.path.join(dirs['abs_blob_upload_dir'], 'report.xml'), # Enable tracing output to log transmission protocol '-vv', ] # Collect all pass-through harness options to the script cmd.extend(self.query_harness_args()) # Translate deprecated --e10s flag if not self.config.get('e10s'): cmd.append('--disable-e10s') if self.symbols_url: cmd.extend(['--symbols-path', self.symbols_url]) if self.config.get('tag'): cmd.extend(['--tag', self.config['tag']]) parser = StructuredOutputParser(config=self.config, log_obj=self.log_obj, strict=False) # Add the default tests to run tests = [os.path.join(dirs['abs_fxui_dir'], 'tests', test) for test in self.default_tests] cmd.extend(tests) # Set further environment settings env = env or self.query_env() env.update({'MINIDUMP_SAVE_PATH': dirs['abs_blob_upload_dir']}) if self.query_minidump_stackwalk(): env.update({'MINIDUMP_STACKWALK': self.minidump_stackwalk_path}) env['RUST_BACKTRACE'] = '1' # If code coverage is enabled, set GCOV_PREFIX env variable if self.config.get('code_coverage'): env['GCOV_PREFIX'] = self.gcov_dir if self.config['allow_software_gl_layers']: env['MOZ_LAYERS_ALLOW_SOFTWARE_GL'] = '1' if self.config['enable_webrender']: env['MOZ_WEBRENDER'] = '1' return_code = self.run_command(cmd, cwd=dirs['abs_work_dir'], output_timeout=300, output_parser=parser, env=env) tbpl_status, log_level = parser.evaluate_parser(return_code) self.buildbot_status(tbpl_status, level=log_level) return return_code
def run_tests(self, args=None, **kw): ''' AWSY test should be implemented here ''' dirs = self.abs_dirs env = {} error_summary_file = os.path.join(dirs['abs_blob_upload_dir'], 'marionette_errorsummary.log') runtime_testvars = { 'webRootDir': self.webroot_dir, 'resultsDir': self.results_dir } runtime_testvars_path = os.path.join(self.awsy_path, 'runtime-testvars.json') runtime_testvars_file = open(runtime_testvars_path, 'wb') runtime_testvars_file.write(json.dumps(runtime_testvars, indent=2)) runtime_testvars_file.close() cmd = ['marionette'] if self.config['test_about_blank']: cmd.append( "--testvars=%s" % os.path.join(self.awsy_path, "conf", "base-testvars.json")) else: cmd.append("--testvars=%s" % os.path.join(self.awsy_path, "conf", "testvars.json")) cmd.append("--testvars=%s" % runtime_testvars_path) cmd.append("--log-raw=-") cmd.append("--log-errorsummary=%s" % error_summary_file) cmd.append("--binary=%s" % self.binary_path) cmd.append("--profile=%s" % (os.path.join(dirs['abs_work_dir'], 'profile'))) if not self.config['e10s']: cmd.append('--disable-e10s') cmd.append('--gecko-log=%s' % os.path.join(dirs["abs_blob_upload_dir"], 'gecko.log')) # TestingMixin._download_and_extract_symbols() should set # self.symbols_path cmd.append('--symbols-path=%s' % self.symbols_path) if self.config['test_about_blank']: test_file = os.path.join(self.awsy_libdir, 'test_base_memory_usage.py') prefs_file = "base-prefs.json" else: test_file = os.path.join(self.awsy_libdir, 'test_memory_usage.py') prefs_file = "prefs.json" cmd.append("--preferences=%s" % os.path.join(self.awsy_path, "conf", prefs_file)) cmd.append(test_file) if self.config['single_stylo_traversal']: env['STYLO_THREADS'] = '1' else: env['STYLO_THREADS'] = '4' # TODO: consider getting rid of this as stylo is enabled by default env['STYLO_FORCE_ENABLED'] = '1' if self.config['enable_webrender']: env['MOZ_WEBRENDER'] = '1' env['MOZ_ACCELERATED'] = '1' env['MOZ_UPLOAD_DIR'] = dirs['abs_blob_upload_dir'] if not os.path.isdir(env['MOZ_UPLOAD_DIR']): self.mkdir_p(env['MOZ_UPLOAD_DIR']) if self.query_minidump_stackwalk(): env['MINIDUMP_STACKWALK'] = self.minidump_stackwalk_path env['MINIDUMP_SAVE_PATH'] = dirs['abs_blob_upload_dir'] env['RUST_BACKTRACE'] = '1' env = self.query_env(partial_env=env) parser = StructuredOutputParser(config=self.config, log_obj=self.log_obj, error_list=self.error_list, strict=False) return_code = self.run_command( command=cmd, cwd=self.awsy_path, output_timeout=self.config.get("cmd_timeout"), env=env, output_parser=parser) level = INFO tbpl_status, log_level, summary = parser.evaluate_parser( return_code=return_code) self.log("AWSY exited with return code %s: %s" % (return_code, tbpl_status), level=level) self.record_status(tbpl_status)
def run_test(self, binary_path, env=None, marionette_port=2828): """All required steps for running the tests against an installer.""" dirs = self.query_abs_dirs() # Import the harness to retrieve the location of the cli scripts import firefox_ui_harness cmd = [ self.query_python_path(), os.path.join(os.path.dirname(firefox_ui_harness.__file__), self.cli_script), "--binary", binary_path, "--address", "localhost:{}".format(marionette_port), # Resource files to serve via local webserver "--server-root", os.path.join(dirs["abs_fxui_dir"], "resources"), # Use the work dir to get temporary data stored "--workspace", dirs["abs_work_dir"], # logging options "--gecko-log=-", # output from the gecko process redirected to stdout "--log-raw=-", # structured log for output parser redirected to stdout # additional reports helpful for Jenkins and inpection via Treeherder "--log-html", os.path.join(dirs["abs_blob_upload_dir"], "report.html"), "--log-xunit", os.path.join(dirs["abs_blob_upload_dir"], "report.xml"), # Enable tracing output to log transmission protocol "-vv", ] if self.config["enable_webrender"]: cmd.append("--enable-webrender") # Collect all pass-through harness options to the script cmd.extend(self.query_harness_args()) if not self.config.get("e10s"): cmd.append("--disable-e10s") cmd.extend( ["--setpref={}".format(p) for p in self.config.get("extra_prefs")]) if self.symbols_url: cmd.extend(["--symbols-path", self.symbols_url]) if self.config.get("tag"): cmd.extend(["--tag", self.config["tag"]]) parser = StructuredOutputParser(config=self.config, log_obj=self.log_obj, strict=False) # Add the default tests to run tests = [ os.path.join(dirs["abs_fxui_manifest_dir"], t) for t in self.default_tests ] cmd.extend(tests) # Set further environment settings env = env or self.query_env() env.update({"MINIDUMP_SAVE_PATH": dirs["abs_blob_upload_dir"]}) if self.query_minidump_stackwalk(): env.update({"MINIDUMP_STACKWALK": self.minidump_stackwalk_path}) env["RUST_BACKTRACE"] = "full" # If code coverage is enabled, set GCOV_PREFIX and JS_CODE_COVERAGE_OUTPUT_DIR # env variables if self.config.get("code_coverage"): env["GCOV_PREFIX"] = self.gcov_dir env["JS_CODE_COVERAGE_OUTPUT_DIR"] = self.jsvm_dir if self.config["allow_software_gl_layers"]: env["MOZ_LAYERS_ALLOW_SOFTWARE_GL"] = "1" return_code = self.run_command( cmd, cwd=dirs["abs_fxui_dir"], output_timeout=1000, output_parser=parser, env=env, ) tbpl_status, log_level, summary = parser.evaluate_parser(return_code) self.record_status(tbpl_status, level=log_level) return return_code
def run_tests(self): dirs = self.query_abs_dirs() self._install_fonts() parser = StructuredOutputParser(config=self.config, log_obj=self.log_obj, log_compact=True, error_list=BaseErrorList + HarnessErrorList) env = {'MINIDUMP_SAVE_PATH': dirs['abs_blob_upload_dir']} env['RUST_BACKTRACE'] = 'full' if self.config['allow_software_gl_layers']: env['MOZ_LAYERS_ALLOW_SOFTWARE_GL'] = '1' if self.config['enable_webrender']: env['MOZ_WEBRENDER'] = '1' env['MOZ_ACCELERATED'] = '1' if self.config['headless']: env['MOZ_HEADLESS'] = '1' env['MOZ_HEADLESS_WIDTH'] = self.config['headless_width'] env['MOZ_HEADLESS_HEIGHT'] = self.config['headless_height'] if self.config['single_stylo_traversal']: env['STYLO_THREADS'] = '1' else: env['STYLO_THREADS'] = '4' env = self.query_env(partial_env=env, log_level=INFO) start_time = datetime.now() max_per_test_time = timedelta(minutes=60) max_per_test_tests = 10 executed_tests = 0 if self.per_test_coverage or self.verify_enabled: suites = self.query_per_test_category_suites(None, None) if "wdspec" in suites: # geckodriver is required for wdspec, but not always available geckodriver_path = self._query_geckodriver() if not geckodriver_path or not os.path.isfile( geckodriver_path): suites.remove("wdspec") self.info("Skipping 'wdspec' tests - no geckodriver") else: test_types = self.config.get("test_type", []) suites = [None] for suite in suites: if suite: test_types = [suite] summary = None for per_test_args in self.query_args(suite): if (datetime.now() - start_time) > max_per_test_time: # Running tests has run out of time. That is okay! Stop running # them so that a task timeout is not triggered, and so that # (partial) results are made available in a timely manner. self.info( "TinderboxPrint: Running tests took too long: Not all tests " "were executed.<br/>") return if executed_tests >= max_per_test_tests: # When changesets are merged between trees or many tests are # otherwise updated at once, there probably is not enough time # to run all tests, and attempting to do so may cause other # problems, such as generating too much log output. self.info( "TinderboxPrint: Too many modified tests: Not all tests " "were executed.<br/>") return executed_tests = executed_tests + 1 cmd = self._query_cmd(test_types) cmd.extend(per_test_args) if self.per_test_coverage: gcov_dir, jsvm_dir = self.set_coverage_env(env) return_code = self.run_command(cmd, cwd=dirs['abs_work_dir'], output_timeout=1000, output_parser=parser, env=env) if self.per_test_coverage: self.add_per_test_coverage_report(gcov_dir, jsvm_dir, suite, per_test_args[-1]) tbpl_status, log_level, summary = parser.evaluate_parser( return_code, previous_summary=summary) self.buildbot_status(tbpl_status, level=log_level) if len(per_test_args) > 0: self.log_per_test_status(per_test_args[-1], tbpl_status, log_level)
def run_test(self, binary_path, env=None, marionette_port=2828): """All required steps for running the tests against an installer.""" dirs = self.query_abs_dirs() # Import the harness to retrieve the location of the cli scripts import firefox_ui_harness cmd = [ self.query_python_path(), os.path.join(os.path.dirname(firefox_ui_harness.__file__), self.cli_script), '--binary', binary_path, '--address', 'localhost:{}'.format(marionette_port), # Resource files to serve via local webserver '--server-root', os.path.join(dirs['abs_fxui_dir'], 'resources'), # Use the work dir to get temporary data stored '--workspace', dirs['abs_work_dir'], # logging options '--gecko-log=-', # output from the gecko process redirected to stdout '--log-raw=-', # structured log for output parser redirected to stdout # additional reports helpful for Jenkins and inpection via Treeherder '--log-html', os.path.join(dirs["abs_reports_dir"], self.reports['html']), '--log-xunit', os.path.join(dirs["abs_reports_dir"], self.reports['xunit']), ] # Collect all pass-through harness options to the script cmd.extend(self.query_harness_args()) # Set further environment settings env = env or self.query_env() if self.symbols_url: cmd.extend(['--symbols-path', self.symbols_url]) if self.query_minidump_stackwalk(): env['MINIDUMP_STACKWALK'] = self.minidump_stackwalk_path parser = StructuredOutputParser(config=self.config, log_obj=self.log_obj, strict=False) # Add the default tests to run tests = [ os.path.join(dirs['abs_fxui_dir'], 'tests', test) for test in self.default_tests ] cmd.extend(tests) return_code = self.run_command(cmd, cwd=dirs['abs_work_dir'], output_timeout=300, output_parser=parser, env=env) tbpl_status, log_level = parser.evaluate_parser(return_code) self.buildbot_status(tbpl_status, level=log_level) return return_code