def test_symlink_objdir(self): """Objdir that is a symlink is loaded properly.""" d = os.path.realpath(tempfile.mkdtemp()) try: topobjdir_real = os.path.join(d, 'objdir') topobjdir_link = os.path.join(d, 'objlink') os.mkdir(topobjdir_real) os.symlink(topobjdir_real, topobjdir_link) mozconfig = os.path.join(d, 'mozconfig') with open(mozconfig, 'wt') as fh: fh.write('mk_add_options MOZ_OBJDIR=%s' % topobjdir_link) mozinfo = os.path.join(topobjdir_real, 'mozinfo.json') with open(mozinfo, 'wt') as fh: json.dump(dict( topsrcdir=d, mozconfig=mozconfig, ), fh) os.chdir(topobjdir_link) obj = MozbuildObject.from_environment(detect_virtualenv_mozinfo=False) self.assertEqual(obj.topobjdir, topobjdir_real) os.chdir(topobjdir_real) obj = MozbuildObject.from_environment(detect_virtualenv_mozinfo=False) self.assertEqual(obj.topobjdir, topobjdir_real) finally: os.chdir(self._old_cwd) shutil.rmtree(d)
def test_objdir_mismatch(self): """Ensure MachCommandBase throwing on objdir mismatch.""" d = os.path.realpath(tempfile.mkdtemp()) try: real_topobjdir = os.path.join(d, 'real-objdir') os.makedirs(real_topobjdir) topobjdir = os.path.join(d, 'objdir') os.makedirs(topobjdir) topsrcdir = os.path.join(d, 'srcdir') os.makedirs(topsrcdir) mozconfig = os.path.join(d, 'mozconfig') with open(mozconfig, 'wt') as fh: fh.write('mk_add_options MOZ_OBJDIR=%s' % real_topobjdir) mozinfo = os.path.join(topobjdir, 'mozinfo.json') with open(mozinfo, 'wt') as fh: json.dump(dict( topsrcdir=topsrcdir, mozconfig=mozconfig, ), fh) os.chdir(topobjdir) with self.assertRaises(ObjdirMismatchException): MozbuildObject.from_environment(detect_virtualenv_mozinfo=False) finally: os.chdir(self._old_cwd) shutil.rmtree(d)
def __init__(self, *args, **kwargs): MozbuildObject.__init__(self, *args, **kwargs) # TODO Bug 794506 remove once mach integrates with virtualenv. build_path = os.path.join(self.topobjdir, "build") if build_path not in sys.path: sys.path.append(build_path) self.tests_dir = os.path.join(self.topobjdir, "_tests") self.mochitest_dir = os.path.join(self.tests_dir, "testing", "mochitest")
def __init__(self, *args, **kwargs): MozbuildObject.__init__(self, *args, **kwargs) # TODO Bug 794506 remove once mach integrates with virtualenv. build_path = os.path.join(self.topobjdir, 'build') if build_path not in sys.path: sys.path.append(build_path) self.tests_dir = os.path.join(self.topobjdir, '_tests') self.reftest_dir = os.path.join(self.tests_dir, 'reftest')
def _get_resolver(self): topobjdir = tempfile.mkdtemp() self._temp_dirs.append(topobjdir) with open(os.path.join(topobjdir, "all-tests.json"), "wt") as fh: fh.write(ALL_TESTS_JSON) o = MozbuildObject(self.FAKE_TOPSRCDIR, None, None, topobjdir=topobjdir) return o._spawn(TestResolver)
def _get_resolver(self): topobjdir = tempfile.mkdtemp() self._temp_dirs.append(topobjdir) with open(os.path.join(topobjdir, 'all-tests.json'), 'wt') as fh: fh.write(ALL_TESTS_JSON) o = MozbuildObject(self.FAKE_TOPSRCDIR, None, None, topobjdir=topobjdir) # Monkey patch the test resolver to avoid tests failing to find make # due to our fake topscrdir. TestResolver._run_make = lambda *a, **b: None return o._spawn(TestResolver)
def __init__(self, *args, **kwargs): MozbuildObject.__init__(self, *args, **kwargs) # TODO Bug 794506 remove once mach integrates with virtualenv. build_path = os.path.join(self.topobjdir, 'build') if build_path not in sys.path: sys.path.append(build_path) build_path = os.path.join(self.topsrcdir, 'build') if build_path not in sys.path: sys.path.append(build_path) self.tests_dir = os.path.join(self.topobjdir, '_tests') self.xpcshell_dir = os.path.join(self.tests_dir, 'xpcshell') self.bin_dir = os.path.join(self.distdir, 'bin')
def test_recursive_make(self): substs = list(BASE_SUBSTS) with self.do_test_backend(RecursiveMakeBackend, substs=substs) as config: build = MozbuildObject(config.topsrcdir, None, None, config.topobjdir) overrides = [ 'install_manifest_depends=', 'TEST_MOZBUILD=1', ] with self.line_handler() as handle_make_line: build._run_make(directory=config.topobjdir, target=overrides, silent=False, line_handler=handle_make_line) self.validate(config)
def _get_resolver(self): topobjdir = tempfile.mkdtemp() self._temp_dirs.append(topobjdir) with open(os.path.join(topobjdir, 'all-tests.pkl'), 'wb') as fh: pickle.dump(ALL_TESTS, fh) with open(os.path.join(topobjdir, 'test-defaults.pkl'), 'wb') as fh: pickle.dump(TEST_DEFAULTS, fh) o = MozbuildObject(self.FAKE_TOPSRCDIR, None, None, topobjdir=topobjdir) # Monkey patch the test resolver to avoid tests failing to find make # due to our fake topscrdir. TestResolver._run_make = lambda *a, **b: None return o._spawn(TestResolver)
def find_and_update_from_json(*dirs): """ Find a mozinfo.json file, load it, and update the info with the contents. :param dirs: Directories in which to look for the file. They will be searched after first looking in the root of the objdir if the current script is being run from a Mozilla objdir. Returns the full path to mozinfo.json if it was found, or None otherwise. """ # First, see if we're in an objdir try: from mozbuild.base import MozbuildObject, BuildEnvironmentNotFoundException from mozbuild.mozconfig import MozconfigFindException build = MozbuildObject.from_environment() json_path = _os.path.join(build.topobjdir, "mozinfo.json") if _os.path.isfile(json_path): update(json_path) return json_path except ImportError: pass except (BuildEnvironmentNotFoundException, MozconfigFindException): pass for d in dirs: d = _os.path.abspath(d) json_path = _os.path.join(d, "mozinfo.json") if _os.path.isfile(json_path): update(json_path) return json_path return None
def get_parser(): here = os.path.abspath(os.path.dirname(__file__)) build_obj = MozbuildObject.from_environment(cwd=here) if conditions.is_android(build_obj): return reftestcommandline.RemoteArgumentsParser() else: return reftestcommandline.DesktopArgumentsParser()
def setup_argument_parser(): build_obj = MozbuildObject.from_environment(cwd=here) build_path = os.path.join(build_obj.topobjdir, 'build') if build_path not in sys.path: sys.path.append(build_path) mochitest_dir = os.path.join(build_obj.topobjdir, '_tests', 'testing', 'mochitest') with warnings.catch_warnings(): warnings.simplefilter('ignore') import imp path = os.path.join(build_obj.topobjdir, mochitest_dir, 'runtests.py') if not os.path.exists(path): path = os.path.join(here, "runtests.py") with open(path, 'r') as fh: imp.load_module('mochitest', fh, path, ('.py', 'r', imp.PY_SOURCE)) from mochitest_options import MochitestArgumentParser if conditions.is_android(build_obj): # On Android, check for a connected device (and offer to start an # emulator if appropriate) before running tests. This check must # be done in this admittedly awkward place because # MochitestArgumentParser initialization fails if no device is found. from mozrunner.devices.android_device import verify_android_device verify_android_device(build_obj, install=True, xre=True) global parser parser = MochitestArgumentParser() return parser
def test_relative_objdir(self): """Relative defined objdirs are loaded properly.""" d = os.path.realpath(tempfile.mkdtemp()) try: mozconfig = os.path.join(d, 'mozconfig') with open(mozconfig, 'wt') as fh: fh.write('mk_add_options MOZ_OBJDIR=./objdir') topobjdir = mozpath.join(d, 'objdir') os.mkdir(topobjdir) mozinfo = os.path.join(topobjdir, 'mozinfo.json') with open(mozinfo, 'wt') as fh: json.dump(dict( topsrcdir=d, mozconfig=mozconfig, ), fh) os.environ[b'MOZCONFIG'] = mozconfig.encode('utf-8') child = os.path.join(topobjdir, 'foo', 'bar') os.makedirs(child) os.chdir(child) obj = MozbuildObject.from_environment( detect_virtualenv_mozinfo=False) self.assertEqual(obj.topobjdir, topobjdir) finally: os.chdir(self._old_cwd) shutil.rmtree(d)
def get_properties_db_from_xpcshell(self): """Generate the static css properties db for devtools from an xpcshell script.""" build = MozbuildObject.from_environment() # Get the paths script_path = resolve_path(self.topsrcdir, 'devtools/shared/css/generated/generate-properties-db.js') gre_path = resolve_path(self.topobjdir, 'dist/bin') browser_path = resolve_path(self.topobjdir, 'dist/bin/browser') xpcshell_path = build.get_binary_path(what='xpcshell') print(browser_path) sub_env = dict(os.environ) if sys.platform.startswith('linux'): sub_env["LD_LIBRARY_PATH"] = gre_path # Run the xcpshell script, and set the appdir flag to the browser path so that # we have the proper dependencies for requiring the loader. contents = subprocess.check_output([xpcshell_path, '-g', gre_path, '-a', browser_path, script_path], env = sub_env) # Extract just the output between the delimiters as the xpcshell output can # have extra output that we don't want. contents = contents.split('DEVTOOLS_CSS_DB_DELIMITER')[1] return json.loads(contents)
def main(platform): build = MozbuildObject.from_environment() topsrcdir = build.topsrcdir distdir = build.distdir srcdir = os.path.join(topsrcdir, "b2g", "simulator") app_buildid = open(os.path.join(build.topobjdir, "config", "buildid")).read().strip() # The simulator uses a shorter version string, # it only keeps the major version digits A.B # whereas MOZ_B2G_VERSION is A.B.C.D b2g_version = build.config_environment.defines["MOZ_B2G_VERSION"].replace('"', '') version = ".".join(str(n) for n in LooseVersion(b2g_version).version[0:2]) # Build a gaia profile specific to the simulator in order to: # - disable the FTU # - set custom prefs to enable devtools debugger server # - set custom settings to disable lockscreen and screen timeout # - only ship production apps gaia_path = build.config_environment.substs["GAIADIR"] builder = GaiaBuilder(build, gaia_path) builder.clean() env = { "NOFTU": "1", "GAIA_APP_TARGET": "production", "SETTINGS_PATH": os.path.join(srcdir, "custom-settings.json") } builder.profile(env) builder.override_prefs(os.path.join(srcdir, "custom-prefs.js")) # Substitute version strings in the package manifest overload file manifest_overload = os.path.join(build.topobjdir, "b2g", "simulator", "package-overload.json") process_package_overload(os.path.join(srcdir, "package-overload.json.in"), manifest_overload, version, app_buildid) # Build the simulator addon xpi xpi_name = XPI_NAME % {"version": version, "platform": platform} xpi_path = os.path.join(distdir, xpi_name) update_path = "%s/%s" % (version, platform) update_link = UPDATE_LINK % {"update_path": update_path, "xpi_name": xpi_name} update_url = UPDATE_URL % {"update_path": update_path} subprocess.check_call([ build.virtualenv_manager.python_path, os.path.join(topsrcdir, "addon-sdk", "source", "bin", "cfx"), "xpi", \ "--pkgdir", srcdir, \ "--manifest-overload", manifest_overload, \ "--strip-sdk", \ "--update-link", update_link, \ "--update-url", update_url, \ "--static-args", "{\"label\": \"Firefox OS %s\"}" % version, \ "--output-file", xpi_path \ ]) # Ship b2g-desktop, but prevent its gaia profile to be shipped in the xpi add_dir_to_zip(xpi_path, os.path.join(distdir, "b2g"), "b2g", ("gaia", "B2G.app/Contents/MacOS/gaia")) # Then ship our own gaia profile add_dir_to_zip(xpi_path, os.path.join(gaia_path, "profile"), "profile")
def mozregression_create_parser(): # Create the mozregression command line parser. # if mozregression is not installed, or not up to date, it will # first be installed. cmd = MozbuildObject.from_environment() cmd._activate_virtualenv() mozregression = mozregression_import() if not mozregression: # mozregression is not here at all, install it cmd.virtualenv_manager.install_pip_package('mozregression') print("mozregression was installed. please re-run your" " command. If you keep getting this message please " " manually run: 'pip install -U mozregression'.") else: # check if there is a new release available release = mozregression.new_release_on_pypi() if release: print(release) # there is one, so install it. Note that install_pip_package # does not work here, so just run pip directly. cmd.virtualenv_manager._run_pip([ 'install', 'mozregression==%s' % release ]) print("mozregression was updated to version %s. please" " re-run your command." % release) else: # mozregression is up to date, return the parser. return mozregression.parser() # exit if we updated or installed mozregression because # we may have already imported mozregression and running it # as this may cause issues. sys.exit(0)
def get_parser(): build_obj = MozbuildObject.from_environment(cwd=here) if conditions.is_android(build_obj): return parser_remote() elif conditions.is_b2g(build_obj): return parser_b2g() else: return parser_desktop()
def _init(self): CommonBackend._init(self) self._backend_files = {} self._cmd = MozbuildObject.from_environment() # This is a 'group' dependency - All rules that list this as an output # will be built before any rules that list this as an input. self._installed_files = '$(MOZ_OBJ_ROOT)/<installed-files>'
def test_objdir_is_srcdir_rejected(self): """Ensure the srcdir configurations are rejected.""" d = os.path.realpath(tempfile.mkdtemp()) try: # The easiest way to do this is to create a mozinfo.json with data # that will never happen. mozinfo = os.path.join(d, 'mozinfo.json') with open(mozinfo, 'wt') as fh: json.dump({'topsrcdir': d}, fh) os.chdir(d) with self.assertRaises(BadEnvironmentException): MozbuildObject.from_environment(detect_virtualenv_mozinfo=False) finally: shutil.rmtree(d)
def test_info(self, **params): import which from mozbuild.base import MozbuildObject self.branches = params['branches'] self.start = params['start'] self.end = params['end'] self.show_info = params['show_info'] self.show_results = params['show_results'] self.show_durations = params['show_durations'] self.show_bugs = params['show_bugs'] self.verbose = params['verbose'] if (not self.show_info and not self.show_results and not self.show_durations and not self.show_bugs): # by default, show everything self.show_info = True self.show_results = True self.show_durations = True self.show_bugs = True here = os.path.abspath(os.path.dirname(__file__)) build_obj = MozbuildObject.from_environment(cwd=here) self._hg = None if conditions.is_hg(build_obj): if self._is_windows(): self._hg = which.which('hg.exe') else: self._hg = which.which('hg') self._git = None if conditions.is_git(build_obj): if self._is_windows(): self._git = which.which('git.exe') else: self._git = which.which('git') for test_name in params['test_names']: print("===== %s =====" % test_name) self.test_name = test_name if len(self.test_name) < 6: print("'%s' is too short for a test name!" % self.test_name) continue if self.show_info: self.set_test_name() if self.show_results: self.report_test_results() if self.show_durations: self.report_test_durations() if self.show_bugs: self.report_bugs()
def make_dmg(source_directory, output_dmg): build = MozbuildObject.from_environment() extra_files = [ (os.path.join(build.distdir, 'branding', 'dsstore'), '.DS_Store'), (os.path.join(build.distdir, 'branding', 'background.png'), '.background/background.png'), (os.path.join(build.distdir, 'branding', 'disk.icns'), '.VolumeIcon.icns'), ] volume_name = build.substs['MOZ_APP_DISPLAYNAME'] dmg.create_dmg(source_directory, output_dmg, volume_name, extra_files)
def main(platform): build = MozbuildObject.from_environment() topsrcdir = build.topsrcdir distdir = build.distdir srcdir = os.path.join(topsrcdir, "b2g", "simulator") app_buildid = open(os.path.join(build.topobjdir, "config", "buildid")).read().strip() # The simulator uses a shorter version string, # it only keeps the major version digits A.B # whereas MOZ_B2G_VERSION is A.B.C.D b2g_version = build.config_environment.defines["MOZ_B2G_VERSION"].replace('"', "") version = ".".join(str(n) for n in LooseVersion(b2g_version).version[0:2]) # Build a gaia profile specific to the simulator in order to: # - disable the FTU # - set custom prefs to enable devtools debugger server # - set custom settings to disable lockscreen and screen timeout # - only ship production apps gaia_path = build.config_environment.substs["GAIADIR"] builder = GaiaBuilder(build, gaia_path) builder.clean() env = {"NOFTU": "1", "GAIA_APP_TARGET": "production", "SETTINGS_PATH": os.path.join(srcdir, "custom-settings.json")} builder.profile(env) builder.override_prefs(os.path.join(srcdir, "custom-prefs.js")) # Build the simulator addon xpi xpi_name = XPI_NAME % {"version": version, "platform": platform} xpi_path = os.path.join(distdir, xpi_name) update_path = "%s/%s" % (version, platform) update_url = UPDATE_URL % {"update_path": update_path} # Preprocess some files... manifest = os.path.join(build.topobjdir, "b2g", "simulator", "install.rdf") preprocess_file(os.path.join(srcdir, "install.rdf.in"), manifest, version, app_buildid, update_url) with JarWriter(xpi_path, optimize=False) as zip: # Ship addon files into the .xpi add_file_to_zip(zip, manifest, "install.rdf") add_file_to_zip(zip, os.path.join(srcdir, "bootstrap.js"), "bootstrap.js") add_file_to_zip(zip, os.path.join(srcdir, "icon.png"), "icon.png") add_file_to_zip(zip, os.path.join(srcdir, "icon64.png"), "icon64.png") # Ship b2g-desktop, but prevent its gaia profile to be shipped in the xpi add_dir_to_zip( zip, os.path.join(distdir, "b2g"), "b2g", ("gaia", "B2G.app/Contents/MacOS/gaia", "B2G.app/Contents/Resources/gaia"), ) # Then ship our own gaia profile add_dir_to_zip(zip, os.path.join(gaia_path, "profile"), "profile")
def test_mozconfig_opt_in(self): """Auto clobber iff AUTOCLOBBER is in the environment.""" topsrcdir = self.get_topsrcdir() topobjdir = self.get_tempdir() obj_clobber = os.path.join(topobjdir, 'CLOBBER') with open(obj_clobber, 'a'): pass dummy_file = os.path.join(topobjdir, 'dummy_file') with open(dummy_file, 'a'): pass self.assertTrue(os.path.exists(dummy_file)) old_time = os.path.getmtime(os.path.join(topsrcdir, 'CLOBBER')) - 60 os.utime(obj_clobber, (old_time, old_time)) # Check auto clobber is off by default env = dict(os.environ) if env.get('AUTOCLOBBER', False): del env['AUTOCLOBBER'] mbo = MozbuildObject(topsrcdir, None, None, topobjdir) build = mbo._spawn(BuildDriver) status = build._check_clobber(build.mozconfig, env) self.assertEqual(status, True) self.assertTrue(os.path.exists(dummy_file)) # Check auto clobber opt-in works env['AUTOCLOBBER'] = '1' status = build._check_clobber(build.mozconfig, env) self.assertFalse(status) self.assertFalse(os.path.exists(dummy_file))
def scheduled_by_push(self, repository, revision): changed_files = files_changed.get_changed_files(repository, revision) mbo = MozbuildObject.from_environment() # the decision task has a sparse checkout, so, mozbuild_reader will use # a MercurialRevisionFinder with revision '.', which should be the same # as `revision`; in other circumstances, it will use a default reader rdr = mbo.mozbuild_reader(config_mode='empty') components = set() for p, m in rdr.files_info(changed_files).items(): components |= set(m['SCHEDULES'].components) return components
def test_faster_recursive_make(self): substs = list(BASE_SUBSTS) + [ ('BUILD_BACKENDS', 'FasterMake+RecursiveMake'), ] with self.do_test_backend(get_backend_class( 'FasterMake+RecursiveMake'), substs=substs) as config: buildid = mozpath.join(config.topobjdir, 'config', 'buildid') ensureParentDir(buildid) with open(buildid, 'w') as fh: fh.write('20100101012345\n') build = MozbuildObject(config.topsrcdir, None, None, config.topobjdir) overrides = [ 'install_manifest_depends=', 'MOZ_JAR_MAKER_FILE_FORMAT=flat', 'TEST_MOZBUILD=1', ] with self.line_handler() as handle_make_line: build._run_make(directory=config.topobjdir, target=overrides, silent=False, line_handler=handle_make_line) self.validate(config)
def test_filesystem_traversal_reading(self): """Reading moz.build according to filesystem traversal works. We attempt to read every known moz.build file via filesystem traversal. If this test fails, it means that metadata extraction will fail. """ mb = MozbuildObject.from_environment(detect_virtualenv_mozinfo=False) config = mb.config_environment reader = BuildReader(config) all_paths = self._mozbuilds(reader) paths, contexts = reader.read_relevant_mozbuilds(all_paths) self.assertEqual(set(paths), all_paths) self.assertGreaterEqual(len(contexts), len(paths))
def __init__(self, *args, **kwargs): MozbuildObject.__init__(self, *args, **kwargs) # If installing tests is going to result in re-generating the build # backend, we need to do this here, so that the updated contents of # all-tests.pkl make it to the set of tests to run. self._run_make( target='backend.TestManifestBackend', pass_thru=True, print_directory=False, filename=mozpath.join(self.topsrcdir, 'build', 'rebuild-backend.mk'), append_env={ b'PYTHON': self.virtualenv_manager.python_path, b'BUILD_BACKEND_FILES': b'backend.TestManifestBackend', b'BACKEND_GENERATION_SCRIPT': mozpath.join( self.topsrcdir, 'build', 'gen_test_backend.py'), }, ) self._tests = TestMetadata(os.path.join(self.topobjdir, 'all-tests.pkl'), test_defaults=os.path.join(self.topobjdir, 'test-defaults.pkl')) self._test_rewrites = { 'a11y': os.path.join(self.topobjdir, '_tests', 'testing', 'mochitest', 'a11y'), 'browser-chrome': os.path.join(self.topobjdir, '_tests', 'testing', 'mochitest', 'browser'), 'chrome': os.path.join(self.topobjdir, '_tests', 'testing', 'mochitest', 'chrome'), 'mochitest': os.path.join(self.topobjdir, '_tests', 'testing', 'mochitest', 'tests'), 'web-platform-tests': os.path.join(self.topobjdir, '_tests', 'testing', 'web-platform'), 'xpcshell': os.path.join(self.topobjdir, '_tests', 'xpcshell'), } self._vcs = None self.verbose = False
def test_faster_make(self): substs = list(BASE_SUBSTS) + [ ('MOZ_BUILD_APP', 'dummy_app'), ('MOZ_WIDGET_TOOLKIT', 'dummy_widget'), ] with self.do_test_backend(RecursiveMakeBackend, FasterMakeBackend, substs=substs) as config: buildid = mozpath.join(config.topobjdir, 'config', 'buildid') ensureParentDir(buildid) with open(buildid, 'w') as fh: fh.write('20100101012345\n') build = MozbuildObject(config.topsrcdir, None, None, config.topobjdir) overrides = [ 'TEST_MOZBUILD=1', ] with self.line_handler() as handle_make_line: build._run_make(directory=mozpath.join(config.topobjdir, 'faster'), target=overrides, silent=False, line_handler=handle_make_line) self.validate(config)
def gen_test_backend(): build_obj = MozbuildObject.from_environment() try: config = build_obj.config_environment except BuildEnvironmentNotFoundException: print("No build detected, test metadata may be incomplete.") config = EmptyConfig(build_obj.topsrcdir) config.topobjdir = build_obj.topobjdir reader = BuildReader(config) emitter = TreeMetadataEmitter(config) backend = TestManifestBackend(config) context = reader.read_topsrcdir() data = emitter.emit(context, emitfn=emitter._process_test_manifests) backend.consume(data)
def main(argv): logging.basicConfig(format='%(levelname)s - %(threadName)s - %(message)s') parser = argparse.ArgumentParser( description='Upload generated source files in ARTIFACT to BUCKET in S3.') parser.add_argument('artifact', help='generated-sources artifact from build task') args = parser.parse_args(argv) region, bucket = get_s3_region_and_bucket() config = MozbuildObject.from_environment() config._activate_virtualenv() config.virtualenv_manager.install_pip_package('boto3==1.4.4') with timed() as elapsed: do_work(region=region, bucket=bucket, artifact=args.artifact) log.info('Finished in {:.03f}s'.format(elapsed())) return 0
import tempfile from mozdevice import DroidADB from mozprofile import DEFAULT_PORTS import mozinfo import mozlog import moznetwork here = os.path.abspath(os.path.dirname(__file__)) try: from mozbuild.base import ( MozbuildObject, MachCommandConditions as conditions, ) build_obj = MozbuildObject.from_environment(cwd=here) except ImportError: build_obj = None conditions = None # Maps test flavors to data needed to run them ALL_FLAVORS = { 'mochitest': { 'suite': 'plain', 'aliases': ('plain', 'mochitest'), 'enabled_apps': ('firefox', 'android'), 'extra_args': { 'flavor': 'plain', }, 'install_subdir': 'tests', },
def post_dispatch_handler(context, handler, instance, result, start_time, end_time, depth, args): """Perform global operations after command dispatch. For now, we will use this to handle build system telemetry. """ # Don't write telemetry data if this mach command was invoked as part of another # mach command. if depth != 1 or os.environ.get('MACH_MAIN_PID') != str(os.getpid()): return # We have not opted-in to telemetry if not context.settings.build.telemetry: return from mozbuild.telemetry import gather_telemetry from mozbuild.base import MozbuildObject import mozpack.path as mozpath if not isinstance(instance, MozbuildObject): instance = MozbuildObject.from_environment() try: substs = instance.substs except Exception: substs = {} # We gather telemetry for every operation. paths = { instance.topsrcdir: '$topsrcdir/', instance.topobjdir: '$topobjdir/', mozpath.normpath(os.path.expanduser('~')): '$HOME/', } # This might override one of the existing entries, that's OK. # We don't use a sigil here because we treat all arguments as potentially relative # paths, so we'd like to get them back as they were specified. paths[mozpath.normpath(os.getcwd())] = '' data = gather_telemetry(command=handler.name, success=(result == 0), start_time=start_time, end_time=end_time, mach_context=context, substs=substs, paths=paths) if data: telemetry_dir = os.path.join(get_state_dir()[0], 'telemetry') try: os.mkdir(telemetry_dir) except OSError as e: if e.errno != errno.EEXIST: raise outgoing_dir = os.path.join(telemetry_dir, 'outgoing') try: os.mkdir(outgoing_dir) except OSError as e: if e.errno != errno.EEXIST: raise with open(os.path.join(outgoing_dir, str(uuid.uuid4()) + '.json'), 'w') as f: json.dump(data, f, sort_keys=True) if should_skip_telemetry_submission(handler): return True state_dir, _ = get_state_dir() machpath = os.path.join(instance.topsrcdir, 'mach') with open(os.devnull, 'wb') as devnull: subprocess.Popen([sys.executable, machpath, 'python', '--no-virtualenv', os.path.join(topsrcdir, 'build', 'submit_telemetry_data.py'), state_dir], stdout=devnull, stderr=devnull)
def environment(self, verbose=False): import platform print('platform:\n\t%s' % platform.platform()) print('python version:\n\t%s' % sys.version) print('python prefix:\n\t%s' % sys.prefix) print('mach cwd:\n\t%s' % self.context.cwd) print('os cwd:\n\t%s' % os.getcwd()) print('mach directory:\n\t%s' % self.context.topdir) print('state directory:\n\t%s' % self.context.state_dir) mb = MozbuildObject(self.context.topdir, self.context.settings, self.context.log_manager) mozconfig = None try: mozconfig = mb.mozconfig print('mozconfig path:\n\t%s' % mozconfig['path']) except MozconfigFindException as e: print('Unable to find mozconfig: %s' % e.message) return 1 except MozconfigLoadException as e: print('Error loading mozconfig: %s' % e.path) print(e.message) if e.output: print('mozconfig evaluation output:') for line in e.output: print(line) return 1 print('object directory:\n\t%s' % mb.topobjdir) if mozconfig: print('mozconfig configure args:') if mozconfig['configure_args']: for arg in mozconfig['configure_args']: print('\t%s' % arg) print('mozconfig extra make args:') if mozconfig['make_extra']: for arg in mozconfig['make_extra']: print('\t%s' % arg) print('mozconfig make flags:') if mozconfig['make_flags']: for arg in mozconfig['make_flags']: print('\t%s' % arg) config = None try: config = mb.config_environment except Exception: pass if config: print('config topsrcdir:\n\t%s' % config.topsrcdir) print('config topobjdir:\n\t%s' % config.topobjdir) if verbose: print('config substitutions:') for k in sorted(config.substs): print('\t%s: %s' % (k, config.substs[k])) print('config defines:') for k in sorted(config.defines): print('\t%s' % k)
def _finalize_telemetry_legacy( context, instance, handler, success, start_time, end_time, topsrcdir ): """Record and submit legacy telemetry. Parameterized by the raw gathered telemetry, this function handles persisting and submission of the data. This has been designated as "legacy" telemetry because modern telemetry is being submitted with "Glean". """ from mozboot.util import get_state_dir from mozbuild.base import MozbuildObject from mozbuild.telemetry import gather_telemetry from mach.telemetry import is_telemetry_enabled, is_applicable_telemetry_environment if not ( is_applicable_telemetry_environment() and is_telemetry_enabled(context.settings) ): return if not isinstance(instance, MozbuildObject): instance = MozbuildObject.from_environment() command_attrs = getattr(context, "command_attrs", {}) # We gather telemetry for every operation. data = gather_telemetry( command=handler.name, success=success, start_time=start_time, end_time=end_time, mach_context=context, instance=instance, command_attrs=command_attrs, ) if data: telemetry_dir = os.path.join(get_state_dir(), "telemetry") try: os.mkdir(telemetry_dir) except OSError as e: if e.errno != errno.EEXIST: raise outgoing_dir = os.path.join(telemetry_dir, "outgoing") try: os.mkdir(outgoing_dir) except OSError as e: if e.errno != errno.EEXIST: raise with open(os.path.join(outgoing_dir, str(uuid.uuid4()) + ".json"), "w") as f: json.dump(data, f, sort_keys=True) # The user is performing a maintenance command, skip the upload if handler.name in ( "bootstrap", "doctor", "mach-commands", "vcs-setup", "create-mach-environment", "install-moz-phab", # We call mach environment in client.mk which would cause the # data submission to block the forward progress of make. "environment", ): return False if "TEST_MACH_TELEMETRY_NO_SUBMIT" in os.environ: # In our telemetry tests, we want telemetry to be collected for analysis, but # we don't want it submitted. return False state_dir = get_state_dir() machpath = os.path.join(instance.topsrcdir, "mach") with open(os.devnull, "wb") as devnull: subprocess.Popen( [ sys.executable, machpath, "python", "--no-virtualenv", os.path.join(topsrcdir, "build", "submit_telemetry_data.py"), state_dir, ], stdout=devnull, stderr=devnull, )
def get_base(self, topobjdir=None): return MozbuildObject(topsrcdir, None, log_manager, topobjdir=topobjdir)
from mozhttpd import MozHttpd from mozprofile import FirefoxProfile, Profile, Preferences from mozprofile.permissions import ServerLocations from mozrunner import FirefoxRunner, CLI PORT = 8888 PATH_MAPPINGS = { '/js-input/webkit/PerformanceTests': 'third_party/webkit/PerformanceTests', } if __name__ == '__main__': cli = CLI() debug_args, interactive = cli.debugger_arguments() build = MozbuildObject.from_environment() path_mappings = { k: os.path.join(build.topsrcdir, v) for k, v in PATH_MAPPINGS.items() } httpd = MozHttpd(port=PORT, docroot=os.path.join(build.topsrcdir, "build", "pgo"), path_mappings=path_mappings) httpd.start(block=False) locations = ServerLocations() locations.add_host(host='127.0.0.1', port=PORT, options='primary,privileged') with TemporaryDirectory() as profilePath:
def get_base(self): return MozbuildObject(topsrcdir, None, log_manager)
def __init__(self, **kwargs): super(ReftestArgumentsParser, self).__init__(**kwargs) # Try to import a MozbuildObject. Success indicates that we are # running from a source tree. This allows some defaults to be set # from the source tree. try: from mozbuild.base import MozbuildObject self.build_obj = MozbuildObject.from_environment(cwd=here) except ImportError: self.build_obj = None self.add_argument( "--xre-path", action="store", type=str, dest="xrePath", # individual scripts will set a sane default default=None, help= "absolute path to directory containing XRE (probably xulrunner)") self.add_argument( "--symbols-path", action="store", type=str, dest="symbolsPath", default=None, help="absolute path to directory containing breakpad symbols, " "or the URL of a zip file containing symbols") self.add_argument( "--debugger", action="store", dest="debugger", help="use the given debugger to launch the application") self.add_argument("--debugger-args", action="store", dest="debuggerArgs", help="pass the given args to the debugger _before_ " "the application on the command line") self.add_argument("--debugger-interactive", action="store_true", dest="debuggerInteractive", help="prevents the test harness from redirecting " "stdout and stderr for interactive debuggers") self.add_argument( "--appname", action="store", type=str, dest="app", default=None, help="absolute path to application, overriding default") self.add_argument("--extra-profile-file", action="append", dest="extraProfileFiles", default=[], help="copy specified files/dirs to testing profile") self.add_argument( "--timeout", action="store", dest="timeout", type=int, default=5 * 60, # 5 minutes per bug 479518 help="reftest will timeout in specified number of seconds. " "[default %(default)s].") self.add_argument("--leak-threshold", action="store", type=int, dest="defaultLeakThreshold", default=0, help="fail if the number of bytes leaked in default " "processes through refcounted objects (or bytes " "in classes with MOZ_COUNT_CTOR and MOZ_COUNT_DTOR) " "is greater than the given number") self.add_argument( "--utility-path", action="store", type=str, dest="utilityPath", default=self.build_obj.bindir if self.build_obj else None, help="absolute path to directory containing utility " "programs (xpcshell, ssltunnel, certutil)") self.add_argument("--total-chunks", type=int, dest="totalChunks", help="how many chunks to split the tests up into") self.add_argument( "--this-chunk", type=int, dest="thisChunk", help="which chunk to run between 1 and --total-chunks") self.add_argument("--log-file", action="store", type=str, dest="logFile", default=None, help="file to log output to in addition to stdout") self.add_argument("--skip-slow-tests", dest="skipSlowTests", action="store_true", default=False, help="skip tests marked as slow when running") self.add_argument( "--ignore-window-size", dest="ignoreWindowSize", action="store_true", default=False, help="ignore the window size, which may cause spurious " "failures and passes") self.add_argument( "--install-extension", action="append", dest="extensionsToInstall", default=[], help="install the specified extension in the testing profile. " "The extension file's name should be <id>.xpi where <id> is " "the extension's id as indicated in its install.rdf. " "An optional path can be specified too.") self.add_argument( "--marionette", default=None, help="host:port to use when connecting to Marionette") self.add_argument("--marionette-port-timeout", default=None, help=argparse.SUPPRESS) self.add_argument("--marionette-socket-timeout", default=None, help=argparse.SUPPRESS) self.add_argument("--setenv", action="append", type=str, default=[], dest="environment", metavar="NAME=VALUE", help="sets the given variable in the application's " "environment") self.add_argument( "--filter", action="store", type=str, dest="filter", help="specifies a regular expression (as could be passed to the JS " "RegExp constructor) to test against URLs in the reftest manifest; " "only test items that have a matching test URL will be run.") self.add_argument("--shuffle", action="store_true", default=False, dest="shuffle", help="run reftests in random order") self.add_argument( "--run-until-failure", action="store_true", default=False, dest="runUntilFailure", help="stop running on the first failure. Useful for RR recordings." ) self.add_argument( "--repeat", action="store", type=int, default=0, dest="repeat", help= "number of times the select test(s) will be executed. Useful for " "finding intermittent failures.") self.add_argument( "--focus-filter-mode", action="store", type=str, dest="focusFilterMode", default="all", help="filters tests to run by whether they require focus. " "Valid values are `all', `needs-focus', or `non-needs-focus'. " "Defaults to `all'.") self.add_argument("--disable-e10s", action="store_false", default=True, dest="e10s", help="disables content processes") self.add_argument("--setpref", action="append", type=str, default=[], dest="extraPrefs", metavar="PREF=VALUE", help="defines an extra user preference") self.add_argument("--reftest-extension-path", action="store", dest="reftestExtensionPath", help="Path to the reftest extension") self.add_argument("--special-powers-extension-path", action="store", dest="specialPowersExtensionPath", help="Path to the special powers extension") self.add_argument("--suite", choices=["reftest", "crashtest", "jstestbrowser"], default=None, help=argparse.SUPPRESS) self.add_argument( "--cleanup-crashes", action="store_true", dest="cleanupCrashes", default=False, help="Delete pending crash reports before running tests.") self.add_argument( "tests", metavar="TEST_PATH", nargs="*", help= "Path to test file, manifest file, or directory containing tests") mozlog.commandline.add_logging_group(self)
import logging import os import sys from collections import defaultdict from mozbuild.base import MozbuildObject from mozlint.pathutils import findobject from mozlint.parser import Parser from mozlint.result import ResultSummary from mozlog.structuredlog import StructuredLogger from mozpack import path import pytest here = path.abspath(path.dirname(__file__)) build = MozbuildObject.from_environment(cwd=here, virtualenv_name="python-test") lintdir = path.dirname(here) sys.path.insert(0, lintdir) logger = logging.getLogger("mozlint") def pytest_generate_tests(metafunc): """Finds, loads and returns the config for the linter name specified by the LINTER global variable in the calling module. This implies that each test file (that uses this fixture) should only be used to test a single linter. If no LINTER variable is defined, the test will fail. """ if "config" in metafunc.fixturenames:
def get_parser(): build_obj = MozbuildObject.from_environment(cwd=here) if conditions.is_android(build_obj): return parser_remote() else: return parser_desktop()
def main(): config = MozbuildObject.from_environment() config.activate_virtualenv() logging.basicConfig() parser = argparse.ArgumentParser( description="Upload symbols in ZIP using token from Taskcluster secrets service." ) parser.add_argument( "archive", help="Symbols archive file - URL or path to local file" ) parser.add_argument( "--ignore-missing", help="No error on missing files", action="store_true" ) args = parser.parse_args() def check_file_exists(url): for i, _ in enumerate(redo.retrier(attempts=MAX_RETRIES), start=1): try: resp = requests.head(url, allow_redirects=True) return resp.status_code == requests.codes.ok except requests.exceptions.RequestException as e: log.error("Error: {0}".format(e)) log.info("Retrying...") return False zip_path = args.archive if args.archive.endswith(".tar.zst"): from mozpack.files import File from mozpack.mozjar import JarWriter import gzip import tarfile import tempfile config._ensure_zstd() import zstandard def prepare_zip_from(archive, tmpdir): if archive.startswith("http"): resp = requests.get(archive, allow_redirects=True, stream=True) resp.raise_for_status() reader = resp.raw # Work around taskcluster generic-worker possibly gzipping the tar.zst. if resp.headers.get("Content-Encoding") == "gzip": reader = gzip.GzipFile(fileobj=reader) else: reader = open(archive, "rb") ctx = zstandard.ZstdDecompressor() uncompressed = ctx.stream_reader(reader) with tarfile.open( mode="r|", fileobj=uncompressed, bufsize=1024 * 1024 ) as tar: while True: info = tar.next() if info is None: break log.info(info.name) data = tar.extractfile(info) path = os.path.join(tmpdir, info.name.lstrip("/")) if info.name.endswith(".dbg"): os.makedirs(os.path.dirname(path), exist_ok=True) with open(path, "wb") as fh: with gzip.GzipFile( fileobj=fh, mode="wb", compresslevel=5 ) as c: shutil.copyfileobj(data, c) jar.add(info.name + ".gz", File(path), compress=False) elif info.name.endswith(".dSYM.tar"): import bz2 os.makedirs(os.path.dirname(path), exist_ok=True) with open(path, "wb") as fh: c = bz2.BZ2Compressor() while True: buf = data.read(16384) if not buf: break fh.write(c.compress(buf)) fh.write(c.flush()) jar.add(info.name + ".bz2", File(path), compress=False) elif info.name.endswith((".pdb", ".exe", ".dll")): import subprocess makecab = os.environ.get("MAKECAB", "makecab") os.makedirs(os.path.dirname(path), exist_ok=True) with open(path, "wb") as fh: shutil.copyfileobj(data, fh) subprocess.check_call( [makecab, "-D", "CompressionType=MSZIP", path, path + "_"], stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT, ) jar.add(info.name[:-1] + "_", File(path + "_"), compress=False) else: jar.add(info.name, data) reader.close() tmpdir = tempfile.TemporaryDirectory() zip_path = os.path.join(tmpdir.name, "symbols.zip") log.info( 'Preparing symbol archive "{0}" from "{1}"'.format(zip_path, args.archive) ) is_existing = False try: for i, _ in enumerate(redo.retrier(attempts=MAX_RETRIES), start=1): with JarWriter(zip_path, compress_level=5) as jar: try: prepare_zip_from(args.archive, tmpdir.name) is_existing = True break except requests.exceptions.RequestException as e: log.error("Error: {0}".format(e)) log.info("Retrying...") except Exception: os.remove(zip_path) raise elif args.archive.startswith("http"): is_existing = check_file_exists(args.archive) else: is_existing = os.path.isfile(args.archive) if not is_existing: if args.ignore_missing: log.info('Archive file "{0}" does not exist!'.format(args.archive)) return 0 else: log.error('Error: archive file "{0}" does not exist!'.format(args.archive)) return 1 secret_name = os.environ.get("SYMBOL_SECRET") if secret_name is not None: auth_token = get_taskcluster_secret(secret_name) elif "SOCORRO_SYMBOL_UPLOAD_TOKEN_FILE" in os.environ: token_file = os.environ["SOCORRO_SYMBOL_UPLOAD_TOKEN_FILE"] if not os.path.isfile(token_file): log.error( 'SOCORRO_SYMBOL_UPLOAD_TOKEN_FILE "{0}" does not exist!'.format( token_file ) ) return 1 auth_token = open(token_file, "r").read().strip() else: log.error( "You must set the SYMBOL_SECRET or SOCORRO_SYMBOL_UPLOAD_TOKEN_FILE " "environment variables!" ) return 1 # Allow overwriting of the upload url with an environmental variable if "SOCORRO_SYMBOL_UPLOAD_URL" in os.environ: url = os.environ["SOCORRO_SYMBOL_UPLOAD_URL"] else: url = DEFAULT_URL log.info('Uploading symbol file "{0}" to "{1}"'.format(zip_path, url)) for i, _ in enumerate(redo.retrier(attempts=MAX_RETRIES), start=1): log.info("Attempt %d of %d..." % (i, MAX_RETRIES)) try: if zip_path.startswith("http"): zip_arg = {"data": {"url": zip_path}} else: zip_arg = {"files": {"symbols.zip": open(zip_path, "rb")}} r = requests.post( url, headers={"Auth-Token": auth_token}, allow_redirects=False, # Allow a longer read timeout because uploading by URL means the server # has to fetch the entire zip file, which can take a while. The load balancer # in front of symbols.mozilla.org has a 300 second timeout, so we'll use that. timeout=(300, 300), **zip_arg ) # 429 or any 5XX is likely to be a transient failure. # Break out for success or other error codes. if r.ok or (r.status_code < 500 and r.status_code != 429): break print_error(r) except requests.exceptions.RequestException as e: log.error("Error: {0}".format(e)) log.info("Retrying...") else: log.warn("Maximum retries hit, giving up!") return 1 if r.status_code >= 200 and r.status_code < 300: log.info("Uploaded successfully!") return 0 print_error(r) return 1
def __init__(self, *args, **kwargs): MozbuildObject.__init__(self, *args, **kwargs) self.test_packages_url = self._test_packages_url() self.installer_url = self._installer_url() desktop_unittest_config = [ "--config-file", lambda: self.config_path( "unittests", "%s_unittest.py" % mozinfo.info["os"] ), "--config-file", lambda: self.config_path("developer_config.py"), ] self.config = { "__defaults__": { "config": [ "--download-symbols", "ondemand", "--installer-url", self.installer_url, "--test-packages-url", self.test_packages_url, ] }, "mochitest-valgrind": { "script": "desktop_unittest.py", "config": desktop_unittest_config + ["--mochitest-suite", "valgrind-plain"], }, "mochitest": { "script": "desktop_unittest.py", "config": desktop_unittest_config + ["--mochitest-suite", "plain"], }, "mochitest-chrome": { "script": "desktop_unittest.py", "config": desktop_unittest_config + ["--mochitest-suite", "chrome"], }, "mochitest-browser-chrome": { "script": "desktop_unittest.py", "config": desktop_unittest_config + ["--mochitest-suite", "browser-chrome"], }, "mochitest-devtools-chrome": { "script": "desktop_unittest.py", "config": desktop_unittest_config + ["--mochitest-suite", "mochitest-devtools-chrome"], }, "mochitest-remote": { "script": "desktop_unittest.py", "config": desktop_unittest_config + ["--mochitest-suite", "mochitest-remote"], }, "crashtest": { "script": "desktop_unittest.py", "config": desktop_unittest_config + ["--reftest-suite", "crashtest"], }, "jsreftest": { "script": "desktop_unittest.py", "config": desktop_unittest_config + ["--reftest-suite", "jsreftest"], }, "reftest": { "script": "desktop_unittest.py", "config": desktop_unittest_config + ["--reftest-suite", "reftest"], }, "reftest-no-accel": { "script": "desktop_unittest.py", "config": desktop_unittest_config + ["--reftest-suite", "reftest-no-accel"], }, "cppunittest": { "script": "desktop_unittest.py", "config": desktop_unittest_config + ["--cppunittest-suite", "cppunittest"], }, "xpcshell": { "script": "desktop_unittest.py", "config": desktop_unittest_config + ["--xpcshell-suite", "xpcshell"], }, "xpcshell-addons": { "script": "desktop_unittest.py", "config": desktop_unittest_config + ["--xpcshell-suite", "xpcshell-addons"], }, "jittest": { "script": "desktop_unittest.py", "config": desktop_unittest_config + ["--jittest-suite", "jittest"], }, "marionette": { "script": "marionette.py", "config": [ "--config-file", self.config_path("marionette", "test_config.py"), ], }, "web-platform-tests": { "script": "web_platform_tests.py", "config": [ "--config-file", self.config_path("web_platform_tests", self.wpt_config), ], }, }