def repackage_mar( topsrcdir, package, mar, output, mar_format="lzma", arch=None, mar_channel_id=None ): if not zipfile.is_zipfile(package) and not tarfile.is_tarfile(package): raise Exception("Package file %s is not a valid .zip or .tar file." % package) if arch and arch not in _BCJ_OPTIONS: raise Exception("Unknown architecture {}, available architectures: {}".format( arch, list(_BCJ_OPTIONS.keys()))) ensureParentDir(output) tmpdir = tempfile.mkdtemp() try: if tarfile.is_tarfile(package): z = tarfile.open(package) z.extractall(tmpdir) filelist = z.getnames() z.close() else: z = zipfile.ZipFile(package) z.extractall(tmpdir) filelist = z.namelist() z.close() toplevel_dirs = set([mozpath.split(f)[0] for f in filelist]) excluded_stuff = set([' ', '.background', '.DS_Store', '.VolumeIcon.icns']) toplevel_dirs = toplevel_dirs - excluded_stuff # Make sure the .zip file just contains a directory like 'firefox/' at # the top, and find out what it is called. if len(toplevel_dirs) != 1: raise Exception("Package file is expected to have a single top-level directory" "(eg: 'firefox'), not: %s" % toplevel_dirs) ffxdir = mozpath.join(tmpdir, toplevel_dirs.pop()) make_full_update = mozpath.join(topsrcdir, 'tools/update-packaging/make_full_update.sh') env = os.environ.copy() env['MOZ_PRODUCT_VERSION'] = get_application_ini_value(tmpdir, 'App', 'Version') env['MAR'] = mozpath.normpath(mar) if arch: env['BCJ_OPTIONS'] = ' '.join(_BCJ_OPTIONS[arch]) if mar_format == 'bz2': env['MAR_OLD_FORMAT'] = '1' if mar_channel_id: env['MAR_CHANNEL_ID'] = mar_channel_id # The Windows build systems have xz installed but it isn't in the path # like it is on Linux and Mac OS X so just use the XZ env var so the mar # generation scripts can find it. xz_path = mozpath.join(topsrcdir, 'xz/xz.exe') if os.path.exists(xz_path): env['XZ'] = mozpath.normpath(xz_path) cmd = [make_full_update, output, ffxdir] if sys.platform == 'win32': # make_full_update.sh is a bash script, and Windows needs to # explicitly call out the shell to execute the script from Python. cmd.insert(0, env['MOZILLABUILD'] + '/msys/bin/bash.exe') subprocess.check_call(cmd, env=ensure_subprocess_env(env)) finally: shutil.rmtree(tmpdir)
def __init__( self, config, gyp_dir_attrs, path, output, executor, action_overrides, non_unified_sources, ): self._path = path self._config = config self._output = output self._non_unified_sources = non_unified_sources self._gyp_dir_attrs = gyp_dir_attrs self._action_overrides = action_overrides self.execution_time = 0.0 self._results = [] # gyp expects plain str instead of unicode. The frontend code gives us # unicode strings, so convert them. if config.substs["CC_TYPE"] == "clang-cl": # This isn't actually used anywhere in this generator, but it's needed # to override the registry detection of VC++ in gyp. os.environ.update( ensure_subprocess_env({ "GYP_MSVS_OVERRIDE_PATH": "fake_path", "GYP_MSVS_VERSION": config.substs["MSVS_VERSION"], })) params = { "parallel": False, "generator_flags": {}, "build_files": [path], "root_targets": None, } if gyp_dir_attrs.no_chromium: includes = [] depth = mozpath.dirname(path) else: depth = chrome_src # Files that gyp_chromium always includes includes = [ mozpath.join(script_dir, "gyp_includes", "common.gypi") ] finder = FileFinder(chrome_src) includes.extend( mozpath.join(chrome_src, name) for name, _ in finder.find("*/supplement.gypi")) str_vars = dict(gyp_dir_attrs.variables) str_vars["python"] = sys.executable self._gyp_loader_future = executor.submit(load_gyp, [path], "mozbuild", str_vars, includes, depth, params)
def wrapper(*args, **kwargs): if 'env' not in kwargs: kwargs['env'] = dict(self._environ) # Subprocess on older Pythons can't handle unicode keys or # values in environment dicts while subprocess on newer Pythons # needs text in the env. Normalize automagically so callers # don't have to deal with this. kwargs['env'] = ensure_subprocess_env(kwargs['env'], encoding=system_encoding) return function(*args, **kwargs)
def push_to_try(self, message): try: subprocess.check_call((self._tool, 'push-to-try', '-m', message), cwd=self.path, env=ensure_subprocess_env(self._env)) except subprocess.CalledProcessError: try: self._run('showconfig', 'extensions.push-to-try') except subprocess.CalledProcessError: raise MissingVCSExtension('push-to-try') raise finally: self._run('revert', '-a')
def __init__(self, config, gyp_dir_attrs, path, output, executor, action_overrides, non_unified_sources): self._path = path self._config = config self._output = output self._non_unified_sources = non_unified_sources self._gyp_dir_attrs = gyp_dir_attrs self._action_overrides = action_overrides self.execution_time = 0.0 self._results = [] # gyp expects plain str instead of unicode. The frontend code gives us # unicode strings, so convert them. path = encode(path) if config.substs['CC_TYPE'] == 'clang-cl': # This isn't actually used anywhere in this generator, but it's needed # to override the registry detection of VC++ in gyp. os.environ.update( ensure_subprocess_env({ 'GYP_MSVS_OVERRIDE_PATH': 'fake_path', 'GYP_MSVS_VERSION': config.substs['MSVS_VERSION'], })) params = { b'parallel': False, b'generator_flags': {}, b'build_files': [path], b'root_targets': None, } if gyp_dir_attrs.no_chromium: includes = [] depth = mozpath.dirname(path) else: depth = chrome_src # Files that gyp_chromium always includes includes = [ encode(mozpath.join(script_dir, 'gyp_includes', 'common.gypi')) ] finder = FileFinder(chrome_src) includes.extend( encode(mozpath.join(chrome_src, name)) for name, _ in finder.find('*/supplement.gypi')) str_vars = dict((name, encode(value)) for name, value in gyp_dir_attrs.variables.items()) self._gyp_loader_future = executor.submit(load_gyp, [path], b'mozbuild', str_vars, includes, encode(depth), params)
def _run(self, *args, **runargs): return_codes = runargs.get('return_codes', []) cmd = (self._tool,) + args try: return subprocess.check_output(cmd, cwd=self.path, env=ensure_subprocess_env(self._env), universal_newlines=True) except subprocess.CalledProcessError as e: if e.returncode in return_codes: return '' raise
def push_to_try(self, message): try: subprocess.check_call( (self._tool, "push-to-try", "-m", message), cwd=self.path, env=ensure_subprocess_env(self._env), ) except subprocess.CalledProcessError: try: self._run("showconfig", "extensions.push-to-try") except subprocess.CalledProcessError: raise MissingVCSExtension("push-to-try") raise finally: self._run("revert", "-a")
def call_process(name, cmd, cwd=None, append_env={}): env = dict(os.environ) env.update(ensure_subprocess_env(append_env)) try: with open(os.devnull, "w") as fnull: subprocess.check_call(cmd, cwd=cwd, stdout=fnull, env=env) except subprocess.CalledProcessError: if cwd: print("\nError installing %s in the %s folder, aborting." % (name, cwd)) else: print("\nError installing %s, aborting." % name) return False return True
def run_fzf(cmd, tasks): env = dict(os.environ) env.update({ 'PYTHONPATH': os.pathsep.join([p for p in sys.path if 'requests' in p]) }) proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stdin=subprocess.PIPE, env=ensure_subprocess_env(env)) out = proc.communicate('\n'.join(tasks))[0].splitlines() selected = [] query = None if out: query = out[0] selected = out[1:] return query, selected
def __init__(self, path, hg='hg'): import hglib.client super(HgRepository, self).__init__(path, tool=hg) self._env[b'HGPLAIN'] = b'1' # Setting this modifies a global variable and makes all future hglib # instances use this binary. Since the tool path was validated, this # should be OK. But ideally hglib would offer an API that defines # per-instance binaries. hglib.HGPATH = self._tool # Without connect=False this spawns a persistent process. We want # the process lifetime tied to a context manager. self._client = hglib.client.hgclient(self.path, encoding=b'UTF-8', configs=None, connect=False) # Work around py3 compat issues in python-hglib self._client._env = ensure_subprocess_env(self._client._env)
def run_fzf(cmd, tasks): env = dict(os.environ) env.update({ "PYTHONPATH": os.pathsep.join([p for p in sys.path if "requests" in p]) }) proc = subprocess.Popen( cmd, stdout=subprocess.PIPE, stdin=subprocess.PIPE, env=ensure_subprocess_env(env), universal_newlines=True, ) out = proc.communicate("\n".join(tasks))[0].splitlines() selected = [] query = None if out: query = out[0] selected = out[1:] return query, selected
def run_process(self, args=None, cwd=None, append_env=None, explicit_env=None, log_name=None, log_level=logging.INFO, line_handler=None, require_unix_environment=False, ensure_exit_code=0, ignore_children=False, pass_thru=False, python_unbuffered=True): """Runs a single process to completion. Takes a list of arguments to run where the first item is the executable. Runs the command in the specified directory and with optional environment variables. append_env -- Dict of environment variables to append to the current set of environment variables. explicit_env -- Dict of environment variables to set for the new process. Any existing environment variables will be ignored. require_unix_environment if True will ensure the command is executed within a UNIX environment. Basically, if we are on Windows, it will execute the command via an appropriate UNIX-like shell. ignore_children is proxied to mozprocess's ignore_children. ensure_exit_code is used to ensure the exit code of a process matches what is expected. If it is an integer, we raise an Exception if the exit code does not match this value. If it is True, we ensure the exit code is 0. If it is False, we don't perform any exit code validation. pass_thru is a special execution mode where the child process inherits this process's standard file handles (stdin, stdout, stderr) as well as additional file descriptors. It should be used for interactive processes where buffering from mozprocess could be an issue. pass_thru does not use mozprocess. Therefore, arguments like log_name, line_handler, and ignore_children have no effect. When python_unbuffered is set, the PYTHONUNBUFFERED environment variable will be set in the child process. This is normally advantageous (see bug 1627873) but is detrimental in certain circumstances (specifically, we have seen issues when using pass_thru mode to open a Python subshell, as in bug 1628838). This variable should be set to False to avoid bustage in those circumstances. """ args = self._normalize_command(args, require_unix_environment) self.log(logging.INFO, 'new_process', {'args': ' '.join(args)}, '{args}') def handleLine(line): # Converts str to unicode on Python 2 and bytes to str on Python 3. if isinstance(line, bytes): line = line.decode(sys.stdout.encoding or 'utf-8', 'replace') if line_handler: line_handler(line) if not log_name: return self.log(log_level, log_name, {'line': line.rstrip()}, '{line}') use_env = {} if explicit_env: use_env = explicit_env else: use_env.update(os.environ) if append_env: use_env.update(append_env) if python_unbuffered: use_env['PYTHONUNBUFFERED'] = '1' self.log(logging.DEBUG, 'process', {'env': use_env}, 'Environment: {env}') use_env = ensure_subprocess_env(use_env) if pass_thru: proc = subprocess.Popen(args, cwd=cwd, env=use_env) status = None # Leave it to the subprocess to handle Ctrl+C. If it terminates as # a result of Ctrl+C, proc.wait() will return a status code, and, # we get out of the loop. If it doesn't, like e.g. gdb, we continue # waiting. while status is None: try: status = proc.wait() except KeyboardInterrupt: pass else: p = ProcessHandlerMixin(args, cwd=cwd, env=use_env, processOutputLine=[handleLine], universal_newlines=True, ignore_children=ignore_children) p.run() p.processOutput() status = None sig = None while status is None: try: if sig is None: status = p.wait() else: status = p.kill(sig=sig) except KeyboardInterrupt: if sig is None: sig = signal.SIGINT elif sig == signal.SIGINT: # If we've already tried SIGINT, escalate. sig = signal.SIGKILL if ensure_exit_code is False: return status if ensure_exit_code is True: ensure_exit_code = 0 if status != ensure_exit_code: raise Exception('Process executed with non-0 exit code %d: %s' % (status, args)) return status
def build(self, what=None, disable_extra_make_dependencies=None, jobs=0, directory=None, verbose=False, keep_going=False): """Build the source tree. With no arguments, this will perform a full build. Positional arguments define targets to build. These can be make targets or patterns like "<dir>/<target>" to indicate a make target within a directory. There are a few special targets that can be used to perform a partial build faster than what `mach build` would perform: * binaries - compiles and links all C/C++ sources and produces shared libraries and executables (binaries). * faster - builds JavaScript, XUL, CSS, etc files. "binaries" and "faster" almost fully complement each other. However, there are build actions not captured by either. If things don't appear to be rebuilding, perform a vanilla `mach build` to rebuild the world. """ from mozbuild.controller.building import ( BuildDriver, ) self.log_manager.enable_all_structured_loggers() loader = MozconfigLoader(self.topsrcdir) mozconfig = loader.read_mozconfig(loader.AUTODETECT) configure_args = mozconfig['configure_args'] doing_pgo = configure_args and 'MOZ_PGO=1' in configure_args append_env = None if doing_pgo: if what: raise Exception('Cannot specify targets (%s) in MOZ_PGO=1 builds' % what) instr = self._spawn(BuildDriver) orig_topobjdir = instr._topobjdir instr._topobjdir = mozpath.join(instr._topobjdir, 'instrumented') append_env = {'MOZ_PROFILE_GENERATE': '1'} status = instr.build( what=what, disable_extra_make_dependencies=disable_extra_make_dependencies, jobs=jobs, directory=directory, verbose=verbose, keep_going=keep_going, mach_context=self._mach_context, append_env=append_env) if status != 0: return status # Packaging the instrumented build is required to get the jarlog # data. status = instr._run_make( directory=".", target='package', silent=not verbose, ensure_exit_code=False, append_env=append_env) if status != 0: return status pgo_env = os.environ.copy() pgo_env['LLVM_PROFDATA'] = instr.config_environment.substs.get('LLVM_PROFDATA') pgo_env['JARLOG_FILE'] = mozpath.join(orig_topobjdir, 'jarlog/en-US.log') pgo_cmd = [ instr.virtualenv_manager.python_path, mozpath.join(self.topsrcdir, 'build/pgo/profileserver.py'), ] subprocess.check_call(pgo_cmd, cwd=instr.topobjdir, env=ensure_subprocess_env(pgo_env)) # Set the default build to MOZ_PROFILE_USE append_env = {'MOZ_PROFILE_USE': '1'} driver = self._spawn(BuildDriver) return driver.build( what=what, disable_extra_make_dependencies=disable_extra_make_dependencies, jobs=jobs, directory=directory, verbose=verbose, keep_going=keep_going, mach_context=self._mach_context, append_env=append_env)
def read_mozconfig(self, path=None): """Read the contents of a mozconfig into a data structure. This takes the path to a mozconfig to load. If the given path is AUTODETECT, will try to find a mozconfig from the environment using find_mozconfig(). mozconfig files are shell scripts. So, we can't just parse them. Instead, we run the shell script in a wrapper which allows us to record state from execution. Thus, the output from a mozconfig is a friendly static data structure. """ if path is self.AUTODETECT: path = self.find_mozconfig() result = { 'path': path, 'topobjdir': None, 'configure_args': None, 'make_flags': None, 'make_extra': None, 'env': None, 'vars': None, } if path is None: return result path = mozpath.normsep(path) result['configure_args'] = [] result['make_extra'] = [] result['make_flags'] = [] # Since mozconfig_loader is a shell script, running it "normally" # actually leads to two shell executions on Windows. Avoid this by # directly calling sh mozconfig_loader. shell = 'sh' if 'MOZILLABUILD' in os.environ: shell = os.environ['MOZILLABUILD'] + '/msys/bin/sh' if sys.platform == 'win32': shell = shell + '.exe' command = [ shell, mozpath.normsep(self._loader_script), mozpath.normsep(self.topsrcdir), path, sys.executable, mozpath.join(mozpath.dirname(self._loader_script), 'action', 'dump_env.py') ] try: env = dict(os.environ) env['PYTHONIOENCODING'] = 'utf-8' # We need to capture stderr because that's where the shell sends # errors if execution fails. output = six.ensure_text( subprocess.check_output(command, stderr=subprocess.STDOUT, cwd=self.topsrcdir, env=ensure_subprocess_env(env), universal_newlines=True)) except subprocess.CalledProcessError as e: lines = e.output.splitlines() # Output before actual execution shouldn't be relevant. try: index = lines.index('------END_BEFORE_SOURCE') lines = lines[index + 1:] except ValueError: pass raise MozconfigLoadException(path, MOZCONFIG_BAD_EXIT_CODE, lines) try: parsed = self._parse_loader_output(output) except AssertionError: # _parse_loader_output uses assertions to verify the # well-formedness of the shell output; when these fail, it # generally means there was a problem with the output, but we # include the assertion traceback just to be sure. print('Assertion failed in _parse_loader_output:') traceback.print_exc() raise MozconfigLoadException(path, MOZCONFIG_BAD_OUTPUT, output.splitlines()) def diff_vars(vars_before, vars_after): set1 = set(vars_before.keys()) - self.IGNORE_SHELL_VARIABLES set2 = set(vars_after.keys()) - self.IGNORE_SHELL_VARIABLES added = set2 - set1 removed = set1 - set2 maybe_modified = set1 & set2 changed = { 'added': {}, 'removed': {}, 'modified': {}, 'unmodified': {}, } for key in added: changed['added'][key] = vars_after[key] for key in removed: changed['removed'][key] = vars_before[key] for key in maybe_modified: if vars_before[key] != vars_after[key]: changed['modified'][key] = (vars_before[key], vars_after[key]) elif key in self.ENVIRONMENT_VARIABLES: # In order for irrelevant environment variable changes not # to incur in re-running configure, only a set of # environment variables are stored when they are # unmodified. Otherwise, changes such as using a different # terminal window, or even rebooting, would trigger # reconfigures. changed['unmodified'][key] = vars_after[key] return changed result['env'] = diff_vars(parsed['env_before'], parsed['env_after']) # Environment variables also appear as shell variables, but that's # uninteresting duplication of information. Filter them out. def filt(x, y): return {k: v for k, v in x.items() if k not in y} result['vars'] = diff_vars( filt(parsed['vars_before'], parsed['env_before']), filt(parsed['vars_after'], parsed['env_after'])) result['configure_args'] = [self._expand(o) for o in parsed['ac']] if 'MOZ_OBJDIR' in parsed['env_before']: result['topobjdir'] = parsed['env_before']['MOZ_OBJDIR'] mk = [self._expand(o) for o in parsed['mk']] for o in mk: match = self.RE_MAKE_VARIABLE.match(o) if match is None: result['make_extra'].append(o) continue name, value = match.group('var'), match.group('value') if name == 'MOZ_MAKE_FLAGS': result['make_flags'] = value.split() continue if name == 'MOZ_OBJDIR': result['topobjdir'] = value if parsed['env_before'].get('MOZ_PROFILE_GENERATE') == '1': # If MOZ_OBJDIR is specified in the mozconfig, we need to # make sure that the '/instrumented' directory gets appended # for the first build to avoid an objdir mismatch when # running 'mach package' on Windows. result['topobjdir'] = mozpath.join(result['topobjdir'], 'instrumented') continue result['make_extra'].append(o) return result