def repackage_mar(
topsrcdir, package, mar, output, mar_format="lzma", arch=None, mar_channel_id=None
):
if not zipfile.is_zipfile(package) and not tarfile.is_tarfile(package):
raise Exception("Package file %s is not a valid .zip or .tar file." % package)
if arch and arch not in _BCJ_OPTIONS:
raise Exception("Unknown architecture {}, available architectures: {}".format(
arch, list(_BCJ_OPTIONS.keys())))
ensureParentDir(output)
tmpdir = tempfile.mkdtemp()
try:
if tarfile.is_tarfile(package):
z = tarfile.open(package)
z.extractall(tmpdir)
filelist = z.getnames()
z.close()
else:
z = zipfile.ZipFile(package)
z.extractall(tmpdir)
filelist = z.namelist()
z.close()
toplevel_dirs = set([mozpath.split(f)[0] for f in filelist])
excluded_stuff = set([' ', '.background', '.DS_Store', '.VolumeIcon.icns'])
toplevel_dirs = toplevel_dirs - excluded_stuff
# Make sure the .zip file just contains a directory like 'firefox/' at
# the top, and find out what it is called.
if len(toplevel_dirs) != 1:
raise Exception("Package file is expected to have a single top-level directory"
"(eg: 'firefox'), not: %s" % toplevel_dirs)
ffxdir = mozpath.join(tmpdir, toplevel_dirs.pop())
make_full_update = mozpath.join(topsrcdir, 'tools/update-packaging/make_full_update.sh')
env = os.environ.copy()
env['MOZ_PRODUCT_VERSION'] = get_application_ini_value(tmpdir, 'App', 'Version')
env['MAR'] = mozpath.normpath(mar)
if arch:
env['BCJ_OPTIONS'] = ' '.join(_BCJ_OPTIONS[arch])
if mar_format == 'bz2':
env['MAR_OLD_FORMAT'] = '1'
if mar_channel_id:
env['MAR_CHANNEL_ID'] = mar_channel_id
# The Windows build systems have xz installed but it isn't in the path
# like it is on Linux and Mac OS X so just use the XZ env var so the mar
# generation scripts can find it.
xz_path = mozpath.join(topsrcdir, 'xz/xz.exe')
if os.path.exists(xz_path):
env['XZ'] = mozpath.normpath(xz_path)
cmd = [make_full_update, output, ffxdir]
if sys.platform == 'win32':
# make_full_update.sh is a bash script, and Windows needs to
# explicitly call out the shell to execute the script from Python.
cmd.insert(0, env['MOZILLABUILD'] + '/msys/bin/bash.exe')
subprocess.check_call(cmd, env=ensure_subprocess_env(env))
finally:
shutil.rmtree(tmpdir)
def __init__(
self,
config,
gyp_dir_attrs,
path,
output,
executor,
action_overrides,
non_unified_sources,
):
self._path = path
self._config = config
self._output = output
self._non_unified_sources = non_unified_sources
self._gyp_dir_attrs = gyp_dir_attrs
self._action_overrides = action_overrides
self.execution_time = 0.0
self._results = []
# gyp expects plain str instead of unicode. The frontend code gives us
# unicode strings, so convert them.
if config.substs["CC_TYPE"] == "clang-cl":
# This isn't actually used anywhere in this generator, but it's needed
# to override the registry detection of VC++ in gyp.
os.environ.update(
ensure_subprocess_env({
"GYP_MSVS_OVERRIDE_PATH":
"fake_path",
"GYP_MSVS_VERSION":
config.substs["MSVS_VERSION"],
}))
params = {
"parallel": False,
"generator_flags": {},
"build_files": [path],
"root_targets": None,
}
if gyp_dir_attrs.no_chromium:
includes = []
depth = mozpath.dirname(path)
else:
depth = chrome_src
# Files that gyp_chromium always includes
includes = [
mozpath.join(script_dir, "gyp_includes", "common.gypi")
]
finder = FileFinder(chrome_src)
includes.extend(
mozpath.join(chrome_src, name)
for name, _ in finder.find("*/supplement.gypi"))
str_vars = dict(gyp_dir_attrs.variables)
str_vars["python"] = sys.executable
self._gyp_loader_future = executor.submit(load_gyp, [path], "mozbuild",
str_vars, includes, depth,
params)
def wrapper(*args, **kwargs):
if 'env' not in kwargs:
kwargs['env'] = dict(self._environ)
# Subprocess on older Pythons can't handle unicode keys or
# values in environment dicts while subprocess on newer Pythons
# needs text in the env. Normalize automagically so callers
# don't have to deal with this.
kwargs['env'] = ensure_subprocess_env(kwargs['env'], encoding=system_encoding)
return function(*args, **kwargs)
def push_to_try(self, message):
try:
subprocess.check_call((self._tool, 'push-to-try', '-m', message), cwd=self.path,
env=ensure_subprocess_env(self._env))
except subprocess.CalledProcessError:
try:
self._run('showconfig', 'extensions.push-to-try')
except subprocess.CalledProcessError:
raise MissingVCSExtension('push-to-try')
raise
finally:
self._run('revert', '-a')
def __init__(self, config, gyp_dir_attrs, path, output, executor,
action_overrides, non_unified_sources):
self._path = path
self._config = config
self._output = output
self._non_unified_sources = non_unified_sources
self._gyp_dir_attrs = gyp_dir_attrs
self._action_overrides = action_overrides
self.execution_time = 0.0
self._results = []
# gyp expects plain str instead of unicode. The frontend code gives us
# unicode strings, so convert them.
path = encode(path)
if config.substs['CC_TYPE'] == 'clang-cl':
# This isn't actually used anywhere in this generator, but it's needed
# to override the registry detection of VC++ in gyp.
os.environ.update(
ensure_subprocess_env({
'GYP_MSVS_OVERRIDE_PATH':
'fake_path',
'GYP_MSVS_VERSION':
config.substs['MSVS_VERSION'],
}))
params = {
b'parallel': False,
b'generator_flags': {},
b'build_files': [path],
b'root_targets': None,
}
if gyp_dir_attrs.no_chromium:
includes = []
depth = mozpath.dirname(path)
else:
depth = chrome_src
# Files that gyp_chromium always includes
includes = [
encode(mozpath.join(script_dir, 'gyp_includes', 'common.gypi'))
]
finder = FileFinder(chrome_src)
includes.extend(
encode(mozpath.join(chrome_src, name))
for name, _ in finder.find('*/supplement.gypi'))
str_vars = dict((name, encode(value))
for name, value in gyp_dir_attrs.variables.items())
self._gyp_loader_future = executor.submit(load_gyp, [path],
b'mozbuild',
str_vars, includes,
encode(depth), params)
def _run(self, *args, **runargs):
return_codes = runargs.get('return_codes', [])
cmd = (self._tool,) + args
try:
return subprocess.check_output(cmd,
cwd=self.path,
env=ensure_subprocess_env(self._env),
universal_newlines=True)
except subprocess.CalledProcessError as e:
if e.returncode in return_codes:
return ''
raise
def push_to_try(self, message):
try:
subprocess.check_call(
(self._tool, "push-to-try", "-m", message),
cwd=self.path,
env=ensure_subprocess_env(self._env),
)
except subprocess.CalledProcessError:
try:
self._run("showconfig", "extensions.push-to-try")
except subprocess.CalledProcessError:
raise MissingVCSExtension("push-to-try")
raise
finally:
self._run("revert", "-a")
def call_process(name, cmd, cwd=None, append_env={}):
env = dict(os.environ)
env.update(ensure_subprocess_env(append_env))
try:
with open(os.devnull, "w") as fnull:
subprocess.check_call(cmd, cwd=cwd, stdout=fnull, env=env)
except subprocess.CalledProcessError:
if cwd:
print("\nError installing %s in the %s folder, aborting." % (name, cwd))
else:
print("\nError installing %s, aborting." % name)
return False
return True
def run_fzf(cmd, tasks):
env = dict(os.environ)
env.update({
'PYTHONPATH':
os.pathsep.join([p for p in sys.path if 'requests' in p])
})
proc = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stdin=subprocess.PIPE,
env=ensure_subprocess_env(env))
out = proc.communicate('\n'.join(tasks))[0].splitlines()
selected = []
query = None
if out:
query = out[0]
selected = out[1:]
return query, selected
def __init__(self, path, hg='hg'):
import hglib.client
super(HgRepository, self).__init__(path, tool=hg)
self._env[b'HGPLAIN'] = b'1'
# Setting this modifies a global variable and makes all future hglib
# instances use this binary. Since the tool path was validated, this
# should be OK. But ideally hglib would offer an API that defines
# per-instance binaries.
hglib.HGPATH = self._tool
# Without connect=False this spawns a persistent process. We want
# the process lifetime tied to a context manager.
self._client = hglib.client.hgclient(self.path, encoding=b'UTF-8',
configs=None, connect=False)
# Work around py3 compat issues in python-hglib
self._client._env = ensure_subprocess_env(self._client._env)
def run_fzf(cmd, tasks):
env = dict(os.environ)
env.update({
"PYTHONPATH":
os.pathsep.join([p for p in sys.path if "requests" in p])
})
proc = subprocess.Popen(
cmd,
stdout=subprocess.PIPE,
stdin=subprocess.PIPE,
env=ensure_subprocess_env(env),
universal_newlines=True,
)
out = proc.communicate("\n".join(tasks))[0].splitlines()
selected = []
query = None
if out:
query = out[0]
selected = out[1:]
return query, selected
def run_process(self,
args=None,
cwd=None,
append_env=None,
explicit_env=None,
log_name=None,
log_level=logging.INFO,
line_handler=None,
require_unix_environment=False,
ensure_exit_code=0,
ignore_children=False,
pass_thru=False,
python_unbuffered=True):
"""Runs a single process to completion.
Takes a list of arguments to run where the first item is the
executable. Runs the command in the specified directory and
with optional environment variables.
append_env -- Dict of environment variables to append to the current
set of environment variables.
explicit_env -- Dict of environment variables to set for the new
process. Any existing environment variables will be ignored.
require_unix_environment if True will ensure the command is executed
within a UNIX environment. Basically, if we are on Windows, it will
execute the command via an appropriate UNIX-like shell.
ignore_children is proxied to mozprocess's ignore_children.
ensure_exit_code is used to ensure the exit code of a process matches
what is expected. If it is an integer, we raise an Exception if the
exit code does not match this value. If it is True, we ensure the exit
code is 0. If it is False, we don't perform any exit code validation.
pass_thru is a special execution mode where the child process inherits
this process's standard file handles (stdin, stdout, stderr) as well as
additional file descriptors. It should be used for interactive processes
where buffering from mozprocess could be an issue. pass_thru does not
use mozprocess. Therefore, arguments like log_name, line_handler,
and ignore_children have no effect.
When python_unbuffered is set, the PYTHONUNBUFFERED environment variable
will be set in the child process. This is normally advantageous (see bug
1627873) but is detrimental in certain circumstances (specifically, we
have seen issues when using pass_thru mode to open a Python subshell, as
in bug 1628838). This variable should be set to False to avoid bustage
in those circumstances.
"""
args = self._normalize_command(args, require_unix_environment)
self.log(logging.INFO, 'new_process', {'args': ' '.join(args)},
'{args}')
def handleLine(line):
# Converts str to unicode on Python 2 and bytes to str on Python 3.
if isinstance(line, bytes):
line = line.decode(sys.stdout.encoding or 'utf-8', 'replace')
if line_handler:
line_handler(line)
if not log_name:
return
self.log(log_level, log_name, {'line': line.rstrip()}, '{line}')
use_env = {}
if explicit_env:
use_env = explicit_env
else:
use_env.update(os.environ)
if append_env:
use_env.update(append_env)
if python_unbuffered:
use_env['PYTHONUNBUFFERED'] = '1'
self.log(logging.DEBUG, 'process', {'env': use_env},
'Environment: {env}')
use_env = ensure_subprocess_env(use_env)
if pass_thru:
proc = subprocess.Popen(args, cwd=cwd, env=use_env)
status = None
# Leave it to the subprocess to handle Ctrl+C. If it terminates as
# a result of Ctrl+C, proc.wait() will return a status code, and,
# we get out of the loop. If it doesn't, like e.g. gdb, we continue
# waiting.
while status is None:
try:
status = proc.wait()
except KeyboardInterrupt:
pass
else:
p = ProcessHandlerMixin(args,
cwd=cwd,
env=use_env,
processOutputLine=[handleLine],
universal_newlines=True,
ignore_children=ignore_children)
p.run()
p.processOutput()
status = None
sig = None
while status is None:
try:
if sig is None:
status = p.wait()
else:
status = p.kill(sig=sig)
except KeyboardInterrupt:
if sig is None:
sig = signal.SIGINT
elif sig == signal.SIGINT:
# If we've already tried SIGINT, escalate.
sig = signal.SIGKILL
if ensure_exit_code is False:
return status
if ensure_exit_code is True:
ensure_exit_code = 0
if status != ensure_exit_code:
raise Exception('Process executed with non-0 exit code %d: %s' %
(status, args))
return status
def build(self, what=None, disable_extra_make_dependencies=None, jobs=0,
directory=None, verbose=False, keep_going=False):
"""Build the source tree.
With no arguments, this will perform a full build.
Positional arguments define targets to build. These can be make targets
or patterns like "<dir>/<target>" to indicate a make target within a
directory.
There are a few special targets that can be used to perform a partial
build faster than what `mach build` would perform:
* binaries - compiles and links all C/C++ sources and produces shared
libraries and executables (binaries).
* faster - builds JavaScript, XUL, CSS, etc files.
"binaries" and "faster" almost fully complement each other. However,
there are build actions not captured by either. If things don't appear to
be rebuilding, perform a vanilla `mach build` to rebuild the world.
"""
from mozbuild.controller.building import (
BuildDriver,
)
self.log_manager.enable_all_structured_loggers()
loader = MozconfigLoader(self.topsrcdir)
mozconfig = loader.read_mozconfig(loader.AUTODETECT)
configure_args = mozconfig['configure_args']
doing_pgo = configure_args and 'MOZ_PGO=1' in configure_args
append_env = None
if doing_pgo:
if what:
raise Exception('Cannot specify targets (%s) in MOZ_PGO=1 builds' %
what)
instr = self._spawn(BuildDriver)
orig_topobjdir = instr._topobjdir
instr._topobjdir = mozpath.join(instr._topobjdir, 'instrumented')
append_env = {'MOZ_PROFILE_GENERATE': '1'}
status = instr.build(
what=what,
disable_extra_make_dependencies=disable_extra_make_dependencies,
jobs=jobs,
directory=directory,
verbose=verbose,
keep_going=keep_going,
mach_context=self._mach_context,
append_env=append_env)
if status != 0:
return status
# Packaging the instrumented build is required to get the jarlog
# data.
status = instr._run_make(
directory=".", target='package',
silent=not verbose, ensure_exit_code=False,
append_env=append_env)
if status != 0:
return status
pgo_env = os.environ.copy()
pgo_env['LLVM_PROFDATA'] = instr.config_environment.substs.get('LLVM_PROFDATA')
pgo_env['JARLOG_FILE'] = mozpath.join(orig_topobjdir, 'jarlog/en-US.log')
pgo_cmd = [
instr.virtualenv_manager.python_path,
mozpath.join(self.topsrcdir, 'build/pgo/profileserver.py'),
]
subprocess.check_call(pgo_cmd, cwd=instr.topobjdir,
env=ensure_subprocess_env(pgo_env))
# Set the default build to MOZ_PROFILE_USE
append_env = {'MOZ_PROFILE_USE': '1'}
driver = self._spawn(BuildDriver)
return driver.build(
what=what,
disable_extra_make_dependencies=disable_extra_make_dependencies,
jobs=jobs,
directory=directory,
verbose=verbose,
keep_going=keep_going,
mach_context=self._mach_context,
append_env=append_env)
def read_mozconfig(self, path=None):
"""Read the contents of a mozconfig into a data structure.
This takes the path to a mozconfig to load. If the given path is
AUTODETECT, will try to find a mozconfig from the environment using
find_mozconfig().
mozconfig files are shell scripts. So, we can't just parse them.
Instead, we run the shell script in a wrapper which allows us to record
state from execution. Thus, the output from a mozconfig is a friendly
static data structure.
"""
if path is self.AUTODETECT:
path = self.find_mozconfig()
result = {
'path': path,
'topobjdir': None,
'configure_args': None,
'make_flags': None,
'make_extra': None,
'env': None,
'vars': None,
}
if path is None:
return result
path = mozpath.normsep(path)
result['configure_args'] = []
result['make_extra'] = []
result['make_flags'] = []
# Since mozconfig_loader is a shell script, running it "normally"
# actually leads to two shell executions on Windows. Avoid this by
# directly calling sh mozconfig_loader.
shell = 'sh'
if 'MOZILLABUILD' in os.environ:
shell = os.environ['MOZILLABUILD'] + '/msys/bin/sh'
if sys.platform == 'win32':
shell = shell + '.exe'
command = [
shell,
mozpath.normsep(self._loader_script),
mozpath.normsep(self.topsrcdir), path, sys.executable,
mozpath.join(mozpath.dirname(self._loader_script), 'action',
'dump_env.py')
]
try:
env = dict(os.environ)
env['PYTHONIOENCODING'] = 'utf-8'
# We need to capture stderr because that's where the shell sends
# errors if execution fails.
output = six.ensure_text(
subprocess.check_output(command,
stderr=subprocess.STDOUT,
cwd=self.topsrcdir,
env=ensure_subprocess_env(env),
universal_newlines=True))
except subprocess.CalledProcessError as e:
lines = e.output.splitlines()
# Output before actual execution shouldn't be relevant.
try:
index = lines.index('------END_BEFORE_SOURCE')
lines = lines[index + 1:]
except ValueError:
pass
raise MozconfigLoadException(path, MOZCONFIG_BAD_EXIT_CODE, lines)
try:
parsed = self._parse_loader_output(output)
except AssertionError:
# _parse_loader_output uses assertions to verify the
# well-formedness of the shell output; when these fail, it
# generally means there was a problem with the output, but we
# include the assertion traceback just to be sure.
print('Assertion failed in _parse_loader_output:')
traceback.print_exc()
raise MozconfigLoadException(path, MOZCONFIG_BAD_OUTPUT,
output.splitlines())
def diff_vars(vars_before, vars_after):
set1 = set(vars_before.keys()) - self.IGNORE_SHELL_VARIABLES
set2 = set(vars_after.keys()) - self.IGNORE_SHELL_VARIABLES
added = set2 - set1
removed = set1 - set2
maybe_modified = set1 & set2
changed = {
'added': {},
'removed': {},
'modified': {},
'unmodified': {},
}
for key in added:
changed['added'][key] = vars_after[key]
for key in removed:
changed['removed'][key] = vars_before[key]
for key in maybe_modified:
if vars_before[key] != vars_after[key]:
changed['modified'][key] = (vars_before[key],
vars_after[key])
elif key in self.ENVIRONMENT_VARIABLES:
# In order for irrelevant environment variable changes not
# to incur in re-running configure, only a set of
# environment variables are stored when they are
# unmodified. Otherwise, changes such as using a different
# terminal window, or even rebooting, would trigger
# reconfigures.
changed['unmodified'][key] = vars_after[key]
return changed
result['env'] = diff_vars(parsed['env_before'], parsed['env_after'])
# Environment variables also appear as shell variables, but that's
# uninteresting duplication of information. Filter them out.
def filt(x, y):
return {k: v for k, v in x.items() if k not in y}
result['vars'] = diff_vars(
filt(parsed['vars_before'], parsed['env_before']),
filt(parsed['vars_after'], parsed['env_after']))
result['configure_args'] = [self._expand(o) for o in parsed['ac']]
if 'MOZ_OBJDIR' in parsed['env_before']:
result['topobjdir'] = parsed['env_before']['MOZ_OBJDIR']
mk = [self._expand(o) for o in parsed['mk']]
for o in mk:
match = self.RE_MAKE_VARIABLE.match(o)
if match is None:
result['make_extra'].append(o)
continue
name, value = match.group('var'), match.group('value')
if name == 'MOZ_MAKE_FLAGS':
result['make_flags'] = value.split()
continue
if name == 'MOZ_OBJDIR':
result['topobjdir'] = value
if parsed['env_before'].get('MOZ_PROFILE_GENERATE') == '1':
# If MOZ_OBJDIR is specified in the mozconfig, we need to
# make sure that the '/instrumented' directory gets appended
# for the first build to avoid an objdir mismatch when
# running 'mach package' on Windows.
result['topobjdir'] = mozpath.join(result['topobjdir'],
'instrumented')
continue
result['make_extra'].append(o)
return result
