def finder(self):
if self._finder:
return self._finder
self._finder = FileFinder(
mozpath.normsep(self.path),
ignore=[mozpath.normsep(e) for e in self.exclude])
return self._finder
def __init__(self,
topsrcdir,
settings,
log_manager,
topobjdir=None,
mozconfig=MozconfigLoader.AUTODETECT,
virtualenv_name=None):
"""Create a new Mozbuild object instance.
Instances are bound to a source directory, a ConfigSettings instance,
and a LogManager instance. The topobjdir may be passed in as well. If
it isn't, it will be calculated from the active mozconfig.
"""
self.topsrcdir = mozpath.normsep(topsrcdir)
self.settings = settings
self.populate_logger()
self.log_manager = log_manager
self._make = None
self._topobjdir = mozpath.normsep(
topobjdir) if topobjdir else topobjdir
self._mozconfig = mozconfig
self._config_environment = None
self._virtualenv_name = virtualenv_name or ('init_py3'
if six.PY3 else 'init')
self._virtualenv_manager = None
def do_include(self, args, filters=True):
"""
Preprocess a given file.
args can either be a file name, or a file-like object.
Files should be opened, and will be closed after processing.
"""
isName = isinstance(args, six.string_types)
oldCheckLineNumbers = self.checkLineNumbers
self.checkLineNumbers = False
if isName:
try:
args = _to_text(args)
if filters:
args = self.applyFilters(args)
if not os.path.isabs(args):
args = os.path.join(self.curdir, args)
args = io.open(args, "rU", encoding="utf-8")
except Preprocessor.Error:
raise
except Exception:
raise Preprocessor.Error(self, "FILE_NOT_FOUND",
_to_text(args))
self.checkLineNumbers = bool(
re.search("\.(js|jsm|java|webidl)(?:\.in)?$", args.name))
oldFile = self.context["FILE"]
oldLine = self.context["LINE"]
oldDir = self.context["DIRECTORY"]
oldCurdir = self.curdir
self.noteLineInfo()
if args.isatty():
# we're stdin, use '-' and '' for file and dir
self.context["FILE"] = "-"
self.context["DIRECTORY"] = ""
self.curdir = "."
else:
abspath = os.path.abspath(args.name)
self.curdir = os.path.dirname(abspath)
self.includes.add(six.ensure_text(abspath))
if self.topobjdir and path_starts_with(abspath, self.topobjdir):
abspath = "$OBJDIR" + normsep(abspath[len(self.topobjdir):])
elif self.topsrcdir and path_starts_with(abspath, self.topsrcdir):
abspath = "$SRCDIR" + normsep(abspath[len(self.topsrcdir):])
self.context["FILE"] = abspath
self.context["DIRECTORY"] = os.path.dirname(abspath)
self.context["LINE"] = 0
for l in args:
self.context["LINE"] += 1
self.handleLine(l)
if isName:
args.close()
self.context["FILE"] = oldFile
self.checkLineNumbers = oldCheckLineNumbers
self.context["LINE"] = oldLine
self.context["DIRECTORY"] = oldDir
self.curdir = oldCurdir
def match(self, patterns):
a = mozpath.normsep(self.path)
for p in patterns:
if isinstance(p, FilterPath):
p = p.path
p = mozpath.normsep(p)
if mozpath.match(a, p):
return True
return False
def find_sdk_tool(binary, log=None):
if binary.lower().endswith(".exe"):
binary = binary[:-4]
maybe = os.environ.get(binary.upper())
if maybe:
log(
logging.DEBUG,
"msix",
{"binary": binary, "path": maybe},
"Found {binary} in environment: {path}",
)
return mozpath.normsep(maybe)
maybe = which(
binary, extra_search_dirs=["c:/Windows/System32/WindowsPowershell/v1.0"]
)
if maybe:
log(
logging.DEBUG,
"msix",
{"binary": binary, "path": maybe},
"Found {binary} on path: {path}",
)
return mozpath.normsep(maybe)
sdk = os.environ.get("WINDOWSSDKDIR") or "C:/Program Files (x86)/Windows Kits/10"
log(
logging.DEBUG,
"msix",
{"binary": binary, "sdk": sdk},
"Looking for {binary} in Windows SDK: {sdk}",
)
if sdk:
# Like `bin/VERSION/ARCH/tool.exe`.
finder = FileFinder(sdk)
# TODO: handle running on ARM.
is_64bits = sys.maxsize > 2 ** 32
arch = "x64" if is_64bits else "x86"
for p, f in finder.find(
"bin/**/{arch}/{binary}.exe".format(arch=arch, binary=binary)
):
maybe = mozpath.normsep(mozpath.join(sdk, p))
log(
logging.DEBUG,
"msix",
{"binary": binary, "path": maybe},
"Found {binary} in Windows SDK: {path}",
)
return maybe
return None
def _environment_client_mk(self, out, verbose):
if self.mozconfig['make_extra']:
for arg in self.mozconfig['make_extra']:
print(arg, file=out)
objdir = mozpath.normsep(self.topobjdir)
print('MOZ_OBJDIR=%s' % objdir, file=out)
if 'MOZ_CURRENT_PROJECT' in os.environ:
objdir = mozpath.join(objdir, os.environ['MOZ_CURRENT_PROJECT'])
print('OBJDIR=%s' % objdir, file=out)
if self.mozconfig['path']:
print('FOUND_MOZCONFIG=%s' % mozpath.normsep(self.mozconfig['path']),
file=out)
def hash_paths(base_path, patterns):
"""
Give a list of path patterns, return a digest of the contents of all
the corresponding files, similarly to git tree objects or mercurial
manifests.
Each file is hashed. The list of all hashes and file paths is then
itself hashed to produce the result.
"""
finder = get_file_finder(base_path)
h = hashlib.sha256()
files = {}
for pattern in patterns:
found = list(finder.find(pattern))
if found:
files.update(found)
else:
raise Exception("%s did not match anything" % pattern)
for path in sorted(files.keys()):
if path.endswith((".pyc", ".pyd", ".pyo")):
continue
h.update(
six.ensure_binary("{} {}\n".format(
hash_path(mozpath.abspath(mozpath.join(base_path, path))),
mozpath.normsep(path),
)))
return h.hexdigest()
def add(self, name, data, compress=None, mode=None):
'''
Add a new member to the jar archive, with the given name and the given
data.
The compress option indicates if the given data should be compressed
(True), not compressed (False), or compressed according to the default
defined when creating the JarWriter (None).
When the data should be compressed (True or None with self.compress ==
True), it is only really compressed if the compressed size is smaller
than the uncompressed size.
The mode option gives the unix permissions that should be stored
for the jar entry.
The given data may be a buffer, a file-like instance, a Deflater or a
JarFileReader instance. The latter two allow to avoid uncompressing
data to recompress it.
'''
name = mozpath.normsep(name)
if name in self._contents:
raise JarWriterError("File %s already in JarWriter" % name)
if compress is None:
compress = self._compress
if (isinstance(data, JarFileReader) and data.compressed == compress) \
or (isinstance(data, Deflater) and data.compress == compress):
deflater = data
else:
deflater = Deflater(compress, compress_level=self._compress_level)
if isinstance(data, basestring):
deflater.write(data)
elif hasattr(data, 'read'):
if hasattr(data, 'seek'):
data.seek(0)
deflater.write(data.read())
else:
raise JarWriterError("Don't know how to handle %s" %
type(data))
# Fill a central directory entry for this new member.
entry = JarCdirEntry()
entry['creator_version'] = 20
if mode is not None:
# Set creator host system (upper byte of creator_version)
# to 3 (Unix) so mode is honored when there is one.
entry['creator_version'] |= 3 << 8
entry['external_attr'] = (mode & 0xFFFF) << 16L
if deflater.compressed:
entry['min_version'] = 20 # Version 2.0 supports deflated streams
entry['general_flag'] = 2 # Max compression
entry['compression'] = JAR_DEFLATED
else:
entry['min_version'] = 10 # Version 1.0 for stored streams
entry['general_flag'] = 0
entry['compression'] = JAR_STORED
# January 1st, 2010. See bug 592369.
entry['lastmod_date'] = ((2010 - 1980) << 9) | (1 << 5) | 1
entry['lastmod_time'] = 0
entry['crc32'] = deflater.crc32
entry['compressed_size'] = deflater.compressed_size
entry['uncompressed_size'] = deflater.uncompressed_size
entry['filename'] = name
self._contents[name] = entry, deflater.compressed_data
def __init__(self, environment):
assert isinstance(environment, ConfigEnvironment)
self.populate_logger()
self.environment = environment
self.summary = BackendConsumeSummary()
# Files whose modification should cause a new read and backend
# generation.
self.backend_input_files = set()
# Files generated by the backend.
self._backend_output_files = set()
# Previously generated files.
self._backend_output_list_file = mozpath.join(
environment.topobjdir, 'backend.%s' % self.__class__.__name__)
self._backend_output_list = set()
if os.path.exists(self._backend_output_list_file):
l = open(self._backend_output_list_file).read().split('\n')
self._backend_output_list.update(mozpath.normsep(p) for p in l)
self._environments = {}
self._environments[environment.topobjdir] = environment
self._init()
def find_shell_scripts(config, paths):
found = dict()
root = config["root"]
exclude = [mozpath.join(root, e) for e in config.get("exclude", [])]
if config.get("extensions"):
pattern = "**/*.{}".format(config.get("extensions")[0])
else:
pattern = "**/*.sh"
files = []
for path in paths:
path = mozpath.normsep(path)
ignore = [
e[len(path):].lstrip("/") for e in exclude
if mozpath.commonprefix((path, e)) == path
]
finder = FileFinder(path, ignore=ignore)
files.extend([os.path.join(path, p) for p, f in finder.find(pattern)])
for filename in files:
shell = determine_shell_from_script(filename)
if shell:
found[filename] = shell
return found
def hash_paths_extended(base_path, patterns):
"""
Works like taskgraph.util.hash.hash_paths, except it is able to account for Thunderbird source
code being part of a separate repository.
Two file finders are created if necessary.
"""
gecko_patterns, comm_patterns = split_patterns_list(patterns)
gecko_finder = get_file_finder(base_path)
comm_finder = get_file_finder(mozpath.join(base_path, "comm"))
h = hashlib.sha256()
files = []
for (patterns, finder, prefix) in [
(gecko_patterns, gecko_finder, None),
(comm_patterns, comm_finder, "comm/"),
]:
for pattern in patterns:
if prefix:
pattern = pattern.lstrip(prefix)
found = list(process_found(finder.find(pattern), prefix))
if found:
files.extend(found)
else:
raise Exception("%s did not match anything" % pattern)
for path in sorted(files):
if path.endswith((".pyc", ".pyd", ".pyo")):
continue
h.update(
six.ensure_binary("{} {}\n".format(
hash_path(mozpath.abspath(mozpath.join(base_path, path))),
mozpath.normsep(path),
)))
return h.hexdigest()
def parse_chrome_manifest(path, base_path, chrome_entries):
for entry in parse_manifest(None, path):
if isinstance(entry, Manifest):
parse_chrome_manifest(
os.path.join(os.path.dirname(path), entry.relpath),
base_path,
chrome_entries,
)
elif isinstance(entry, ManifestLocale):
entry_path = os.path.join(
os.path.relpath(os.path.dirname(path), base_path),
entry.relpath)
chrome_entries.append({
"type":
"locale",
"alias":
entry.name,
"locale":
entry.id,
"platforms":
convert_entry_flags_to_platform_codes(entry.flags),
"path":
mozpath.normsep(entry_path),
})
else:
raise Exception("Unknown type {0}".format(entry.name))
def parse_chrome_manifest(path, base_path, chrome_entries):
for entry in parse_manifest(None, path):
if isinstance(entry, Manifest):
parse_chrome_manifest(
os.path.join(os.path.dirname(path), entry.relpath),
base_path,
chrome_entries
)
elif isinstance(entry, ManifestLocale):
entry_path = os.path.join(
os.path.relpath(
os.path.dirname(path),
base_path
),
entry.relpath
)
chrome_entries.append({
'type': 'locale',
'alias': entry.name,
'locale': entry.id,
'platforms': convert_entry_flags_to_platform_codes(entry.flags),
'path': mozpath.normsep(entry_path)
})
else:
raise Exception('Unknown type {0}'.format(entry.name))
def _environment_configure(self, out, verbose):
if self.mozconfig['path']:
# Replace ' with '"'"', so that shell quoting e.g.
# a'b becomes 'a'"'"'b'.
quote = lambda s: s.replace("'", """'"'"'""")
if self.mozconfig['configure_args'] and \
'COMM_BUILD' not in os.environ:
print('echo Adding configure options from %s' %
mozpath.normsep(self.mozconfig['path']),
file=out)
for arg in self.mozconfig['configure_args']:
quoted_arg = quote(arg)
print("echo ' %s'" % quoted_arg, file=out)
print("""set -- "$@" '%s'""" % quoted_arg, file=out)
for key, value in self.mozconfig['env']['added'].items():
print("export %s='%s'" % (key, quote(value)), file=out)
for key, (old, value) in self.mozconfig['env']['modified'].items():
print("export %s='%s'" % (key, quote(value)), file=out)
for key, value in self.mozconfig['vars']['added'].items():
print("%s='%s'" % (key, quote(value)), file=out)
for key, (old,
value) in self.mozconfig['vars']['modified'].items():
print("%s='%s'" % (key, quote(value)), file=out)
for key in self.mozconfig['env']['removed'].keys() + \
self.mozconfig['vars']['removed'].keys():
print("unset %s" % key, file=out)
def _add_manifest_file(self, path, file):
'''
Add the given BaseFile with manifest file contents with the given path.
'''
self._manifests.add(path)
base = ''
if hasattr(file, 'path'):
# Find the directory the given path is relative to.
b = mozpath.normsep(file.path)
if b.endswith('/' + path) or b == path:
base = os.path.normpath(b[:-len(path)])
for e in parse_manifest(base, path, file.open()):
# ManifestResources need to be given after ManifestChrome, so just
# put all ManifestChrome in a separate queue to make them first.
if isinstance(e, ManifestChrome):
# e.move(e.base) just returns a clone of the entry.
self._chrome_queue.append(self.formatter.add_manifest,
e.move(e.base))
elif not isinstance(e, (Manifest, ManifestInterfaces)):
self._queue.append(self.formatter.add_manifest, e.move(e.base))
# If a binary component is added to an addon, prevent the addon
# from being packed.
if isinstance(e, ManifestBinaryComponent):
addon = mozpath.basedir(e.base, self._addons)
if addon:
self._addons[addon] = 'unpacked'
if isinstance(e, Manifest):
if e.flags:
errors.fatal('Flags are not supported on ' +
'"manifest" entries')
self._included_manifests[e.path] = path
def test_list_files(self):
env = self._consume('linkage', RecursiveMakeBackend)
expected_list_files = {
'prog/MyProgram_exe.list': [
'../static/bar/bar1.o',
'../static/bar/bar2.o',
'../static/bar/bar_helper/bar_helper1.o',
],
'shared/baz_so.list': [
'baz/baz1.o',
],
}
actual_list_files = {}
for name in expected_list_files.keys():
with open(os.path.join(env.topobjdir, name), 'rb') as fh:
actual_list_files[name] = [
mozpath.normsep(line.rstrip()) for line in fh.readlines()
]
for name in expected_list_files:
self.assertEqual(actual_list_files[name],
expected_list_files[name])
# We don't produce a list file for a shared library composed only of
# object files in its directory, but instead list them in a variable.
with open(os.path.join(env.topobjdir, 'prog', 'qux', 'backend.mk'),
'rb') as fh:
lines = [line.rstrip() for line in fh.readlines()]
self.assertIn('qux.so_OBJS := qux1.o', lines)
def __init__(self, environment):
assert isinstance(environment, ConfigEnvironment)
self.populate_logger()
self.environment = environment
self.summary = BackendConsumeSummary()
# Files whose modification should cause a new read and backend
# generation.
self.backend_input_files = set()
# Files generated by the backend.
self._backend_output_files = set()
# Previously generated files.
self._backend_output_list_file = mozpath.join(environment.topobjdir,
'backend.%s' % self.__class__.__name__)
self._backend_output_list = set()
if os.path.exists(self._backend_output_list_file):
l = open(self._backend_output_list_file).read().split('\n')
self._backend_output_list.update(mozpath.normsep(p) for p in l)
# Pull in all loaded Python as dependencies so any Python changes that
# could influence our output result in a rescan.
self.backend_input_files |= set(iter_modules_in_path(environment.topsrcdir))
self.backend_input_files |= set(iter_modules_in_path(environment.topobjdir))
self._environments = {}
self._environments[environment.topobjdir] = environment
self._init()
def test_config_environment(self):
d = os.path.realpath(tempfile.mkdtemp())
try:
with open(os.path.join(d, 'config.status'), 'w') as fh:
fh.write('# coding=utf-8\n')
fh.write('from __future__ import unicode_literals\n')
fh.write("topobjdir = '%s'\n" % mozpath.normsep(d))
fh.write("topsrcdir = '%s'\n" % topsrcdir)
fh.write("mozconfig = None\n")
fh.write("defines = { 'FOO': 'foo' }\n")
fh.write("non_global_defines = ['BAR']\n")
fh.write("substs = { 'QUX': 'qux' }\n")
fh.write("__all__ = ['topobjdir', 'topsrcdir', 'defines', "
"'non_global_defines', 'substs', 'mozconfig']")
base = self.get_base(topobjdir=d)
ce = base.config_environment
self.assertIsInstance(ce, ConfigEnvironment)
self.assertEqual(base.defines, ce.defines)
self.assertEqual(base.substs, ce.substs)
self.assertEqual(base.defines, {'FOO': 'foo'})
self.assertEqual(
base.substs, {
'ACDEFINES': '-DFOO=foo',
'ALLEMPTYSUBSTS': '',
'ALLSUBSTS': 'ACDEFINES = -DFOO=foo\nQUX = qux',
'QUX': 'qux',
})
finally:
shutil.rmtree(d)
def canonicalize(url):
'''
The jar path is stored in a MOZ_JAR_LOG_FILE log as a url. This method
returns a unique value corresponding to such urls.
- file:///{path} becomes {path}
- jar:file:///{path}!/{subpath} becomes ({path}, {subpath})
- jar:jar:file:///{path}!/{subpath}!/{subpath2} becomes
({path}, {subpath}, {subpath2})
'''
if not isinstance(url, ParseResult):
# Assume that if it doesn't start with jar: or file:, it's a path.
if not url.startswith(('jar:', 'file:')):
url = 'file:///' + os.path.abspath(url)
url = urlparse(url)
assert url.scheme
assert url.scheme in ('jar', 'file')
if url.scheme == 'jar':
path = JarLog.canonicalize(url.path)
if isinstance(path, tuple):
return path[:-1] + tuple(path[-1].split('!/', 1))
return tuple(path.split('!/', 1))
if url.scheme == 'file':
assert os.path.isabs(url.path)
path = url.path
# On Windows, url.path will be /drive:/path ; on Unix systems,
# /path. As we want drive:/path instead of /drive:/path on Windows,
# remove the leading /.
if os.path.isabs(path[1:]):
path = path[1:]
path = os.path.realpath(path)
return mozpath.normsep(os.path.normcase(path))
def __init__(self, config):
self.populate_logger()
self.config = config
mozinfo.find_and_update_from_json(config.topobjdir)
# Python 2.6 doesn't allow unicode keys to be used for keyword
# arguments. This gross hack works around the problem until we
# rid ourselves of 2.6.
self.info = {}
for k, v in mozinfo.info.items():
if isinstance(k, unicode):
k = k.encode('ascii')
self.info[k] = v
self._libs = OrderedDefaultDict(list)
self._binaries = OrderedDict()
self._linkage = []
self._static_linking_shared = set()
# Keep track of external paths (third party build systems), starting
# from what we run a subconfigure in. We'll eliminate some directories
# as we traverse them with moz.build (e.g. js/src).
subconfigures = os.path.join(self.config.topobjdir, 'subconfigures')
paths = []
if os.path.exists(subconfigures):
paths = open(subconfigures).read().splitlines()
self._external_paths = set(mozpath.normsep(d) for d in paths)
# Add security/nss manually, since it doesn't have a subconfigure.
self._external_paths.add('security/nss')
def setup(root, **lintargs):
log = lintargs["log"]
virtualenv_bin_path = lintargs.get("virtualenv_bin_path")
# Using `which` searches multiple directories and handles `.exe` on Windows.
binary = which("black", path=(virtualenv_bin_path, default_bindir()))
if binary and os.path.exists(binary):
binary = mozpath.normsep(binary)
log.debug("Looking for black at {}".format(binary))
version = get_black_version(binary)
versions = [
line.split()[0].strip()
for line in open(BLACK_REQUIREMENTS_PATH).readlines()
if line.startswith("black==")
]
if ["black=={}".format(version)] == versions:
log.debug(
"Black is present with expected version {}".format(version))
return 0
else:
log.debug(
"Black is present but unexpected version {}".format(version))
log.debug("Black needs to be installed or updated")
virtualenv_manager = lintargs["virtualenv_manager"]
try:
virtualenv_manager.install_pip_requirements(BLACK_REQUIREMENTS_PATH,
quiet=True)
except subprocess.CalledProcessError:
print(BLACK_INSTALL_ERROR)
return 1
def test_include_failures(self):
with self.assertRaises(ConfigureError) as e:
with self.moz_configure('include("../foo.configure")'):
self.get_config()
self.assertEquals(
e.exception.message,
'Cannot include `%s` because it is not in a subdirectory of `%s`' %
(mozpath.normpath(
mozpath.join(test_data_path, '..', 'foo.configure')),
mozpath.normsep(test_data_path)))
with self.assertRaises(ConfigureError) as e:
with self.moz_configure('''
include('extra.configure')
include('extra.configure')
'''):
self.get_config()
self.assertEquals(
e.exception.message,
'Cannot include `%s` because it was included already.' %
mozpath.normpath(mozpath.join(test_data_path, 'extra.configure')))
with self.assertRaises(TypeError) as e:
with self.moz_configure('''
include(42)
'''):
self.get_config()
self.assertEquals(e.exception.message, "Unexpected type: 'int'")
def __init__(self,
topsrcdir,
topobjdir,
dist,
group=Grouping.NO,
abspaths=False):
topsrcdir = mozpath.normsep(
os.path.normcase(os.path.abspath(topsrcdir)))
topobjdir = mozpath.normsep(
os.path.normcase(os.path.abspath(topobjdir)))
dist = mozpath.normsep(os.path.normcase(os.path.abspath(dist)))
if abspaths:
topsrcdir_value = topsrcdir
topobjdir_value = topobjdir
dist_value = dist
else:
topsrcdir_value = '$(topsrcdir)'
topobjdir_value = '$(DEPTH)'
dist_value = '$(DIST)'
self._normpaths = {
topsrcdir: topsrcdir_value,
topobjdir: topobjdir_value,
dist: dist_value,
'$(topsrcdir)': topsrcdir_value,
'$(DEPTH)': topobjdir_value,
'$(DIST)': dist_value,
'$(depth)':
topobjdir_value, # normcase may lowercase variable refs when
'$(dist)': dist_value, # they are in the original dependency file
mozpath.relpath(topobjdir, os.curdir): topobjdir_value,
mozpath.relpath(dist, os.curdir): dist_value,
}
try:
# mozpath.relpath(topsrcdir, os.curdir) fails when source directory
# and object directory are not on the same drive on Windows. In
# this case, the value is not useful in self._normpaths anyways.
self._normpaths[mozpath.relpath(topsrcdir,
os.curdir)] = topsrcdir_value
except ValueError:
pass
Makefile.__init__(self)
self._group = group
self._targets = OrderedDict()
def test_mach_command_base_inside_objdir(self):
"""Ensure a MachCommandBase constructed from inside the objdir works."""
d = os.path.realpath(tempfile.mkdtemp())
try:
topobjdir = os.path.join(d, "objdir")
os.makedirs(topobjdir)
topsrcdir = os.path.join(d, "srcdir")
prepare_tmp_topsrcdir(topsrcdir)
mozinfo = os.path.join(topobjdir, "mozinfo.json")
with open(mozinfo, "wt") as fh:
json.dump(
dict(
topsrcdir=topsrcdir,
),
fh,
)
os.chdir(topobjdir)
class MockMachContext(object):
pass
context = MockMachContext()
context.cwd = topobjdir
context.topdir = topsrcdir
context.settings = None
context.log_manager = None
context.detect_virtualenv_mozinfo = False
o = MachCommandBase(context, None)
self.assertEqual(o.topobjdir, mozpath.normsep(topobjdir))
self.assertEqual(o.topsrcdir, mozpath.normsep(topsrcdir))
finally:
os.chdir(self._old_cwd)
shutil.rmtree(d)
def resolve_mozconfig_topobjdir(self, default=None):
topobjdir = self.mozconfig['topobjdir'] or default
if not topobjdir:
return None
if '@CONFIG_GUESS@' in topobjdir:
topobjdir = topobjdir.replace('@CONFIG_GUESS@',
self.resolve_config_guess())
if not os.path.isabs(topobjdir):
topobjdir = os.path.abspath(os.path.join(self.topsrcdir, topobjdir))
return mozpath.normsep(os.path.normpath(topobjdir))
def update_sources(topsrcdir):
print('Updating ICU sources lists...')
sys.path.append(mozpath.join(topsrcdir, 'build/pymake'))
for d in ['common', 'i18n']:
base_path = mozpath.join(topsrcdir, 'intl/icu/source/%s' % d)
makefile = mozpath.join(base_path, 'Makefile.in')
mozbuild = mozpath.join(topsrcdir,
'config/external/icu/%s/sources.mozbuild' % d)
sources = [mozpath.relpath(s, topsrcdir)
for s in get_sources_from_makefile(makefile)]
headers = [mozpath.normsep(os.path.relpath(s, topsrcdir))
for s in list_headers(mozpath.join(base_path, 'unicode'))]
write_sources(mozbuild, sources, headers)
def resolve_mozconfig_topobjdir(topsrcdir, mozconfig, default=None):
topobjdir = mozconfig['topobjdir'] or default
if not topobjdir:
return None
if '@CONFIG_GUESS@' in topobjdir:
topobjdir = topobjdir.replace('@CONFIG_GUESS@',
MozbuildObject.resolve_config_guess(mozconfig, topsrcdir))
if not os.path.isabs(topobjdir):
topobjdir = os.path.abspath(os.path.join(topsrcdir, topobjdir))
return mozpath.normsep(os.path.normpath(topobjdir))
def __init__(self,
topsrcdir,
topobjdir,
dist,
group=Grouping.NO,
abspaths=False):
topsrcdir = mozpath.normsep(
os.path.normcase(os.path.abspath(topsrcdir)))
topobjdir = mozpath.normsep(
os.path.normcase(os.path.abspath(topobjdir)))
dist = mozpath.normsep(os.path.normcase(os.path.abspath(dist)))
if abspaths:
topsrcdir_value = topsrcdir
topobjdir_value = topobjdir
dist_value = dist
else:
topsrcdir_value = '$(topsrcdir)'
topobjdir_value = '$(DEPTH)'
dist_value = '$(DIST)'
self._normpaths = {
topsrcdir: topsrcdir_value,
topobjdir: topobjdir_value,
dist: dist_value,
'$(topsrcdir)': topsrcdir_value,
'$(DEPTH)': topobjdir_value,
'$(DIST)': dist_value,
'$(depth)':
topobjdir_value, # normcase may lowercase variable refs when
'$(dist)': dist_value, # they are in the original dependency file
mozpath.relpath(topsrcdir, os.curdir): topsrcdir_value,
mozpath.relpath(topobjdir, os.curdir): topobjdir_value,
mozpath.relpath(dist, os.curdir): dist_value,
}
Makefile.__init__(self)
self._group = group
self._targets = OrderedDict()
def test_jarlog(self):
base = 'file:' + pathname2url(os.path.abspath(os.curdir))
s = StringIO('\n'.join([
base + '/bar/baz.jar first',
base + '/bar/baz.jar second',
base + '/bar/baz.jar third',
base + '/bar/baz.jar second',
base + '/bar/baz.jar second',
'jar:' + base + '/qux.zip!/omni.ja stuff',
base + '/bar/baz.jar first',
'jar:' + base + '/qux.zip!/omni.ja other/stuff',
'jar:' + base + '/qux.zip!/omni.ja stuff',
base + '/bar/baz.jar third',
'jar:jar:' + base + '/qux.zip!/baz/baz.jar!/omni.ja nested/stuff',
'jar:jar:jar:' + base + '/qux.zip!/baz/baz.jar!/foo.zip!/omni.ja' +
' deeply/nested/stuff',
]))
log = JarLog(fileobj=s)
canonicalize = lambda p: \
mozpath.normsep(os.path.normcase(os.path.realpath(p)))
baz_jar = canonicalize('bar/baz.jar')
qux_zip = canonicalize('qux.zip')
self.assertEqual(
set(log.keys()),
set([
baz_jar,
(qux_zip, 'omni.ja'),
(qux_zip, 'baz/baz.jar', 'omni.ja'),
(qux_zip, 'baz/baz.jar', 'foo.zip', 'omni.ja'),
]))
self.assertEqual(log[baz_jar], [
'first',
'second',
'third',
])
self.assertEqual(log[(qux_zip, 'omni.ja')], [
'stuff',
'other/stuff',
])
self.assertEqual(log[(qux_zip, 'baz/baz.jar', 'omni.ja')],
['nested/stuff'])
self.assertEqual(log[(qux_zip, 'baz/baz.jar', 'foo.zip', 'omni.ja')],
['deeply/nested/stuff'])
# The above tests also indirectly check the value returned by
# JarLog.canonicalize for various jar: and file: urls, but
# JarLog.canonicalize also supports plain paths.
self.assertEqual(JarLog.canonicalize(os.path.abspath('bar/baz.jar')),
baz_jar)
self.assertEqual(JarLog.canonicalize('bar/baz.jar'), baz_jar)
def test_objdir_config_status(self):
"""Ensure @CONFIG_GUESS@ is handled when loading mozconfig."""
base = self.get_base()
cmd = base._normalize_command(
[os.path.join(topsrcdir, 'build', 'autoconf', 'config.guess')],
True)
guess = subprocess.check_output(cmd, cwd=topsrcdir).strip()
# There may be symlinks involved, so we use real paths to ensure
# path consistency.
d = os.path.realpath(tempfile.mkdtemp())
try:
mozconfig = os.path.join(d, 'mozconfig')
with open(mozconfig, 'wt') as fh:
fh.write(
'mk_add_options MOZ_OBJDIR=@TOPSRCDIR@/foo/@CONFIG_GUESS@')
print('Wrote mozconfig %s' % mozconfig)
topobjdir = os.path.join(d, 'foo', guess)
os.makedirs(topobjdir)
# Create a fake topsrcdir.
guess_path = os.path.join(d, 'build', 'autoconf', 'config.guess')
os.makedirs(os.path.dirname(guess_path))
shutil.copy(
os.path.join(
topsrcdir,
'build',
'autoconf',
'config.guess',
), guess_path)
mozinfo = os.path.join(topobjdir, 'mozinfo.json')
with open(mozinfo, 'wt') as fh:
json.dump(dict(
topsrcdir=d,
mozconfig=mozconfig,
), fh)
os.environ[b'MOZCONFIG'] = mozconfig.encode('utf-8')
os.chdir(topobjdir)
obj = MozbuildObject.from_environment(
detect_virtualenv_mozinfo=False)
self.assertEqual(obj.topobjdir, mozpath.normsep(topobjdir))
finally:
os.chdir(self._old_cwd)
shutil.rmtree(d)