def find_dupes(source, allowed_dupes, bail=True):
md5_chunk_size = 1024 * 10
allowed_dupes = set(allowed_dupes)
md5s = OrderedDict()
for p, f in UnpackFinder(source):
md5 = hashlib.md5()
content_size = 0
for buf in iter(functools.partial(f.open().read, md5_chunk_size), b""):
md5.update(six.ensure_binary(buf))
content_size += len(six.ensure_binary(buf))
m = md5.digest()
if m not in md5s:
if isinstance(f, DeflatedFile):
compressed = f.file.compressed_size
else:
compressed = content_size
md5s[m] = (content_size, compressed, [])
md5s[m][2].append(p)
total = 0
total_compressed = 0
num_dupes = 0
unexpected_dupes = []
for m, (size, compressed, paths) in sorted(six.iteritems(md5s),
key=lambda x: x[1][1]):
if len(paths) > 1:
_compressed = " (%d compressed)" % compressed if compressed != size else ""
_times = " (%d times)" % (len(paths) - 1) if len(paths) > 2 else ""
print("Duplicates {} bytes{}{}:".format(size, _compressed, _times))
print("".join(" %s\n" % p for p in paths))
total += (len(paths) - 1) * size
total_compressed += (len(paths) - 1) * compressed
num_dupes += 1
for p in paths:
if not is_l10n_file(p) and normalize_path(
p) not in allowed_dupes:
unexpected_dupes.append(p)
if num_dupes:
total_compressed = ("%d compressed" % total_compressed
if total_compressed != total else "uncompressed")
print("WARNING: Found {} duplicated files taking {} bytes ({})".format(
num_dupes, total, total_compressed))
if unexpected_dupes:
errortype = "ERROR" if bail else "WARNING"
print("{}: The following duplicated files are not allowed:".format(
errortype))
print("\n".join(unexpected_dupes))
if bail:
sys.exit(1)
def process_package_artifact(self, filename, processed_filename):
# Extract all .so files into the root, which will get copied into dist/bin.
with JarWriter(file=processed_filename, optimize=False, compress_level=5) as writer:
for p, f in UnpackFinder(JarFinder(filename, JarReader(filename))):
if not any(mozpath.match(p, pat) for pat in self.package_artifact_patterns):
continue
dirname, basename = os.path.split(p)
self.log(logging.INFO, 'artifact',
{'basename': basename},
'Adding {basename} to processed archive')
basedir = 'bin'
if not basename.endswith('.so'):
basedir = mozpath.join('bin', dirname.lstrip('assets/'))
basename = mozpath.join(basedir, basename)
writer.add(basename.encode('utf-8'), f.open())
def find_dupes(source, allowed_dupes, bail=True):
allowed_dupes = set(allowed_dupes)
md5s = OrderedDict()
for p, f in UnpackFinder(source):
content = f.open().read()
m = hashlib.md5(content).digest()
if m not in md5s:
if isinstance(f, DeflatedFile):
compressed = f.file.compressed_size
else:
compressed = len(content)
md5s[m] = (len(content), compressed, [])
md5s[m][2].append(p)
total = 0
total_compressed = 0
num_dupes = 0
unexpected_dupes = []
for m, (size, compressed, paths) in sorted(md5s.iteritems(),
key=lambda x: x[1][1]):
if len(paths) > 1:
print 'Duplicates %d bytes%s%s:' % (
size, ' (%d compressed)' %
compressed if compressed != size else '', ' (%d times)' %
(len(paths) - 1) if len(paths) > 2 else '')
print ''.join(' %s\n' % p for p in paths)
total += (len(paths) - 1) * size
total_compressed += (len(paths) - 1) * compressed
num_dupes += 1
for p in paths:
if not is_l10n_file(p) and normalize_path(
p) not in allowed_dupes:
unexpected_dupes.append(p)
if num_dupes:
print "WARNING: Found %d duplicated files taking %d bytes (%s)" % \
(num_dupes, total,
'%d compressed' % total_compressed if total_compressed != total
else 'uncompressed')
if unexpected_dupes:
errortype = "ERROR" if bail else "WARNING"
print "%s: The following duplicated files are not allowed:" % errortype
print "\n".join(unexpected_dupes)
if bail:
sys.exit(1)
def process_package_artifact(self, filename, processed_filename):
added_entry = False
with JarWriter(file=processed_filename, optimize=False, compress_level=5) as writer:
for p, f in UnpackFinder(JarFinder(filename, JarReader(filename))):
if not any(mozpath.match(p, pat) for pat in self.package_artifact_patterns):
continue
# strip off the relative "firefox/" bit from the path:
basename = mozpath.relpath(p, "firefox")
basename = mozpath.join('bin', basename)
self.log(logging.INFO, 'artifact',
{'basename': basename},
'Adding {basename} to processed archive')
writer.add(basename.encode('utf-8'), f.open(), mode=f.mode)
added_entry = True
if not added_entry:
raise ValueError('Archive format changed! No pattern from "{patterns}"'
'matched an archive path.'.format(
patterns=self.artifact_patterns))
def process_package_artifact(self, filename, processed_filename):
added_entry = False
with JarWriter(file=processed_filename, optimize=False, compress_level=5) as writer:
with tarfile.open(filename) as reader:
for p, f in UnpackFinder(TarFinder(filename, reader)):
if not any(mozpath.match(p, pat) for pat in self.package_artifact_patterns):
continue
# We strip off the relative "firefox/" bit from the path,
# but otherwise preserve it.
destpath = mozpath.join('bin',
mozpath.relpath(p, "firefox"))
self.log(logging.INFO, 'artifact',
{'destpath': destpath},
'Adding {destpath} to processed archive')
writer.add(destpath.encode('utf-8'), f.open(), mode=f.mode)
added_entry = True
if not added_entry:
raise ValueError('Archive format changed! No pattern from "{patterns}" '
'matched an archive path.'.format(
patterns=LinuxArtifactJob.package_artifact_patterns))
def process_package_artifact(self, filename, processed_filename):
tempdir = tempfile.mkdtemp()
try:
self.log(logging.INFO, 'artifact', {'tempdir': tempdir},
'Unpacking DMG into {tempdir}')
mozinstall.install(
filename,
tempdir) # Doesn't handle already mounted DMG files nicely:
# InstallError: Failed to install "/Users/nalexander/.mozbuild/package-frontend/b38eeeb54cdcf744-firefox-44.0a1.en-US.mac.dmg (local variable 'appDir' referenced before assignment)"
# File "/Users/nalexander/Mozilla/gecko/mobile/android/mach_commands.py", line 250, in artifact_install
# return artifacts.install_from(source, self.distdir)
# File "/Users/nalexander/Mozilla/gecko/python/mozbuild/mozbuild/artifacts.py", line 457, in install_from
# return self.install_from_hg(source, distdir)
# File "/Users/nalexander/Mozilla/gecko/python/mozbuild/mozbuild/artifacts.py", line 445, in install_from_hg
# return self.install_from_url(url, distdir)
# File "/Users/nalexander/Mozilla/gecko/python/mozbuild/mozbuild/artifacts.py", line 418, in install_from_url
# return self.install_from_file(filename, distdir)
# File "/Users/nalexander/Mozilla/gecko/python/mozbuild/mozbuild/artifacts.py", line 336, in install_from_file
# mozinstall.install(filename, tempdir)
# File "/Users/nalexander/Mozilla/gecko/objdir-dce/_virtualenv/lib/python2.7/site-packages/mozinstall/mozinstall.py", line 117, in install
# install_dir = _install_dmg(src, dest)
# File "/Users/nalexander/Mozilla/gecko/objdir-dce/_virtualenv/lib/python2.7/site-packages/mozinstall/mozinstall.py", line 261, in _install_dmg
# subprocess.call('hdiutil detach %s -quiet' % appDir,
bundle_dirs = glob.glob(mozpath.join(tempdir, '*.app'))
if len(bundle_dirs) != 1:
raise ValueError(
'Expected one source bundle, found: {}'.format(
bundle_dirs))
[source] = bundle_dirs
# These get copied into dist/bin without the path, so "root/a/b/c" -> "dist/bin/c".
paths_no_keep_path = (
'Contents/MacOS',
[
'crashreporter.app/Contents/MacOS/crashreporter',
'firefox',
'firefox-bin',
'libfreebl3.dylib',
'liblgpllibs.dylib',
# 'liblogalloc.dylib',
'libmozglue.dylib',
'libnss3.dylib',
'libnssckbi.dylib',
'libnssdbm3.dylib',
'libplugin_child_interpose.dylib',
# 'libreplace_jemalloc.dylib',
# 'libreplace_malloc.dylib',
'libmozavutil.dylib',
'libmozavcodec.dylib',
'libsoftokn3.dylib',
'plugin-container.app/Contents/MacOS/plugin-container',
'updater.app/Contents/MacOS/org.mozilla.updater',
# 'xpcshell',
'XUL',
])
# These get copied into dist/bin with the path, so "root/a/b/c" -> "dist/bin/a/b/c".
paths_keep_path = (
'Contents/Resources',
[
'browser/components/libbrowsercomps.dylib',
'dependentlibs.list',
# 'firefox',
'gmp-clearkey/0.1/libclearkey.dylib',
# 'gmp-fake/1.0/libfake.dylib',
# 'gmp-fakeopenh264/1.0/libfakeopenh264.dylib',
'**/interfaces.xpt',
])
with JarWriter(file=processed_filename,
optimize=False,
compress_level=5) as writer:
root, paths = paths_no_keep_path
finder = UnpackFinder(mozpath.join(source, root))
for path in paths:
for p, f in finder.find(path):
self.log(logging.INFO, 'artifact', {'path': p},
'Adding {path} to processed archive')
destpath = mozpath.join('bin', os.path.basename(p))
writer.add(destpath.encode('utf-8'), f, mode=f.mode)
root, paths = paths_keep_path
finder = UnpackFinder(mozpath.join(source, root))
for path in paths:
for p, f in finder.find(path):
self.log(logging.INFO, 'artifact', {'path': p},
'Adding {path} to processed archive')
destpath = mozpath.join('bin', p)
writer.add(destpath.encode('utf-8'),
f.open(),
mode=f.mode)
finally:
try:
shutil.rmtree(tempdir)
except (OSError, IOError):
self.log(logging.WARN, 'artifact', {'tempdir': tempdir},
'Unable to delete {tempdir}')
pass
def process_package_artifact(self, filename, processed_filename):
tempdir = tempfile.mkdtemp()
oldcwd = os.getcwd()
try:
self.log(logging.INFO, 'artifact',
{'tempdir': tempdir},
'Unpacking DMG into {tempdir}')
if self._substs['HOST_OS_ARCH'] == 'Linux':
# This is a cross build, use hfsplus and dmg tools to extract the dmg.
os.chdir(tempdir)
with open(os.devnull, 'wb') as devnull:
subprocess.check_call([
self._substs['DMG_TOOL'],
'extract',
filename,
'extracted_img',
], stdout=devnull)
subprocess.check_call([
self._substs['HFS_TOOL'],
'extracted_img',
'extractall'
], stdout=devnull)
else:
mozinstall.install(filename, tempdir)
bundle_dirs = glob.glob(mozpath.join(tempdir, '*.app'))
if len(bundle_dirs) != 1:
raise ValueError('Expected one source bundle, found: {}'.format(bundle_dirs))
[source] = bundle_dirs
# These get copied into dist/bin without the path, so "root/a/b/c" -> "dist/bin/c".
paths_no_keep_path = ('Contents/MacOS', [
'crashreporter.app/Contents/MacOS/crashreporter',
'crashreporter.app/Contents/MacOS/minidump-analyzer',
'firefox',
'firefox-bin',
'libfreebl3.dylib',
'liblgpllibs.dylib',
# 'liblogalloc.dylib',
'libmozglue.dylib',
'libnss3.dylib',
'libnssckbi.dylib',
'libnssdbm3.dylib',
'libplugin_child_interpose.dylib',
# 'libreplace_jemalloc.dylib',
# 'libreplace_malloc.dylib',
'libmozavutil.dylib',
'libmozavcodec.dylib',
'libsoftokn3.dylib',
'pingsender',
'plugin-container.app/Contents/MacOS/plugin-container',
'updater.app/Contents/MacOS/org.mozilla.updater',
# 'xpcshell',
'XUL',
])
# These get copied into dist/bin with the path, so "root/a/b/c" -> "dist/bin/a/b/c".
paths_keep_path = ('Contents/Resources', [
'browser/components/libbrowsercomps.dylib',
'dependentlibs.list',
# 'firefox',
'gmp-clearkey/0.1/libclearkey.dylib',
# 'gmp-fake/1.0/libfake.dylib',
# 'gmp-fakeopenh264/1.0/libfakeopenh264.dylib',
'**/interfaces.xpt',
])
with JarWriter(file=processed_filename, optimize=False, compress_level=5) as writer:
root, paths = paths_no_keep_path
finder = UnpackFinder(mozpath.join(source, root))
for path in paths:
for p, f in finder.find(path):
self.log(logging.INFO, 'artifact',
{'path': p},
'Adding {path} to processed archive')
destpath = mozpath.join('bin', os.path.basename(p))
writer.add(destpath.encode('utf-8'), f, mode=f.mode)
root, paths = paths_keep_path
finder = UnpackFinder(mozpath.join(source, root))
for path in paths:
for p, f in finder.find(path):
self.log(logging.INFO, 'artifact',
{'path': p},
'Adding {path} to processed archive')
destpath = mozpath.join('bin', p)
writer.add(destpath.encode('utf-8'), f.open(), mode=f.mode)
finally:
os.chdir(oldcwd)
try:
shutil.rmtree(tempdir)
except (OSError, IOError):
self.log(logging.WARN, 'artifact',
{'tempdir': tempdir},
'Unable to delete {tempdir}')
pass
def process_package_artifact(self, filename, processed_filename):
tempdir = tempfile.mkdtemp()
try:
self.log(logging.INFO, 'artifact',
{'tempdir': tempdir},
'Unpacking DMG into {tempdir}')
mozinstall.install(filename, tempdir) # Doesn't handle already mounted DMG files nicely:
# InstallError: Failed to install "/Users/nalexander/.mozbuild/package-frontend/b38eeeb54cdcf744-firefox-44.0a1.en-US.mac.dmg (local variable 'appDir' referenced before assignment)"
# File "/Users/nalexander/Mozilla/gecko/mobile/android/mach_commands.py", line 250, in artifact_install
# return artifacts.install_from(source, self.distdir)
# File "/Users/nalexander/Mozilla/gecko/python/mozbuild/mozbuild/artifacts.py", line 457, in install_from
# return self.install_from_hg(source, distdir)
# File "/Users/nalexander/Mozilla/gecko/python/mozbuild/mozbuild/artifacts.py", line 445, in install_from_hg
# return self.install_from_url(url, distdir)
# File "/Users/nalexander/Mozilla/gecko/python/mozbuild/mozbuild/artifacts.py", line 418, in install_from_url
# return self.install_from_file(filename, distdir)
# File "/Users/nalexander/Mozilla/gecko/python/mozbuild/mozbuild/artifacts.py", line 336, in install_from_file
# mozinstall.install(filename, tempdir)
# File "/Users/nalexander/Mozilla/gecko/objdir-dce/_virtualenv/lib/python2.7/site-packages/mozinstall/mozinstall.py", line 117, in install
# install_dir = _install_dmg(src, dest)
# File "/Users/nalexander/Mozilla/gecko/objdir-dce/_virtualenv/lib/python2.7/site-packages/mozinstall/mozinstall.py", line 261, in _install_dmg
# subprocess.call('hdiutil detach %s -quiet' % appDir,
bundle_dirs = glob.glob(mozpath.join(tempdir, '*.app'))
if len(bundle_dirs) != 1:
raise ValueError('Expected one source bundle, found: {}'.format(bundle_dirs))
[source] = bundle_dirs
# These get copied into dist/bin without the path, so "root/a/b/c" -> "dist/bin/c".
paths_no_keep_path = ('Contents/MacOS', [
'crashreporter.app/Contents/MacOS/crashreporter',
'firefox',
'firefox-bin',
'libfreebl3.dylib',
'liblgpllibs.dylib',
# 'liblogalloc.dylib',
'libmozglue.dylib',
'libnss3.dylib',
'libnssckbi.dylib',
'libnssdbm3.dylib',
'libplugin_child_interpose.dylib',
# 'libreplace_jemalloc.dylib',
# 'libreplace_malloc.dylib',
'libmozavutil.dylib',
'libmozavcodec.dylib',
'libsoftokn3.dylib',
'plugin-container.app/Contents/MacOS/plugin-container',
'updater.app/Contents/MacOS/org.mozilla.updater',
# 'xpcshell',
'XUL',
])
# These get copied into dist/bin with the path, so "root/a/b/c" -> "dist/bin/a/b/c".
paths_keep_path = ('Contents/Resources', [
'browser/components/libbrowsercomps.dylib',
'dependentlibs.list',
# 'firefox',
'gmp-clearkey/0.1/libclearkey.dylib',
# 'gmp-fake/1.0/libfake.dylib',
# 'gmp-fakeopenh264/1.0/libfakeopenh264.dylib',
'**/interfaces.xpt',
])
with JarWriter(file=processed_filename, optimize=False, compress_level=5) as writer:
root, paths = paths_no_keep_path
finder = UnpackFinder(mozpath.join(source, root))
for path in paths:
for p, f in finder.find(path):
self.log(logging.INFO, 'artifact',
{'path': p},
'Adding {path} to processed archive')
destpath = mozpath.join('bin', os.path.basename(p))
writer.add(destpath.encode('utf-8'), f, mode=f.mode)
root, paths = paths_keep_path
finder = UnpackFinder(mozpath.join(source, root))
for path in paths:
for p, f in finder.find(path):
self.log(logging.INFO, 'artifact',
{'path': p},
'Adding {path} to processed archive')
destpath = mozpath.join('bin', p)
writer.add(destpath.encode('utf-8'), f.open(), mode=f.mode)
finally:
try:
shutil.rmtree(tempdir)
except (OSError, IOError):
self.log(logging.WARN, 'artifact',
{'tempdir': tempdir},
'Unable to delete {tempdir}')
pass
def fat_aar(distdir,
aars_paths,
no_process=False,
no_compatibility_check=False):
if no_process:
print('Not processing architecture-specific artifact Maven AARs.')
return 0
# Map {filename: {fingerprint: [arch1, arch2, ...]}}.
diffs = defaultdict(lambda: defaultdict(list))
missing_arch_prefs = set()
# Collect multi-architecture inputs to the fat AAR.
copier = FileCopier()
for arch, aar_path in aars_paths.items():
# Map old non-architecture-specific path to new architecture-specific path.
old_rewrite_map = {
'greprefs.js':
'{}/greprefs.js'.format(arch),
'defaults/pref/geckoview-prefs.js':
'defaults/pref/{}/geckoview-prefs.js'.format(arch),
}
# Architecture-specific preferences files.
arch_prefs = set(old_rewrite_map.values())
missing_arch_prefs |= set(arch_prefs)
jar_finder = JarFinder(aar_path, JarReader(aar_path))
for path, fileobj in UnpackFinder(jar_finder):
# Native libraries go straight through.
if mozpath.match(path, 'jni/**'):
copier.add(path, fileobj)
elif path in arch_prefs:
copier.add(path, fileobj)
elif path in ('classes.jar', 'annotations.zip'):
# annotations.zip differs due to timestamps, but the contents should not.
# `JarReader` fails on the non-standard `classes.jar` produced by Gradle/aapt,
# and it's not worth working around, so we use Python's zip functionality
# instead.
z = ZipFile(BytesIO(fileobj.open().read()))
for r in z.namelist():
fingerprint = sha1(z.open(r).read()).hexdigest()
diffs['{}!/{}'.format(path, r)][fingerprint].append(arch)
else:
fingerprint = sha1(fileobj.open().read()).hexdigest()
# There's no need to distinguish `target.maven.zip` from `assets/omni.ja` here,
# since in practice they will never overlap.
diffs[path][fingerprint].append(arch)
missing_arch_prefs.discard(path)
# Some differences are allowed across the architecture-specific AARs. We could allow-list
# the actual content, but it's not necessary right now.
allow_pattern_list = {
'AndroidManifest.xml', # Min SDK version is different for 32- and 64-bit builds.
'classes.jar!/org/mozilla/gecko/util/HardwareUtils.class', # Min SDK as well.
'classes.jar!/org/mozilla/geckoview/BuildConfig.class',
# Each input captures its CPU architecture.
'chrome/toolkit/content/global/buildconfig.html',
# Bug 1556162: localized resources are not deterministic across
# per-architecture builds triggered from the same push.
'**/*.ftl',
'**/*.dtd',
'**/*.properties',
}
not_allowed = OrderedDict()
def format_diffs(ds):
# Like ' armeabi-v7a, arm64-v8a -> XXX\n x86, x86_64 -> YYY'.
return '\n'.join(
sorted(' {archs} -> {fingerprint}'.format(
archs=', '.join(sorted(archs)), fingerprint=fingerprint)
for fingerprint, archs in ds.iteritems()))
for p, ds in sorted(diffs.iteritems()):
if len(ds) <= 1:
# Only one hash across all inputs: roll on.
continue
if any(mozpath.match(p, pat) for pat in allow_pattern_list):
print(
'Allowed: Path "{path}" has architecture-specific versions:\n{ds_repr}'
.format(path=p, ds_repr=format_diffs(ds)))
continue
not_allowed[p] = ds
for p, ds in not_allowed.iteritems():
print(
'Disallowed: Path "{path}" has architecture-specific versions:\n{ds_repr}'
.format(path=p, ds_repr=format_diffs(ds)))
for missing in sorted(missing_arch_prefs):
print(
'Disallowed: Inputs missing expected architecture-specific input: {missing}'
.format(missing=missing))
if not no_compatibility_check and (missing_arch_prefs or not_allowed):
return 1
output_dir = mozpath.join(distdir, 'output')
copier.copy(output_dir)
return 0
def process_package_artifact(self, filename, processed_filename):
tempdir = tempfile.mkdtemp()
oldcwd = os.getcwd()
try:
self.log(logging.INFO, 'artifact',
{'tempdir': tempdir},
'Unpacking DMG into {tempdir}')
if self._substs['HOST_OS_ARCH'] == 'Linux':
# This is a cross build, use hfsplus and dmg tools to extract the dmg.
os.chdir(tempdir)
with open(os.devnull, 'wb') as devnull:
subprocess.check_call([
self._substs['DMG_TOOL'],
'extract',
filename,
'extracted_img',
], stdout=devnull)
subprocess.check_call([
self._substs['HFS_TOOL'],
'extracted_img',
'extractall'
], stdout=devnull)
else:
mozinstall.install(filename, tempdir)
bundle_dirs = glob.glob(mozpath.join(tempdir, '*.app'))
if len(bundle_dirs) != 1:
raise ValueError('Expected one source bundle, found: {}'.format(bundle_dirs))
[source] = bundle_dirs
# These get copied into dist/bin without the path, so "root/a/b/c" -> "dist/bin/c".
paths_no_keep_path = ('Contents/MacOS', [
'crashreporter.app/Contents/MacOS/crashreporter',
'firefox',
'firefox-bin',
'libfreebl3.dylib',
'liblgpllibs.dylib',
# 'liblogalloc.dylib',
'libmozglue.dylib',
'libnss3.dylib',
'libnssckbi.dylib',
'libnssdbm3.dylib',
'libplugin_child_interpose.dylib',
# 'libreplace_jemalloc.dylib',
# 'libreplace_malloc.dylib',
'libmozavutil.dylib',
'libmozavcodec.dylib',
'libsoftokn3.dylib',
'pingsender',
'plugin-container.app/Contents/MacOS/plugin-container',
'updater.app/Contents/MacOS/org.mozilla.updater',
# 'xpcshell',
'XUL',
])
# These get copied into dist/bin with the path, so "root/a/b/c" -> "dist/bin/a/b/c".
paths_keep_path = [
('Contents/MacOS', [
'crashreporter.app/Contents/MacOS/minidump-analyzer',
]),
('Contents/Resources', [
'browser/components/libbrowsercomps.dylib',
'dependentlibs.list',
# 'firefox',
'gmp-clearkey/0.1/libclearkey.dylib',
# 'gmp-fake/1.0/libfake.dylib',
# 'gmp-fakeopenh264/1.0/libfakeopenh264.dylib',
'**/interfaces.xpt',
]),
]
with JarWriter(file=processed_filename, optimize=False, compress_level=5) as writer:
root, paths = paths_no_keep_path
finder = UnpackFinder(mozpath.join(source, root))
for path in paths:
for p, f in finder.find(path):
self.log(logging.INFO, 'artifact',
{'path': p},
'Adding {path} to processed archive')
destpath = mozpath.join('bin', os.path.basename(p))
writer.add(destpath.encode('utf-8'), f, mode=f.mode)
for root, paths in paths_keep_path:
finder = UnpackFinder(mozpath.join(source, root))
for path in paths:
for p, f in finder.find(path):
self.log(logging.INFO, 'artifact',
{'path': p},
'Adding {path} to processed archive')
destpath = mozpath.join('bin', p)
writer.add(destpath.encode('utf-8'), f.open(), mode=f.mode)
finally:
os.chdir(oldcwd)
try:
shutil.rmtree(tempdir)
except (OSError, IOError):
self.log(logging.WARN, 'artifact',
{'tempdir': tempdir},
'Unable to delete {tempdir}')
pass
def repack(source, l10n):
finder = UnpackFinder(source)
l10n_finder = UnpackFinder(l10n)
copier = FileCopier()
if finder.kind == 'flat':
formatter = FlatFormatter(copier)
elif finder.kind == 'jar':
formatter = JarFormatter(copier, optimize=finder.optimizedjars)
elif finder.kind == 'omni':
formatter = OmniJarFormatter(copier, finder.omnijar,
optimize=finder.optimizedjars)
# Read all manifest entries from the packaged directory.
manifests = dict((p, m) for p, m in finder.find('**/*.manifest')
if is_manifest(p))
assert all(isinstance(m, ManifestFile) for m in manifests.itervalues())
entries = [e for m in manifests.itervalues() for e in m if e.localized]
# Find unique locales used in these manifest entries.
locales = list(set(e.id for e in entries if isinstance(e, ManifestLocale)))
# Find all paths whose manifest are included by no other manifest.
includes = set(mozpack.path.join(e.base, e.relpath)
for m in manifests.itervalues()
for e in m if isinstance(e, Manifest))
bases = [mozpack.path.dirname(p) for p in set(manifests.keys()) - includes]
# Read all manifest entries from the langpack directory.
manifests = [m for p, m in l10n_finder.find('**/*.manifest')
if is_manifest(p)]
assert all(isinstance(m, ManifestFile) for m in manifests)
l10n_entries = [e for m in manifests for e in m if e.localized]
# Find unique locales used in these manifest entries.
l10n_locales = list(set(e.id for e in l10n_entries
if isinstance(e, ManifestLocale)))
# The code further below assumes there's only one locale replaced with
# another one.
if len(locales) > 1 or len(l10n_locales) > 1:
errors.fatal("Multiple locales aren't supported")
locale = locales[0]
l10n_locale = l10n_locales[0]
# For each base directory, store what path a locale chrome package name
# corresponds to.
# e.g., for the following entry under app/chrome:
# locale foo en-US path/to/files
# keep track that the locale path for foo in app is
# app/chrome/path/to/files.
l10n_paths = {}
for e in l10n_entries:
if isinstance(e, ManifestChrome):
base = mozpack.path.basedir(e.path, bases)
if not base in l10n_paths:
l10n_paths[base] = {}
l10n_paths[base][e.name] = e.path
# For chrome and non chrome files or directories, store what langpack path
# corresponds to a package path.
paths = dict((e.path,
l10n_paths[mozpack.path.basedir(e.path, bases)][e.name])
for e in entries if isinstance(e, ManifestEntryWithRelPath))
for path in NON_CHROME:
for p, f in l10n_finder.find(path):
paths[p] = p
# Create a new package, with non localized bits coming from the original
# package, and localized bits coming from the langpack.
packager = SimplePackager(formatter)
for p, f in finder:
if is_manifest(p):
# Remove localized manifest entries.
for e in [e for e in f if e.localized]:
f.remove(e)
base = mozpack.path.basedir(p, paths.keys())
if base:
# If the path is one that needs a locale replacement, use the
# corresponding file from the langpack.
subpath = mozpack.path.relpath(p, base)
path = mozpack.path.normpath(mozpack.path.join(paths[base],
subpath))
files = [f for p, f in l10n_finder.find(path)]
if len(files) == 0 and base in NON_CHROME:
path = path.replace(locale, l10n_locale)
files = [f for p, f in l10n_finder.find(path)]
if len(files) == 0:
if not base in NON_CHROME:
errors.error("Missing file: %s" % os.path.join(l10n, path))
else:
packager.add(path, files[0])
else:
packager.add(p, f)
# Add localized manifest entries from the langpack.
l10n_manifests = []
for base in set(e.base for e in l10n_entries):
m = ManifestFile(base)
for e in l10n_entries:
if e.base == base:
m.add(e)
path = mozpack.path.join(base, 'chrome.%s.manifest' % l10n_locale)
l10n_manifests.append((path, m))
bases = packager.get_bases()
for path, m in l10n_manifests:
base = mozpack.path.basedir(path, bases)
packager.add(path, m)
# Add a "manifest $path" entry in the top manifest under that base.
m = ManifestFile(base)
m.add(Manifest(base, mozpack.path.relpath(path, base)))
packager.add(mozpack.path.join(base, 'chrome.manifest'), m)
packager.close()
# Add any remaining non chrome files.
for base in NON_CHROME:
for p, f in l10n_finder.find(base):
if not formatter.contains(p):
formatter.add(p, f)
# Transplant jar preloading information.
for path, log in finder.jarlogs.iteritems():
assert isinstance(copier[path], Jarrer)
copier[path].preload([l.replace(locale, l10n_locale) for l in log])
copier.copy(source)
generate_precomplete(source)
def repackage_msix(
dir_or_package,
channel=None,
branding=None,
template=None,
distribution_dirs=[],
locale_allowlist=set(),
version=None,
vendor=None,
displayname=None,
app_name="firefox",
identity=None,
publisher=None,
publisher_display_name="Mozilla Corporation",
arch=None,
output=None,
force=False,
log=None,
verbose=False,
makeappx=None,
):
if not channel:
raise Exception("channel is required")
if channel not in ["official", "beta", "aurora", "nightly", "unofficial"]:
raise Exception("channel is unrecognized: {}".format(channel))
if not branding:
raise Exception("branding dir is required")
if not os.path.isdir(branding):
raise Exception("branding dir {} does not exist".format(branding))
# TODO: maybe we can fish this from the package directly? Maybe from a DLL,
# maybe from application.ini?
if arch is None or arch not in _MSIX_ARCH.keys():
raise Exception(
"arch name must be provided and one of {}.".format(_MSIX_ARCH.keys())
)
if not os.path.exists(dir_or_package):
raise Exception("{} does not exist".format(dir_or_package))
if (
os.path.isfile(dir_or_package)
and os.path.splitext(dir_or_package)[1] == ".msix"
):
# The convention is $MOZBUILD_STATE_PATH/cache/$FEATURE.
msix_dir = mozpath.normsep(
mozpath.join(
get_state_dir(),
"cache",
"mach-msix",
"msix-unpack",
)
)
if os.path.exists(msix_dir):
shutil.rmtree(msix_dir)
ensureParentDir(msix_dir)
dir_or_package = unpack_msix(dir_or_package, msix_dir, log=log, verbose=verbose)
log(
logging.INFO,
"msix",
{
"input": dir_or_package,
},
"Adding files from '{input}'",
)
if os.path.isdir(dir_or_package):
finder = FileFinder(dir_or_package)
else:
finder = JarFinder(dir_or_package, JarReader(dir_or_package))
values = get_application_ini_values(
finder,
dict(section="App", value="CodeName", fallback="Name"),
dict(section="App", value="Vendor"),
)
first = next(values)
displayname = displayname or "Mozilla {}".format(first)
second = next(values)
vendor = vendor or second
# For `AppConstants.jsm` and `brand.properties`, which are in the omnijar in packaged builds.
# The nested langpack XPI files can't be read by `mozjar.py`.
unpack_finder = UnpackFinder(finder, unpack_xpi=False)
if not version:
values = get_appconstants_jsm_values(
unpack_finder, "MOZ_APP_VERSION_DISPLAY", "MOZ_BUILDID"
)
display_version = next(values)
buildid = next(values)
version = get_embedded_version(display_version, buildid)
log(
logging.INFO,
"msix",
{
"version": version,
"display_version": display_version,
"buildid": buildid,
},
"AppConstants.jsm display version is '{display_version}' and build ID is '{buildid}':"
+ " embedded version will be '{version}'",
)
# TODO: Bug 1721922: localize this description via Fluent.
lines = []
for _, f in unpack_finder.find("**/chrome/en-US/locale/branding/brand.properties"):
lines.extend(
line
for line in f.open().read().decode("utf-8").splitlines()
if "brandFullName" in line
)
(brandFullName,) = lines # We expect exactly one definition.
_, _, brandFullName = brandFullName.partition("=")
brandFullName = brandFullName.strip()
# We don't have a build at repackage-time to gives us this value, and the
# source of truth is a branding-specific `configure.sh` shell script that we
# can't easily evaluate completely here. Instead, we take the last value
# from `configure.sh`.
lines = [
line
for line in open(mozpath.join(branding, "configure.sh")).readlines()
if "MOZ_IGECKOBACKCHANNEL_IID" in line
]
MOZ_IGECKOBACKCHANNEL_IID = lines[-1]
_, _, MOZ_IGECKOBACKCHANNEL_IID = MOZ_IGECKOBACKCHANNEL_IID.partition("=")
MOZ_IGECKOBACKCHANNEL_IID = MOZ_IGECKOBACKCHANNEL_IID.strip()
if MOZ_IGECKOBACKCHANNEL_IID.startswith(('"', "'")):
MOZ_IGECKOBACKCHANNEL_IID = MOZ_IGECKOBACKCHANNEL_IID[1:-1]
# The convention is $MOZBUILD_STATE_PATH/cache/$FEATURE.
output_dir = mozpath.normsep(
mozpath.join(
get_state_dir(), "cache", "mach-msix", "msix-temp-{}".format(channel)
)
)
if channel == "beta":
# Release (official) and Beta share branding. Differentiate Beta a little bit.
displayname += " Beta"
brandFullName += " Beta"
# Like 'Firefox Package Root', 'Firefox Nightly Package Root', 'Firefox Beta
# Package Root'. This is `BrandFullName` in the installer, and we want to
# be close but to not match. By not matching, we hope to prevent confusion
# and/or errors between regularly installed builds and App Package builds.
instdir = "{} Package Root".format(displayname)
# The standard package name is like "CompanyNoSpaces.ProductNoSpaces".
identity = identity or "{}.{}".format(vendor, displayname).replace(" ", "")
# We might want to include the publisher ID hash here. I.e.,
# "__{publisherID}". My locally produced MSIX was named like
# `Mozilla.MozillaFirefoxNightly_89.0.0.0_x64__4gf61r4q480j0`, suggesting also a
# missing field, but it's not necessary, since this is just an output file name.
package_output_name = "{identity}_{version}_{arch}".format(
identity=identity, version=version, arch=_MSIX_ARCH[arch]
)
# The convention is $MOZBUILD_STATE_PATH/cache/$FEATURE.
default_output = mozpath.normsep(
mozpath.join(
get_state_dir(), "cache", "mach-msix", "{}.msix".format(package_output_name)
)
)
output = output or default_output
log(logging.INFO, "msix", {"output": output}, "Repackaging to: {output}")
m = InstallManifest()
m.add_copy(mozpath.join(template, "Resources.pri"), "Resources.pri")
m.add_pattern_copy(mozpath.join(branding, "msix", "Assets"), "**", "Assets")
m.add_pattern_copy(mozpath.join(template, "VFS"), "**", "VFS")
copier = FileCopier()
# TODO: Bug 1710147: filter out MSVCRT files and use a dependency instead.
for p, f in finder:
if not os.path.isdir(dir_or_package):
# In archived builds, `p` is like "firefox/firefox.exe"; we want just "firefox.exe".
pp = os.path.relpath(p, "firefox")
else:
# In local builds and unpacked MSIX directories, `p` is like "firefox.exe" already.
pp = p
if pp.startswith("distribution"):
# Treat any existing distribution as a distribution directory,
# potentially with language packs. This makes it easy to repack
# unpacked MSIXes.
distribution_dir = mozpath.join(dir_or_package, "distribution")
if distribution_dir not in distribution_dirs:
distribution_dirs.append(distribution_dir)
continue
copier.add(mozpath.normsep(mozpath.join("VFS", "ProgramFiles", instdir, pp)), f)
# Locales to declare as supported in `AppxManifest.xml`.
locales = set(["en-US"])
for distribution_dir in [
mozpath.join(template, "distribution")
] + distribution_dirs:
log(
logging.INFO,
"msix",
{"dir": distribution_dir},
"Adding distribution files from {dir}",
)
# In automation, we have no easy way to remap the names of artifacts fetched from dependent
# tasks. In particular, langpacks will be named like `target.langpack.xpi`. The fetch
# tasks do allow us to put them in a per-locale directory, so that the entire set can be
# fetched. Here we remap the names.
finder = FileFinder(distribution_dir)
for p, f in finder:
locale = None
if os.path.basename(p) == "target.langpack.xpi":
# Turn "/path/to/LOCALE/target.langpack.xpi" into "LOCALE". This is how langpacks
# are presented in CI.
base, locale = os.path.split(os.path.dirname(p))
# Like "locale-LOCALE/[email protected]". This is what AMO
# serves and how flatpak builds name langpacks, but not how snap builds name
# langpacks. I can't explain the discrepancy.
dest = mozpath.normsep(
mozpath.join(
base,
f"locale-{locale}",
f"langpack-{locale}@firefox.mozilla.org.xpi",
)
)
log(
logging.DEBUG,
"msix",
{"path": p, "dest": dest},
"Renaming langpack {path} to {dest}",
)
elif os.path.basename(p).startswith("langpack-"):
# Turn "/path/to/[email protected]" into "LOCALE". This is
# how langpacks are presented from an unpacked MSIX.
_, _, locale = os.path.basename(p).partition("langpack-")
locale, _, _ = locale.partition("@")
dest = p
else:
dest = p
if locale:
locale = locale.strip().lower()
locales.add(locale)
log(
logging.DEBUG,
"msix",
{"locale": locale, "dest": dest},
"Distributing locale '{locale}' from {dest}",
)
dest = mozpath.normsep(
mozpath.join("VFS", "ProgramFiles", instdir, "distribution", dest)
)
if copier.contains(dest):
log(
logging.INFO,
"msix",
{"dest": dest, "path": mozpath.join(finder.base, p)},
"Skipping duplicate: {dest} from {path}",
)
continue
log(
logging.DEBUG,
"msix",
{"dest": dest, "path": mozpath.join(finder.base, p)},
"Adding distribution path: {dest} from {path}",
)
copier.add(
dest,
f,
)
locales.remove("en-US")
# Windows MSIX packages support a finite set of locales: see
# https://docs.microsoft.com/en-us/windows/uwp/publish/supported-languages, which is encoded in
# https://searchfox.org/mozilla-central/source/browser/installer/windows/msix/msix-all-locales.
# We distribute all of the langpacks supported by the release channel in our MSIX, which is
# encoded in https://searchfox.org/mozilla-central/source/browser/locales/all-locales. But we
# only advertise support in the App manifest for the intersection of that set and the set of
# supported locales.
#
# We distribute all langpacks to avoid the following issue. Suppose a user manually installs a
# langpack that is not supported by Windows, and then updates the installed MSIX package. MSIX
# package upgrades are essentially paveover installs, so there is no opportunity for Firefox to
# update the langpack before the update. But, since all langpacks are bundled with the MSIX,
# that langpack will be up-to-date, preventing one class of YSOD.
unadvertised = set()
if locale_allowlist:
unadvertised = locales - locale_allowlist
locales = locales & locale_allowlist
for locale in sorted(unadvertised):
log(
logging.INFO,
"msix",
{"locale": locale},
"Not advertising distributed locale '{locale}' that is not recognized by Windows",
)
locales = ["en-US"] + list(sorted(locales))
resource_language_list = "\n".join(
f' <Resource Language="{locale}" />' for locale in sorted(locales)
)
defines = {
"APPX_ARCH": _MSIX_ARCH[arch],
"APPX_DISPLAYNAME": brandFullName,
"APPX_DESCRIPTION": brandFullName,
# Like 'Mozilla.MozillaFirefox', 'Mozilla.MozillaFirefoxBeta', or
# 'Mozilla.MozillaFirefoxNightly'.
"APPX_IDENTITY": identity,
# Like 'Firefox Package Root', 'Firefox Nightly Package Root', 'Firefox
# Beta Package Root'. See above.
"APPX_INSTDIR": instdir,
# Like 'Firefox%20Package%20Root'.
"APPX_INSTDIR_QUOTED": urllib.parse.quote(instdir),
"APPX_PUBLISHER": publisher,
"APPX_PUBLISHER_DISPLAY_NAME": publisher_display_name,
"APPX_RESOURCE_LANGUAGE_LIST": resource_language_list,
"APPX_VERSION": version,
"MOZ_APP_DISPLAYNAME": displayname,
"MOZ_APP_NAME": app_name,
"MOZ_IGECKOBACKCHANNEL_IID": MOZ_IGECKOBACKCHANNEL_IID,
}
m.add_preprocess(
mozpath.join(template, "AppxManifest.xml.in"),
"AppxManifest.xml",
[],
defines=defines,
marker="<!-- #", # So that we can have well-formed XML.
)
m.populate_registry(copier)
output_dir = mozpath.abspath(output_dir)
ensureParentDir(output_dir)
start = time.time()
result = copier.copy(
output_dir, remove_empty_directories=True, skip_if_older=not force
)
if log:
log_copy_result(log, time.time() - start, output_dir, result)
if verbose:
# Dump AppxManifest.xml contents for ease of debugging.
log(logging.DEBUG, "msix", {}, "AppxManifest.xml")
log(logging.DEBUG, "msix", {}, ">>>")
for line in open(mozpath.join(output_dir, "AppxManifest.xml")).readlines():
log(logging.DEBUG, "msix", {}, line[:-1]) # Drop trailing line terminator.
log(logging.DEBUG, "msix", {}, "<<<")
if not makeappx:
makeappx = find_sdk_tool("makeappx.exe", log=log)
if not makeappx:
raise ValueError(
"makeappx is required; " "set MAKEAPPX or WINDOWSSDKDIR or PATH"
)
# `makeappx.exe` supports both slash and hyphen style arguments; `makemsix`
# supports only hyphen style. `makeappx.exe` allows to overwrite and to
# provide more feedback, so we prefer invoking with these flags. This will
# also accommodate `wine makeappx.exe`.
stdout = subprocess.run(
[makeappx], check=False, capture_output=True, universal_newlines=True
).stdout
is_makeappx = "MakeAppx Tool" in stdout
if is_makeappx:
args = [makeappx, "pack", "/d", output_dir, "/p", output, "/overwrite"]
else:
args = [makeappx, "pack", "-d", output_dir, "-p", output]
if verbose and is_makeappx:
args.append("/verbose")
joined = " ".join(shlex_quote(arg) for arg in args)
log(logging.INFO, "msix", {"args": args, "joined": joined}, "Invoking: {joined}")
sys.stdout.flush() # Otherwise the subprocess output can be interleaved.
if verbose:
subprocess.check_call(args, universal_newlines=True)
else:
# Suppress output unless we fail.
try:
subprocess.check_output(args, universal_newlines=True)
except subprocess.CalledProcessError as e:
sys.stderr.write(e.output)
raise
return output
