def repackage_mar(topsrcdir, package, mar, output):
if not zipfile.is_zipfile(package):
raise Exception("Package file %s is not a valid .zip file." % package)
ensureParentDir(output)
tmpdir = tempfile.mkdtemp()
try:
z = zipfile.ZipFile(package)
z.extractall(tmpdir)
filelist = z.namelist()
z.close()
# Make sure the .zip file just contains a directory like 'firefox/' at
# the top, and find out what it is called.
toplevel_dirs = set([mozpath.split(f)[0] for f in filelist])
if len(toplevel_dirs) != 1:
raise Exception("Package file is expected to have a single top-level directory (eg: 'firefox'), not: %s" % toplevel_dirs)
ffxdir = mozpath.join(tmpdir, toplevel_dirs.pop())
make_full_update = mozpath.join(topsrcdir, 'tools/update-packaging/make_full_update.sh')
env = os.environ.copy()
env['MOZ_FULL_PRODUCT_VERSION'] = get_application_ini_value(tmpdir, 'App', 'Version')
env['MAR'] = mozpath.normpath(mar)
cmd = [make_full_update, output, ffxdir]
if sys.platform == 'win32':
# make_full_update.sh is a bash script, and Windows needs to
# explicitly call out the shell to execute the script from Python.
cmd.insert(0, env['MOZILLABUILD'] + '/msys/bin/bash.exe')
subprocess.check_call(cmd, env=env)
finally:
shutil.rmtree(tmpdir)
def canonicalize(url):
'''
The jar path is stored in a MOZ_JAR_LOG_FILE log as a url. This method
returns a unique value corresponding to such urls.
- file:///{path} becomes {path}
- jar:file:///{path}!/{subpath} becomes ({path}, {subpath})
- jar:jar:file:///{path}!/{subpath}!/{subpath2} becomes
({path}, {subpath}, {subpath2})
'''
if not isinstance(url, ParseResult):
# Assume that if it doesn't start with jar: or file:, it's a path.
if not url.startswith(('jar:', 'file:')):
url = 'file:///' + os.path.abspath(url)
url = urlparse(url)
assert url.scheme
assert url.scheme in ('jar', 'file')
if url.scheme == 'jar':
path = JarLog.canonicalize(url.path)
if isinstance(path, tuple):
return path[:-1] + tuple(path[-1].split('!/', 1))
return tuple(path.split('!/', 1))
if url.scheme == 'file':
assert os.path.isabs(url.path)
path = url.path
# On Windows, url.path will be /drive:/path ; on Unix systems,
# /path. As we want drive:/path instead of /drive:/path on Windows,
# remove the leading /.
if os.path.isabs(path[1:]):
path = path[1:]
path = os.path.realpath(path)
return mozpack.path.normsep(os.path.normcase(path))
def repackage_mar(
topsrcdir, package, mar, output, mar_format="lzma", arch=None, mar_channel_id=None
):
if not zipfile.is_zipfile(package) and not tarfile.is_tarfile(package):
raise Exception("Package file %s is not a valid .zip or .tar file." % package)
if arch and arch not in _BCJ_OPTIONS:
raise Exception("Unknown architecture {}, available architectures: {}".format(
arch, list(_BCJ_OPTIONS.keys())))
ensureParentDir(output)
tmpdir = tempfile.mkdtemp()
try:
if tarfile.is_tarfile(package):
z = tarfile.open(package)
z.extractall(tmpdir)
filelist = z.getnames()
z.close()
else:
z = zipfile.ZipFile(package)
z.extractall(tmpdir)
filelist = z.namelist()
z.close()
toplevel_dirs = set([mozpath.split(f)[0] for f in filelist])
excluded_stuff = set([' ', '.background', '.DS_Store', '.VolumeIcon.icns'])
toplevel_dirs = toplevel_dirs - excluded_stuff
# Make sure the .zip file just contains a directory like 'firefox/' at
# the top, and find out what it is called.
if len(toplevel_dirs) != 1:
raise Exception("Package file is expected to have a single top-level directory"
"(eg: 'firefox'), not: %s" % toplevel_dirs)
ffxdir = mozpath.join(tmpdir, toplevel_dirs.pop())
make_full_update = mozpath.join(topsrcdir, 'tools/update-packaging/make_full_update.sh')
env = os.environ.copy()
env['MOZ_PRODUCT_VERSION'] = get_application_ini_value(tmpdir, 'App', 'Version')
env['MAR'] = mozpath.normpath(mar)
if arch:
env['BCJ_OPTIONS'] = ' '.join(_BCJ_OPTIONS[arch])
if mar_format == 'bz2':
env['MAR_OLD_FORMAT'] = '1'
if mar_channel_id:
env['MAR_CHANNEL_ID'] = mar_channel_id
# The Windows build systems have xz installed but it isn't in the path
# like it is on Linux and Mac OS X so just use the XZ env var so the mar
# generation scripts can find it.
xz_path = mozpath.join(topsrcdir, 'xz/xz.exe')
if os.path.exists(xz_path):
env['XZ'] = mozpath.normpath(xz_path)
cmd = [make_full_update, output, ffxdir]
if sys.platform == 'win32':
# make_full_update.sh is a bash script, and Windows needs to
# explicitly call out the shell to execute the script from Python.
cmd.insert(0, env['MOZILLABUILD'] + '/msys/bin/bash.exe')
subprocess.check_call(cmd, env=ensure_subprocess_env(env))
finally:
shutil.rmtree(tmpdir)
def _add_features(self, target, path):
path_parts = mozpath.split(path)
if all([
target == 'dist/bin/browser', path_parts[0] == 'features',
len(path_parts) > 1
]):
self._built_in_addons.add(path_parts[1])
def __call__(self, result):
paths = set(
list(result.issues.keys()) +
list(result.suppressed_warnings.keys()))
commonprefix = mozpath.commonprefix(
[mozpath.abspath(p) for p in paths])
commonprefix = commonprefix.rsplit("/", 1)[0] + "/"
summary = defaultdict(lambda: [0, 0])
for path in paths:
abspath = mozpath.abspath(path)
assert abspath.startswith(commonprefix)
if abspath != commonprefix:
parts = mozpath.split(mozpath.relpath(
abspath, commonprefix))[:self.depth]
abspath = mozpath.join(commonprefix, *parts)
summary[abspath][0] += len(
[r for r in result.issues[path] if r.level == "error"])
summary[abspath][1] += len(
[r for r in result.issues[path] if r.level == "warning"])
summary[abspath][1] += result.suppressed_warnings[path]
msg = []
for path, (errors, warnings) in sorted(summary.items()):
warning_str = (", {}".format(pluralize("warning", warnings))
if warnings else "")
msg.append("{}: {}{}".format(path, pluralize("error", errors),
warning_str))
return "\n".join(msg)
def is_resource(self, path, base=None):
'''
Return whether the given path corresponds to a resource to be put in an
omnijar archive.
'''
if base is None:
base = self._get_base(path)
path = mozpath.relpath(path, base)
if any(mozpath.match(path, p.replace('*', '**'))
for p in self._non_resources):
return False
path = mozpath.split(path)
if path[0] == 'chrome':
return len(path) == 1 or path[1] != 'icons'
if path[0] == 'components':
return path[-1].endswith(('.js', '.xpt'))
if path[0] == 'res':
return len(path) == 1 or \
(path[1] != 'cursors' and path[1] != 'MainMenu.nib')
if path[0] == 'defaults':
return len(path) != 3 or \
not (path[2] == 'channel-prefs.js' and
path[1] in ['pref', 'preferences'])
return path[0] in [
'modules',
'greprefs.js',
'hyphenation',
'update.locale',
] or path[0] in STARTUP_CACHE_PATHS
def is_resource(self, path):
"""
Return whether the given path corresponds to a resource to be put in an
omnijar archive.
"""
if any(
mozpath.match(path, p.replace("*", "**"))
for p in self._non_resources):
return False
path = mozpath.split(path)
if path[0] == "chrome":
return len(path) == 1 or path[1] != "icons"
if path[0] == "components":
return path[-1].endswith((".js", ".xpt"))
if path[0] == "res":
return len(path) == 1 or (path[1] != "cursors"
and path[1] != "touchbar"
and path[1] != "MainMenu.nib")
if path[0] == "defaults":
return len(path) != 3 or not (path[2] == "channel-prefs.js" and
path[1] in ["pref", "preferences"])
if len(path) <= 2 and path[-1] == "greprefs.js":
# Accommodate `greprefs.js` and `$ANDROID_CPU_ARCH/greprefs.js`.
return True
return path[0] in [
"modules",
"actors",
"dictionaries",
"hyphenation",
"localization",
"update.locale",
"contentaccessible",
]
def is_resource(self, path):
'''
Return whether the given path corresponds to a resource to be put in an
omnijar archive.
'''
if any(
mozpath.match(path, p.replace('*', '**'))
for p in self._non_resources):
return False
path = mozpath.split(path)
if path[0] == 'chrome':
return len(path) == 1 or path[1] != 'icons'
if path[0] == 'components':
return path[-1].endswith(('.js', '.xpt'))
if path[0] == 'res':
return len(path) == 1 or \
(path[1] != 'cursors' and path[1] != 'MainMenu.nib')
if path[0] == 'defaults':
return len(path) != 3 or \
not (path[2] == 'channel-prefs.js' and
path[1] in ['pref', 'preferences'])
return path[0] in [
'modules',
'greprefs.js',
'hyphenation',
'update.locale',
] or path[0] in STARTUP_CACHE_PATHS
def __call__(self, result):
paths = set(result.issues.keys() + result.suppressed_warnings.keys())
commonprefix = mozpath.commonprefix(
[mozpath.abspath(p) for p in paths])
commonprefix = commonprefix.rsplit('/', 1)[0] + '/'
summary = defaultdict(lambda: [0, 0])
for path in paths:
abspath = mozpath.abspath(path)
assert abspath.startswith(commonprefix)
if abspath != commonprefix:
parts = mozpath.split(mozpath.relpath(
abspath, commonprefix))[:self.depth]
abspath = mozpath.join(commonprefix, *parts)
summary[abspath][0] += len(
[r for r in result.issues[path] if r.level == 'error'])
summary[abspath][1] += len(
[r for r in result.issues[path] if r.level == 'warning'])
summary[abspath][1] += result.suppressed_warnings[path]
msg = []
for path, (errors, warnings) in sorted(summary.items()):
warning_str = ", {}".format(pluralize(
'warning', warnings)) if warnings else ''
msg.append('{}: {}{}'.format(path, pluralize('error', errors),
warning_str))
return '\n'.join(msg)
def canonicalize(url):
'''
The jar path is stored in a MOZ_JAR_LOG_FILE log as a url. This method
returns a unique value corresponding to such urls.
- file:///{path} becomes {path}
- jar:file:///{path}!/{subpath} becomes ({path}, {subpath})
- jar:jar:file:///{path}!/{subpath}!/{subpath2} becomes
({path}, {subpath}, {subpath2})
'''
if not isinstance(url, ParseResult):
# Assume that if it doesn't start with jar: or file:, it's a path.
if not url.startswith(('jar:', 'file:')):
url = 'file:///' + os.path.abspath(url)
url = urlparse(url)
assert url.scheme
assert url.scheme in ('jar', 'file')
if url.scheme == 'jar':
path = JarLog.canonicalize(url.path)
if isinstance(path, tuple):
return path[:-1] + tuple(path[-1].split('!/', 1))
return tuple(path.split('!/', 1))
if url.scheme == 'file':
assert os.path.isabs(url.path)
path = url.path
# On Windows, url.path will be /drive:/path ; on Unix systems,
# /path. As we want drive:/path instead of /drive:/path on Windows,
# remove the leading /.
if os.path.isabs(path[1:]):
path = path[1:]
path = os.path.realpath(path)
return mozpack.path.normsep(os.path.normcase(path))
def is_resource(self, path):
'''
Return whether the given path corresponds to a resource to be put in an
omnijar archive.
'''
if any(
mozpath.match(path, p.replace('*', '**'))
for p in self._non_resources):
return False
path = mozpath.split(path)
if path[0] == 'chrome':
return len(path) == 1 or path[1] != 'icons'
if path[0] == 'components':
return path[-1].endswith(('.js', '.xpt'))
if path[0] == 'res':
return len(path) == 1 or \
(path[1] != 'cursors' and
path[1] != 'touchbar' and
path[1] != 'MainMenu.nib')
if path[0] == 'defaults':
return len(path) != 3 or \
not (path[2] == 'channel-prefs.js' and
path[1] in ['pref', 'preferences'])
if len(path) <= 2 and path[-1] == 'greprefs.js':
# Accommodate `greprefs.js` and `$ANDROID_CPU_ARCH/greprefs.js`.
return True
return path[0] in [
'modules',
'actors',
'dictionaries',
'hyphenation',
'localization',
'update.locale',
'contentaccessible',
]
def taskcluster_yml(self):
if path.split(self.root_dir)[-2:] != ['taskcluster', 'ci']:
raise Exception("Not guessing path to `.taskcluster.yml`. "
"Graph config in non-standard location.")
return os.path.join(
os.path.dirname(os.path.dirname(self.root_dir)),
".taskcluster.yml",
)
def _jarize(self, entry, relpath):
"""
Transform a manifest entry in one pointing to chrome data in a jar.
Return the corresponding chrome path and the new entry.
"""
base = entry.base
basepath = mozpath.split(relpath)[0]
chromepath = mozpath.join(base, basepath)
entry = (entry.rebase(chromepath).move(
mozpath.join(base, "jar:%s.jar!" % basepath)).rebase(base))
return chromepath, entry
def unpack_to_registry(source, registry):
'''
Transform a jar chrome or omnijar packaged directory into a flat package.
The given registry is filled with the flat package.
'''
finder = UnpackFinder(source)
packager = SimplePackager(FlatFormatter(registry))
for p, f in finder.find('*'):
if mozpath.split(p)[0] not in STARTUP_CACHE_PATHS:
packager.add(p, f)
packager.close()
def _jarize(self, entry, relpath):
'''
Transform a manifest entry in one pointing to chrome data in a jar.
Return the corresponding chrome path and the new entry.
'''
base = entry.base
basepath = mozpath.split(relpath)[0]
chromepath = mozpath.join(base, basepath)
entry = entry.rebase(chromepath) \
.move(mozpath.join(base, 'jar:%s.jar!' % basepath)) \
.rebase(base)
return chromepath, entry
def unpack_to_registry(source, registry):
'''
Transform a jar chrome or omnijar packaged directory into a flat package.
The given registry is filled with the flat package.
'''
finder = UnpackFinder(source)
packager = SimplePackager(FlatFormatter(registry))
for p, f in finder.find('*'):
if mozpath.split(p)[0] not in STARTUP_CACHE_PATHS:
packager.add(p, f)
packager.close()
def _find(self, pattern):
'''
Actual implementation of FileFinder.find(), dispatching to specialized
member functions depending on what kind of pattern was given.
Note all files with a name starting with a '.' are ignored when
scanning directories, but are not ignored when explicitely requested.
'''
if '*' in pattern:
return self._find_glob('', mozpath.split(pattern))
elif os.path.isdir(os.path.join(self.base, pattern)):
return self._find_dir(pattern)
else:
return self._find_file(pattern)
def _find(self, pattern):
'''
Actual implementation of FileFinder.find(), dispatching to specialized
member functions depending on what kind of pattern was given.
Note all files with a name starting with a '.' are ignored when
scanning directories, but are not ignored when explicitely requested.
'''
if '*' in pattern:
return self._find_glob('', mozpath.split(pattern))
elif os.path.isdir(os.path.join(self.base, pattern)):
return self._find_dir(pattern)
else:
return self._find_file(pattern)
def _find_install_prefix(self, objdir_path):
def _prefix(s):
for p in mozpath.split(s):
if '*' not in p:
yield p + '/'
offset = 0
for leaf in reversed(mozpath.split(objdir_path)):
offset += len(leaf)
if objdir_path[:-offset] in self._install_mapping:
pattern_prefix, is_pp = self._install_mapping[
objdir_path[:-offset]]
full_leaf = objdir_path[len(objdir_path) - offset:]
src_prefix = ''.join(_prefix(pattern_prefix))
self._install_mapping[objdir_path] = (mozpath.join(
src_prefix, full_leaf), is_pp)
offset += 1
def _find_install_prefix(self, objdir_path):
def _prefix(s):
for p in mozpath.split(s):
if '*' not in p:
yield p + '/'
offset = 0
for leaf in reversed(mozpath.split(objdir_path)):
offset += len(leaf)
if objdir_path[:-offset] in self._install_mapping:
pattern_prefix, is_pp = self._install_mapping[objdir_path[:-offset]]
full_leaf = objdir_path[len(objdir_path) - offset:]
src_prefix = ''.join(_prefix(pattern_prefix))
self._install_mapping[objdir_path] = (mozpath.join(src_prefix, full_leaf),
is_pp)
break
offset += 1
def __call__(self, result, **kwargs):
commonprefix = mozpath.commonprefix(
[mozpath.abspath(p) for p in result])
commonprefix = commonprefix.rsplit('/', 1)[0] + '/'
summary = defaultdict(int)
for path, errors in result.iteritems():
path = mozpath.abspath(path)
assert path.startswith(commonprefix)
if path == commonprefix:
summary[path] += len(errors)
continue
parts = mozpath.split(mozpath.relpath(path,
commonprefix))[:self.depth]
path = mozpath.join(commonprefix, *parts)
summary[path] += len(errors)
return '\n'.join(
['{}: {}'.format(k, summary[k]) for k in sorted(summary)])
def generate_pp_info(path, topsrcdir):
with open(path) as fh:
# (start, end) -> (included_source, start)
section_info = dict()
this_section = None
def finish_section(pp_end):
pp_start, inc_source, inc_start = this_section
section_info[str(pp_start) + ',' +
str(pp_end)] = inc_source, inc_start
for count, line in enumerate(fh):
# Regex are quite slow, so bail out early.
if not line.startswith('//@line'):
continue
m = re.match(_line_comment_re, line)
if m:
if this_section:
finish_section(count + 1)
inc_start, inc_source = m.groups()
# Special case to handle $SRCDIR prefixes
src_dir_prefix = '$SRCDIR'
parts = mozpath.split(inc_source)
if parts[0] == src_dir_prefix:
inc_source = mozpath.join(*parts[1:])
else:
inc_source = mozpath.relpath(inc_source, topsrcdir)
pp_start = count + 2
this_section = pp_start, inc_source, int(inc_start)
if this_section:
finish_section(count + 2)
return section_info
def populate_registry(self, registry):
"""Populate a mozpack.copier.FileRegistry instance with data from us.
The caller supplied a FileRegistry instance (or at least something that
conforms to its interface) and that instance is populated with data
from this manifest.
"""
for dest in sorted(self._dests):
entry = self._dests[dest]
install_type = entry[0]
if install_type == self.SYMLINK:
registry.add(dest, AbsoluteSymlinkFile(entry[1]))
continue
if install_type == self.COPY:
registry.add(dest, File(entry[1]))
continue
if install_type in (self.REQUIRED_EXISTS, self.OPTIONAL_EXISTS):
registry.add(
dest, ExistingFile(install_type == self.REQUIRED_EXISTS, self._decode_field_entry(entry[1]))
)
continue
if install_type in (self.PATTERN_SYMLINK, self.PATTERN_COPY):
_, base, pattern, dest = entry
if pattern.startswith("/"):
patterns = mozpath.split(pattern)
pattern = ""
hasPattern = False
for p in patterns:
if "*" in p:
hasPattern = True
if hasPattern:
pattern = mozpath.join(pattern, p)
else:
base = mozpath.join(base, p)
finder = FileFinder(base, find_executables=False)
paths = [f[0] for f in finder.find(pattern)]
if install_type == self.PATTERN_SYMLINK:
cls = AbsoluteSymlinkFile
else:
cls = File
for path in paths:
source = mozpath.join(base, path)
registry.add(mozpath.join(dest, path), cls(source))
continue
if install_type == self.PREPROCESS:
registry.add(
dest,
PreprocessedFile(
self._decode_field_entry(entry[1]),
depfile_path=entry[2],
marker=entry[3],
defines=self._decode_field_entry(entry[4]),
extra_depends=self._source_files,
),
)
continue
raise Exception("Unknown install type defined in manifest: %d" % install_type)
def test_split(self):
self.assertEqual(split(self.SEP.join(("foo", "bar", "baz"))),
["foo", "bar", "baz"])
def _prefix(s):
for p in mozpath.split(s):
if '*' not in p:
yield p + '/'
def _prefix(s):
for p in mozpath.split(s):
if "*" not in p:
yield p + "/"
def itermozbuild(path):
subpath = ''
yield 'moz.build'
for part in mozpath.split(path):
subpath = mozpath.join(subpath, part)
yield mozpath.join(subpath, 'moz.build')
def fake_short_path(path):
if sys.platform.startswith('win'):
return '/'.join(
p.split(' ', 1)[0] + '~1' if ' ' in p else p
for p in mozpath.split(path))
return path
def test_split(self):
self.assertEqual(split(os.path.join('foo', 'bar', 'baz')),
['foo', 'bar', 'baz'])
def test_split(self):
self.assertEqual(split(os.path.join("foo", "bar", "baz")), ["foo", "bar", "baz"])
def fake_short_path(path):
if sys.platform.startswith("win"):
return "/".join(
p.split(" ", 1)[0] + "~1" if " " in p else p
for p in mozpath.split(path))
return path
def test_split(self):
self.assertEqual(split(self.SEP.join(('foo', 'bar', 'baz'))),
['foo', 'bar', 'baz'])
def _prefix(s):
for p in mozpath.split(s):
if '*' not in p:
yield p + '/'
def itermozbuild(path):
subpath = ''
yield 'moz.build'
for part in mozpath.split(path):
subpath = mozpath.join(subpath, part)
yield mozpath.join(subpath, 'moz.build')
def _prefix(s):
for p in mozpath.split(s):
if "*" not in p:
yield p + "/"
def fake_short_path(path):
if sys.platform.startswith('win'):
return '/'.join(p.split(' ', 1)[0] + '~1' if ' 'in p else p
for p in mozpath.split(path))
return path
def test_split(self):
self.assertEqual(split(os.path.join('foo', 'bar', 'baz')),
['foo', 'bar', 'baz'])
def is_native(path):
path = os.path.abspath(path)
return platform.machine() in mozpath.split(path)
def is_native(path):
path = os.path.abspath(path)
return platform.machine() in mozpath.split(path)