def __init__(self, name, destdir='', xz_compress=False):
if name.find(' ') > 0:
errors.fatal('Malformed manifest: space in component name "%s"' %
name)
self._name = name
self._destdir = destdir
self._xz_compress = xz_compress
def _add_manifest_file(self, path, file):
'''
Add the given BaseFile with manifest file contents with the given path.
'''
self._manifests.add(path)
base = ''
if hasattr(file, 'path'):
# Find the directory the given path is relative to.
b = mozpath.normsep(file.path)
if b.endswith('/' + path) or b == path:
base = os.path.normpath(b[:-len(path)])
for e in parse_manifest(base, path, file.open()):
# ManifestResources need to be given after ManifestChrome, so just
# put all ManifestChrome in a separate queue to make them first.
if isinstance(e, ManifestChrome):
# e.move(e.base) just returns a clone of the entry.
self._chrome_queue.append(self.formatter.add_manifest,
e.move(e.base))
elif not isinstance(e, (Manifest, ManifestInterfaces)):
self._queue.append(self.formatter.add_manifest, e.move(e.base))
# If a binary component is added to an addon, prevent the addon
# from being packed.
if isinstance(e, ManifestBinaryComponent):
addon = mozpath.basedir(e.base, self._addons)
if addon:
self._addons[addon] = 'unpacked'
if isinstance(e, Manifest):
if e.flags:
errors.fatal('Flags are not supported on ' +
'"manifest" entries')
self._included_manifests[e.path] = path
def _add_manifest_file(self, path, file):
'''
Add the given BaseFile with manifest file contents with the given path.
'''
self._manifests.add(path)
base = ''
if hasattr(file, 'path'):
# Find the directory the given path is relative to.
b = mozpath.normsep(file.path)
if b.endswith('/' + path) or b == path:
base = os.path.normpath(b[:-len(path)])
for e in parse_manifest(base, path, file.open()):
# ManifestResources need to be given after ManifestChrome, so just
# put all ManifestChrome in a separate queue to make them first.
if isinstance(e, ManifestChrome):
# e.move(e.base) just returns a clone of the entry.
self._chrome_queue.append(self.formatter.add_manifest,
e.move(e.base))
elif not isinstance(e, (Manifest, ManifestInterfaces)):
self._queue.append(self.formatter.add_manifest, e.move(e.base))
# If a binary component is added to an addon, prevent the addon
# from being packed.
if isinstance(e, ManifestBinaryComponent):
addon = mozpath.basedir(e.base, self._addons)
if addon:
self._addons[addon] = 'unpacked'
if isinstance(e, Manifest):
if e.flags:
errors.fatal('Flags are not supported on ' +
'"manifest" entries')
self._included_manifests[e.path] = path
def add_manifest(self, entry):
# Store manifest entries in a single manifest per directory, named
# after their parent directory, except for root manifests, all named
# chrome.manifest.
if entry.base:
name = mozpath.basename(entry.base)
else:
name = 'chrome'
path = mozpath.normpath(mozpath.join(entry.base, '%s.manifest' % name))
if not self.copier.contains(path):
# Add a reference to the manifest file in the parent manifest, if
# the manifest file is not a root manifest.
if entry.base:
parent = mozpath.dirname(entry.base)
relbase = mozpath.basename(entry.base)
relpath = mozpath.join(relbase,
mozpath.basename(path))
self.add_manifest(Manifest(parent, relpath))
self.copier.add(path, ManifestFile(entry.base))
if isinstance(entry, ManifestChrome):
data = self._chrome_db.setdefault(entry.name, {})
entries = data.setdefault(entry.type, [])
for e in entries:
# Ideally, we'd actually check whether entry.flags are more
# specific than e.flags, but in practice the following test
# is enough for now.
if not entry.flags or e.flags and entry.flags == e.flags:
errors.fatal('"%s" overrides "%s"' % (entry, e))
entries.append(entry)
self.copier[path].add(entry)
def precompile_cache(formatter, source_path, gre_path, app_path):
'''
Create startup cache for the given application directory, using the
given GRE path.
- formatter is a Formatter instance where to add the startup cache.
- source_path is the base path of the package.
- gre_path is the GRE path, relative to source_path.
- app_path is the application path, relative to source_path.
Startup cache for all resources under resource://app/ are generated,
except when gre_path == app_path, in which case it's under
resource://gre/.
'''
from tempfile import mkstemp
source_path = os.path.abspath(source_path)
if app_path != gre_path:
resource = 'app'
else:
resource = 'gre'
app_path = os.path.join(source_path, app_path)
gre_path = os.path.join(source_path, gre_path)
fd, cache = mkstemp('.zip')
os.close(fd)
os.remove(cache)
# For VC12, make sure we can find the right bitness of pgort120.dll
env = os.environ.copy()
if 'VS120COMNTOOLS' in env and not buildconfig.substs['HAVE_64BIT_OS']:
vc12dir = os.path.abspath(
os.path.join(env['VS120COMNTOOLS'], '../../VC/bin'))
if os.path.exists(vc12dir):
env['PATH'] = vc12dir + ';' + env['PATH']
try:
if launcher.launch([
'xpcshell', '-g', gre_path, '-a', app_path, '-f',
os.path.join(os.path.dirname(__file__), 'precompile_cache.js'),
'-e',
'precompile_startupcache("resource://%s/");' % resource
],
extra_linker_path=gre_path,
extra_env={
'MOZ_STARTUP_CACHE': cache,
'PATH': env['PATH']
}):
errors.fatal('Error while running startup cache precompilation')
return
from mozpack.mozjar import JarReader
jar = JarReader(cache)
resource = '/resource/%s/' % resource
for f in jar:
if resource in f.filename:
path = f.filename[f.filename.index(resource) + len(resource):]
if formatter.contains(path):
formatter.add(f.filename, GeneratedFile(f.read()))
jar.close()
finally:
if os.path.exists(cache):
os.remove(cache)
def elfhack(path):
'''
Execute the elfhack command on the given path.
'''
from buildconfig import topobjdir
cmd = [os.path.join(topobjdir, 'build/unix/elfhack/elfhack'), path]
if subprocess.call(cmd) != 0:
errors.fatal('Error executing ' + ' '.join(cmd))
def copy(self, dest, skip_if_older=True):
if isinstance(dest, basestring):
dest = Dest(dest)
else:
assert isinstance(dest, Dest)
if not dest.exists():
errors.fatal("Required existing file doesn't exist: %s" %
dest.path)
def elfhack(path):
"""
Execute the elfhack command on the given path.
"""
from buildconfig import topobjdir
cmd = [os.path.join(topobjdir, "build/unix/elfhack/elfhack"), path]
if subprocess.call(cmd) != 0:
errors.fatal("Error executing " + " ".join(cmd))
def elfhack(path):
'''
Execute the elfhack command on the given path.
'''
cmd = [os.path.join(topobjdir, 'build/unix/elfhack/elfhack'), path]
if 'ELF_HACK_FLAGS' in os.environ:
cmd[1:0] = os.environ['ELF_HACK_FLAGS'].split()
if subprocess.call(cmd) != 0:
errors.fatal('Error executing ' + ' '.join(cmd))
def strip(path):
"""
Execute the STRIP command with STRIP_FLAGS on the given path.
"""
strip = substs["STRIP"]
flags = substs["STRIP_FLAGS"].split() if "STRIP_FLAGS" in substs else []
cmd = [strip] + flags + [path]
if subprocess.call(cmd) != 0:
errors.fatal("Error executing " + " ".join(cmd))
def elfhack(path):
"""
Execute the elfhack command on the given path.
"""
cmd = [os.path.join(topobjdir, "build/unix/elfhack/elfhack"), path]
if "ELF_HACK_FLAGS" in os.environ:
cmd[1:0] = os.environ["ELF_HACK_FLAGS"].split()
if subprocess.call(cmd) != 0:
errors.fatal("Error executing " + " ".join(cmd))
def copy(self, dest, skip_if_older=True):
if isinstance(dest, basestring):
dest = Dest(dest)
else:
assert isinstance(dest, Dest)
if not dest.exists():
errors.fatal("Required existing file doesn't exist: %s" %
dest.path)
def strip(path):
'''
Execute the STRIP command with STRIP_FLAGS on the given path.
'''
strip = substs['STRIP']
flags = substs['STRIP_FLAGS'].split() if 'STRIP_FLAGS' in substs else []
cmd = [strip] + flags + [path]
if subprocess.call(cmd) != 0:
errors.fatal('Error executing ' + ' '.join(cmd))
def precompile_cache(formatter, source_path, gre_path, app_path):
'''
Create startup cache for the given application directory, using the
given GRE path.
- formatter is a Formatter instance where to add the startup cache.
- source_path is the base path of the package.
- gre_path is the GRE path, relative to source_path.
- app_path is the application path, relative to source_path.
Startup cache for all resources under resource://app/ are generated,
except when gre_path == app_path, in which case it's under
resource://gre/.
'''
from tempfile import mkstemp
source_path = os.path.abspath(source_path)
if app_path != gre_path:
resource = 'app'
else:
resource = 'gre'
app_path = os.path.join(source_path, app_path)
gre_path = os.path.join(source_path, gre_path)
fd, cache = mkstemp('.zip')
os.close(fd)
os.remove(cache)
# For VC12, make sure we can find the right bitness of pgort120.dll
env = os.environ.copy()
if 'VS120COMNTOOLS' in env and not buildconfig.substs['HAVE_64BIT_OS']:
vc12dir = os.path.abspath(os.path.join(env['VS120COMNTOOLS'],
'../../VC/bin'))
if os.path.exists(vc12dir):
env['PATH'] = vc12dir + ';' + env['PATH']
try:
if launcher.launch(['xpcshell', '-g', gre_path, '-a', app_path,
'-f', os.path.join(os.path.dirname(__file__),
'precompile_cache.js'),
'-e', 'precompile_startupcache("resource://%s/");'
% resource],
extra_linker_path=gre_path,
extra_env={'MOZ_STARTUP_CACHE': cache,
'PATH': env['PATH']}):
errors.fatal('Error while running startup cache precompilation')
return
from mozpack.mozjar import JarReader
jar = JarReader(cache)
resource = '/resource/%s/' % resource
for f in jar:
if resource in f.filename:
path = f.filename[f.filename.index(resource) + len(resource):]
if formatter.contains(path):
formatter.add(f.filename, GeneratedFile(f.read()))
jar.close()
finally:
if os.path.exists(cache):
os.remove(cache)
def __init__(self, base, *flags):
'''
Initialize a manifest entry with the given base path and flags.
'''
self.base = base
self.flags = Flags(*flags)
if not all(f in self.allowed_flags for f in self.flags):
errors.fatal('%s unsupported for %s manifest entries' %
(','.join(f for f in self.flags
if not f in self.allowed_flags), self.type))
def __init__(self, base, *flags):
'''
Initialize a manifest entry with the given base path and flags.
'''
self.base = base
self.flags = Flags(*flags)
if not all(f in self.allowed_flags for f in self.flags):
errors.fatal('%s unsupported for %s manifest entries' %
(','.join(f for f in self.flags
if not f in self.allowed_flags), self.type))
def strip(path):
'''
Execute the STRIP command with STRIP_FLAGS on the given path.
'''
from buildconfig import substs
strip = substs['STRIP']
flags = substs.get('STRIP_FLAGS', [])
cmd = [strip] + flags + [path]
if subprocess.call(cmd) != 0:
errors.fatal('Error executing ' + ' '.join(cmd))
def precompile_cache(registry, source_path, gre_path, app_path):
'''
Create startup cache for the given application directory, using the
given GRE path.
- registry is a FileRegistry-like instance where to add the startup cache.
- source_path is the base path of the package.
- gre_path is the GRE path, relative to source_path.
- app_path is the application path, relative to source_path.
Startup cache for all resources under resource://app/ are generated,
except when gre_path == app_path, in which case it's under
resource://gre/.
'''
from tempfile import mkstemp
source_path = os.path.abspath(source_path)
if app_path != gre_path:
resource = 'app'
else:
resource = 'gre'
app_path = os.path.join(source_path, app_path)
gre_path = os.path.join(source_path, gre_path)
fd, cache = mkstemp('.zip')
if os.name == 'os2':
# NS_NewLocalFile is strict about slashes
cache = cache.replace('/', '\\')
os.close(fd)
os.remove(cache)
try:
extra_env = {'MOZ_STARTUP_CACHE': cache}
if buildconfig.substs.get('MOZ_TSAN'):
extra_env['TSAN_OPTIONS'] = 'report_bugs=0'
if buildconfig.substs.get('MOZ_ASAN'):
extra_env['ASAN_OPTIONS'] = 'detect_leaks=0'
if launcher.launch(['xpcshell', '-g', gre_path, '-a', app_path,
'-f', os.path.join(os.path.dirname(__file__),
'precompile_cache.js'),
'-e', 'precompile_startupcache("resource://%s/");'
% resource],
extra_linker_path=gre_path,
extra_env=extra_env):
errors.fatal('Error while running startup cache precompilation')
return
from mozpack.mozjar import JarReader
jar = JarReader(cache)
resource = '/resource/%s/' % resource
for f in jar:
if resource in f.filename:
path = f.filename[f.filename.index(resource) + len(resource):]
if registry.contains(path):
registry.add(f.filename, GeneratedFile(f.read()))
jar.close()
finally:
if os.path.exists(cache):
os.remove(cache)
def __init__(self, base, *flags):
"""
Initialize a manifest entry with the given base path and flags.
"""
self.base = base
self.flags = Flags(*flags)
if not all(f in self.allowed_flags for f in self.flags):
errors.fatal("%s unsupported for %s manifest entries" % (
",".join(f for f in self.flags if f not in self.allowed_flags),
self.type,
))
def copy(self, dest, skip_if_older=True):
assert isinstance(dest, basestring)
# os.path.getmtime returns a result in seconds with precision up to the
# microsecond. But microsecond is too precise because shutil.copystat
# only copies milliseconds, and seconds is not enough precision.
if os.path.exists(dest) and skip_if_older and \
int(os.path.getmtime(self.path) * 1000) <= \
int(os.path.getmtime(dest) * 1000):
return False
if launcher.launch(['shlibsign', '-v', '-o', dest, '-i', self.path]):
errors.fatal('Error while signing %s' % self.path)
def strip(path):
"""
Execute the STRIP command with STRIP_FLAGS on the given path.
"""
from buildconfig import substs
strip = substs["STRIP"]
flags = substs.get("STRIP_FLAGS", [])
cmd = [strip] + flags + [path]
if subprocess.call(cmd) != 0:
errors.fatal("Error executing " + " ".join(cmd))
def copy(self, dest, skip_if_older=True):
assert isinstance(dest, basestring)
# os.path.getmtime returns a result in seconds with precision up to the
# microsecond. But microsecond is too precise because shutil.copystat
# only copies milliseconds, and seconds is not enough precision.
if os.path.exists(dest) and skip_if_older and \
int(os.path.getmtime(self.path) * 1000) <= \
int(os.path.getmtime(dest) * 1000):
return False
if launcher.launch(['shlibsign', '-v', '-o', dest, '-i', self.path]):
errors.fatal('Error while signing %s' % self.path)
def precompile_cache(registry, source_path, gre_path, app_path):
'''
Create startup cache for the given application directory, using the
given GRE path.
- registry is a FileRegistry-like instance where to add the startup cache.
- source_path is the base path of the package.
- gre_path is the GRE path, relative to source_path.
- app_path is the application path, relative to source_path.
Startup cache for all resources under resource://app/ are generated,
except when gre_path == app_path, in which case it's under
resource://gre/.
'''
from tempfile import mkstemp
source_path = os.path.abspath(source_path)
if app_path != gre_path:
resource = 'app'
else:
resource = 'gre'
app_path = os.path.join(source_path, app_path)
gre_path = os.path.join(source_path, gre_path)
fd, cache = mkstemp('.zip')
os.close(fd)
os.remove(cache)
try:
extra_env = {'MOZ_STARTUP_CACHE': cache}
if buildconfig.substs.get('MOZ_TSAN'):
extra_env['TSAN_OPTIONS'] = 'report_bugs=0'
if buildconfig.substs.get('MOZ_ASAN'):
extra_env['ASAN_OPTIONS'] = 'detect_leaks=0'
if launcher.launch([
'xpcshell', '-g', gre_path, '-a', app_path, '-f',
os.path.join(os.path.dirname(__file__), 'precompile_cache.js'),
'-e',
'precompile_startupcache("resource://%s/");' % resource
],
extra_linker_path=gre_path,
extra_env=extra_env):
errors.fatal('Error while running startup cache precompilation')
return
from mozpack.mozjar import JarReader
jar = JarReader(cache)
resource = '/resource/%s/' % resource
for f in jar:
if resource in f.filename:
path = f.filename[f.filename.index(resource) + len(resource):]
if registry.contains(path):
registry.add(f.filename, GeneratedFile(f.read()))
jar.close()
finally:
if os.path.exists(cache):
os.remove(cache)
def from_string(string):
'''
Create a component from a string.
'''
try:
name, options = Component._split_component_and_options(string)
except ValueError as e:
errors.fatal('Malformed manifest: %s' % e)
return
destdir = options.pop('destdir', '')
if options:
errors.fatal('Malformed manifest: options %s not recognized'
% options.keys())
return Component(name, destdir=destdir)
def precompile_cache(formatter, source_path, gre_path, app_path):
'''
Create startup cache for the given application directory, using the
given GRE path.
- formatter is a Formatter instance where to add the startup cache.
- source_path is the base path of the package.
- gre_path is the GRE path, relative to source_path.
- app_path is the application path, relative to source_path.
Startup cache for all resources under resource://app/ are generated,
except when gre_path == app_path, in which case it's under
resource://gre/.
'''
from tempfile import mkstemp
source_path = os.path.abspath(source_path)
if app_path != gre_path:
resource = 'app'
else:
resource = 'gre'
app_path = os.path.join(source_path, app_path)
gre_path = os.path.join(source_path, gre_path)
fd, cache = mkstemp('.zip')
if os.name == 'os2':
# NS_NewLocalFile is strict about slashes
cache = cache.replace('/', '\\')
os.close(fd)
os.remove(cache)
try:
if launcher.launch(['xpcshell', '-g', gre_path, '-a', app_path,
'-f', os.path.join(os.path.dirname(__file__),
'precompile_cache.js'),
'-e', 'precompile_startupcache("resource://%s/");'
% resource],
extra_linker_path=gre_path,
extra_env={'MOZ_STARTUP_CACHE': cache}):
errors.fatal('Error while running startup cache precompilation')
return
from mozpack.mozjar import JarReader
jar = JarReader(cache)
resource = '/resource/%s/' % resource
for f in jar:
if resource in f.filename:
path = f.filename[f.filename.index(resource) + len(resource):]
if formatter.contains(path):
formatter.add(f.filename, GeneratedFile(f.read()))
jar.close()
finally:
if os.path.exists(cache):
os.remove(cache)
def from_string(string):
'''
Create a component from a string.
'''
try:
name, options = Component._split_component_and_options(string)
except ValueError as e:
errors.fatal('Malformed manifest: %s' % e)
return
destdir = options.pop('destdir', '')
if options:
errors.fatal('Malformed manifest: options %s not recognized'
% options.keys())
return Component(name, destdir=destdir)
def from_string(string):
"""
Create a component from a string.
"""
try:
name, options = Component._split_component_and_options(string)
except ValueError as e:
errors.fatal("Malformed manifest: %s" % e)
return
destdir = options.pop("destdir", "")
xz_compress = options.pop("xz_compress", "0") != "0"
if options:
errors.fatal("Malformed manifest: options %s not recognized" %
options.keys())
return Component(name, destdir=destdir, xz_compress=xz_compress)
def add_definition(self, definition):
'''
Add a flag value definition. Replaces any previously set value.
'''
if definition == self.name:
self.value = True
return
assert(definition.startswith(self.name))
if definition[len(self.name)] != '=':
return errors.fatal('Malformed flag: %s' % definition)
value = definition[len(self.name) + 1:]
if value in ('yes', 'true', '1', 'no', 'false', '0'):
self.value = value
else:
return errors.fatal('Unknown value in: %s' % definition)
def __init__(self, *flags):
'''
Initialize a set of flags given in string form.
flags = Flags('contentaccessible=yes', 'appversion>=3.5')
'''
OrderedDict.__init__(self)
for f in flags:
name = self.RE.split(f)
name = name[0]
if not name in self.FLAGS:
errors.fatal('Unknown flag: %s' % name)
continue
if not name in self:
self[name] = self.FLAGS[name](name)
self[name].add_definition(f)
def add_definition(self, definition):
'''
Add a flag value definition. Replaces any previously set value.
'''
if definition == self.name:
self.value = True
return
assert (definition.startswith(self.name))
if definition[len(self.name)] != '=':
return errors.fatal('Malformed flag: %s' % definition)
value = definition[len(self.name) + 1:]
if value in ('yes', 'true', '1', 'no', 'false', '0'):
self.value = value
else:
return errors.fatal('Unknown value in: %s' % definition)
def add_definition(self, definition):
"""
Add a flag value definition. Replaces any previously set value.
"""
if definition == self.name:
self.value = True
return
assert definition.startswith(self.name)
if definition[len(self.name)] != "=":
return errors.fatal("Malformed flag: %s" % definition)
value = definition[len(self.name) + 1 :]
if value in ("yes", "true", "1", "no", "false", "0"):
self.value = value
else:
return errors.fatal("Unknown value in: %s" % definition)
def __init__(self, *flags):
'''
Initialize a set of flags given in string form.
flags = Flags('contentaccessible=yes', 'appversion>=3.5')
'''
OrderedDict.__init__(self)
for f in flags:
name = self.RE.split(f)
name = name[0]
if not name in self.FLAGS:
errors.fatal('Unknown flag: %s' % name)
continue
if not name in self:
self[name] = self.FLAGS[name](name)
self[name].add_definition(f)
def precompile_cache(formatter, source_path, gre_path, app_path):
'''
Create startup cache for the given application directory, using the
given GRE path.
- formatter is a Formatter instance where to add the startup cache.
- source_path is the base path of the package.
- gre_path is the GRE path, relative to source_path.
- app_path is the application path, relative to source_path.
Startup cache for all resources under resource://app/ are generated,
except when gre_path == app_path, in which case it's under
resource://gre/.
'''
from tempfile import mkstemp
source_path = os.path.abspath(source_path)
if app_path != gre_path:
resource = 'app'
else:
resource = 'gre'
app_path = os.path.join(source_path, app_path)
gre_path = os.path.join(source_path, gre_path)
fd, cache = mkstemp('.zip')
os.close(fd)
os.remove(cache)
try:
if launcher.launch(['xpcshell', '-g', gre_path, '-a', app_path,
'-f', os.path.join(os.path.dirname(__file__),
'precompile_cache.js'),
'-e', 'precompile_startupcache("resource://%s/");'
% resource],
extra_linker_path=gre_path,
extra_env={'MOZ_STARTUP_CACHE': cache}):
errors.fatal('Error while running startup cache precompilation')
return
from mozpack.mozjar import JarReader
jar = JarReader(cache)
resource = '/resource/%s/' % resource
for f in jar:
if resource in f.filename:
path = f.filename[f.filename.index(resource) + len(resource):]
if formatter.contains(path):
formatter.add(f.filename, GeneratedFile(f.read()))
jar.close()
finally:
if os.path.exists(cache):
os.remove(cache)
def xz_compress(path):
"""
Execute xz to compress the given path.
"""
if open(path, "rb").read(5)[1:] == "7zXZ":
print("%s is already compressed" % path)
return
from buildconfig import substs
xz = substs.get("XZ")
cmd = [xz, "-zkf", path]
# For now, the mozglue XZStream ELF loader can only support xz files
# with a single stream that contains a single block. In xz, there is no
# explicit option to set the max block count. Instead, we force xz to use
# single thread mode, which results in a single block.
cmd.extend(["--threads=1"])
bcj = None
if substs.get("MOZ_THUMB2"):
bcj = "--armthumb"
elif substs.get("CPU_ARCH") == "arm":
bcj = "--arm"
elif substs.get("CPU_ARCH") == "x86":
bcj = "--x86"
if bcj:
cmd.extend([bcj])
# We need to explicitly specify the LZMA filter chain to ensure consistent builds
# across platforms. Note that the dict size must be less then 16MiB per the hardcoded
# value in mozglue/linker/XZStream.cpp. This is the default LZMA filter chain for for
# xz-utils version 5.0. See:
# https://github.com/xz-mirror/xz/blob/v5.0.0/src/liblzma/lzma/lzma_encoder_presets.c
# https://github.com/xz-mirror/xz/blob/v5.0.0/src/liblzma/api/lzma/container.h#L31
cmd.extend([
"--lzma2=dict=8MiB,lc=3,lp=0,pb=2,mode=normal,nice=64,mf=bt4,depth=0"
])
print("xz-compressing %s with %s" % (path, " ".join(cmd)))
if subprocess.call(cmd) != 0:
errors.fatal("Error executing " + " ".join(cmd))
return
os.rename(path + ".xz", path)
def add_definition(self, definition):
'''
Add a version flag definition.
'''
assert (definition.startswith(self.name))
value = definition[len(self.name):]
if value.startswith('='):
self.values.append(('==', LooseVersion(value[1:])))
elif len(value) > 1 and value[0] in ['<', '>']:
if value[1] == '=':
if len(value) < 3:
return errors.fatal('Malformed flag: %s' % definition)
self.values.append((value[0:2], LooseVersion(value[2:])))
else:
self.values.append((value[0], LooseVersion(value[1:])))
else:
return errors.fatal('Malformed flag: %s' % definition)
def add_definition(self, definition):
"""
Add a version flag definition.
"""
assert definition.startswith(self.name)
value = definition[len(self.name) :]
if value.startswith("="):
self.values.append(("==", LooseVersion(value[1:])))
elif len(value) > 1 and value[0] in ["<", ">"]:
if value[1] == "=":
if len(value) < 3:
return errors.fatal("Malformed flag: %s" % definition)
self.values.append((value[0:2], LooseVersion(value[2:])))
else:
self.values.append((value[0], LooseVersion(value[1:])))
else:
return errors.fatal("Malformed flag: %s" % definition)
def add_definition(self, definition):
'''
Add a version flag definition.
'''
assert(definition.startswith(self.name))
value = definition[len(self.name):]
if value.startswith('='):
self.values.append(('==', LooseVersion(value[1:])))
elif len(value) > 1 and value[0] in ['<', '>']:
if value[1] == '=':
if len(value) < 3:
return errors.fatal('Malformed flag: %s' % definition)
self.values.append((value[0:2], LooseVersion(value[2:])))
else:
self.values.append((value[0], LooseVersion(value[1:])))
else:
return errors.fatal('Malformed flag: %s' % definition)
def handle_line(self, str):
'''
Handle a line of input and push the parsed information to the sink
object.
'''
# Remove comments.
str = str.strip()
if not str or str.startswith(';'):
return
if str.startswith('[') and str.endswith(']'):
self._component = Component.from_string(str[1:-1])
elif str.startswith('-'):
str = str[1:]
self._sink.remove(self._component, str)
elif ',' in str:
errors.fatal('Incompatible syntax')
else:
self._sink.add(self._component, str)
def close(self):
'''
Push all instructions to the formatter.
'''
self._closed = True
broken_addons = sorted(m for m in self._included_manifests
if mozpack.path.dirname(m) in self._addons)
if broken_addons:
errors.fatal(
'Addon base manifest (%s) is included in some other manifest' %
', '.join(broken_addons)
)
for base in self.get_bases():
if base:
self.formatter.add_base(base, base in self._addons)
self._chrome_queue.execute()
self._queue.execute()
self._file_queue.execute()
def handle_line(self, str):
'''
Handle a line of input and push the parsed information to the sink
object.
'''
# Remove comments.
str = str.strip()
if not str or str.startswith(';'):
return
if str.startswith('[') and str.endswith(']'):
self._component = Component.from_string(str[1:-1])
elif str.startswith('-'):
str = str[1:]
self._sink.remove(self._component, str)
elif ',' in str:
errors.fatal('Incompatible syntax')
else:
self._sink.add(self._component, str)
def handle_line(self, str):
"""
Handle a line of input and push the parsed information to the sink
object.
"""
# Remove comments.
str = str.strip()
if not str or str.startswith(";"):
return
if str.startswith("[") and str.endswith("]"):
self._component = Component.from_string(str[1:-1])
elif str.startswith("-"):
str = str[1:]
self._sink.remove(self._component, str)
elif "," in str:
errors.fatal("Incompatible syntax")
else:
self._sink.add(self._component, str)
def close(self):
'''
Push all instructions to the formatter.
'''
self._closed = True
bases = self.get_bases()
broken_bases = sorted(
m for m, includer in self._included_manifests.iteritems()
if mozpath.basedir(m, bases) != mozpath.basedir(includer, bases))
for m in broken_bases:
errors.fatal('"%s" is included from "%s", which is outside "%s"' %
(m, self._included_manifests[m],
mozpath.basedir(m, bases)))
for base in sorted(bases):
self.formatter.add_base(base, self._addons.get(base, False))
self._chrome_queue.execute()
self._queue.execute()
self._file_queue.execute()
def close(self):
'''
Push all instructions to the formatter.
'''
self._closed = True
bases = self.get_bases()
broken_bases = sorted(
m for m, includer in six.iteritems(self._included_manifests)
if mozpath.basedir(m, bases) != mozpath.basedir(includer, bases))
for m in broken_bases:
errors.fatal(
'"%s" is included from "%s", which is outside "%s"' %
(m, self._included_manifests[m], mozpath.basedir(m, bases)))
for base in sorted(bases):
self.formatter.add_base(base, self._addons.get(base, False))
self._chrome_queue.execute()
self._queue.execute()
self._file_queue.execute()
def _add_manifest_file(self, path, file):
'''
Add the given BaseFile with manifest file contents with the given path.
'''
self._manifests.add(path)
base = ''
if hasattr(file, 'path'):
# Find the directory the given path is relative to.
b = mozpack.path.normsep(file.path)
if b.endswith('/' + path) or b == path:
base = os.path.normpath(b[:-len(path)])
for e in parse_manifest(base, path, file.open()):
if not isinstance(e, (Manifest, ManifestInterfaces)):
# e.move(e.base) just returns a clone of the entry.
self._queue.append(self.formatter.add_manifest, e.move(e.base))
if isinstance(e, Manifest):
if e.flags:
errors.fatal('Flags are not supported on ' +
'"manifest" entries')
self._included_manifests.add(e.path)
def _add_manifest_file(self, path, file):
'''
Add the given BaseFile with manifest file contents with the given path.
'''
self._manifests.add(path)
base = ''
if hasattr(file, 'path'):
# Find the directory the given path is relative to.
b = mozpack.path.normsep(file.path)
if b.endswith('/' + path) or b == path:
base = os.path.normpath(b[:-len(path)])
for e in parse_manifest(base, path, file.open()):
if not isinstance(e, (Manifest, ManifestInterfaces)):
# e.move(e.base) just returns a clone of the entry.
self._queue.append(self.formatter.add_manifest, e.move(e.base))
if isinstance(e, Manifest):
if e.flags:
errors.fatal('Flags are not supported on ' +
'"manifest" entries')
self._included_manifests.add(e.path)
def handle_line(self, str):
'''
Handle a line of input and push the parsed information to the sink
object.
'''
# Remove comments.
str = str.strip()
if not str or str.startswith(';'):
return
if str.startswith('[') and str.endswith(']'):
if str == '[]' or re.search(r'[\[\]\s]', str[1:-1]):
errors.fatal('Malformed manifest')
else:
self._component = str[1:-1]
elif str.startswith('-'):
str = str[1:]
self._sink.remove(self._component, str)
elif ',' in str:
errors.fatal('Incompatible syntax')
else:
self._sink.add(self._component, str)
def add_definition(self, definition):
'''
Add a string flag definition.
'''
assert(definition.startswith(self.name))
value = definition[len(self.name):]
if value.startswith('='):
self.values.append(('==', value[1:]))
elif value.startswith('!='):
self.values.append(('!=', value[2:]))
else:
return errors.fatal('Malformed flag: %s' % definition)
def parse_manifest_line(base, line):
'''
Parse a line from a manifest file with the given base directory and
return the corresponding ManifestEntry instance.
'''
# Remove comments
cmd = MANIFEST_RE.sub('', line).strip().split()
if not cmd:
return None
if not cmd[0] in MANIFESTS_TYPES:
return errors.fatal('Unknown manifest directive: %s' % cmd[0])
return MANIFESTS_TYPES[cmd[0]](base, *cmd[1:])
def handle_line(self, str):
'''
Handle a line of input and push the parsed information to the sink
object.
'''
# Remove comments.
str = str.strip()
if not str or str.startswith(';'):
return
if str.startswith('[') and str.endswith(']'):
if str == '[]' or re.search(r'[\[\]\s]', str[1:-1]):
errors.fatal('Malformed manifest')
else:
self._component = str[1:-1]
elif str.startswith('-'):
str = str[1:]
self._sink.remove(self._component, str)
elif ',' in str:
errors.fatal('Incompatible syntax')
else:
self._sink.add(self._component, str)
def __init__(self, name, destdir=''):
if name.find(' ') > 0:
errors.fatal('Malformed manifest: space in component name "%s"'
% component)
self._name = name
self._destdir = destdir
def repack(source, l10n):
finder = UnpackFinder(source)
l10n_finder = UnpackFinder(l10n)
copier = FileCopier()
if finder.kind == 'flat':
formatter = FlatFormatter(copier)
elif finder.kind == 'jar':
formatter = JarFormatter(copier, optimize=finder.optimizedjars)
elif finder.kind == 'omni':
formatter = OmniJarFormatter(copier, finder.omnijar,
optimize=finder.optimizedjars)
# Read all manifest entries from the packaged directory.
manifests = dict((p, m) for p, m in finder.find('**/*.manifest')
if is_manifest(p))
assert all(isinstance(m, ManifestFile) for m in manifests.itervalues())
entries = [e for m in manifests.itervalues() for e in m if e.localized]
# Find unique locales used in these manifest entries.
locales = list(set(e.id for e in entries if isinstance(e, ManifestLocale)))
# Find all paths whose manifest are included by no other manifest.
includes = set(mozpack.path.join(e.base, e.relpath)
for m in manifests.itervalues()
for e in m if isinstance(e, Manifest))
bases = [mozpack.path.dirname(p) for p in set(manifests.keys()) - includes]
# Read all manifest entries from the langpack directory.
manifests = [m for p, m in l10n_finder.find('**/*.manifest')
if is_manifest(p)]
assert all(isinstance(m, ManifestFile) for m in manifests)
l10n_entries = [e for m in manifests for e in m if e.localized]
# Find unique locales used in these manifest entries.
l10n_locales = list(set(e.id for e in l10n_entries
if isinstance(e, ManifestLocale)))
# The code further below assumes there's only one locale replaced with
# another one.
if len(locales) > 1 or len(l10n_locales) > 1:
errors.fatal("Multiple locales aren't supported")
locale = locales[0]
l10n_locale = l10n_locales[0]
# For each base directory, store what path a locale chrome package name
# corresponds to.
# e.g., for the following entry under app/chrome:
# locale foo en-US path/to/files
# keep track that the locale path for foo in app is
# app/chrome/path/to/files.
l10n_paths = {}
for e in l10n_entries:
if isinstance(e, ManifestChrome):
base = mozpack.path.basedir(e.path, bases)
if not base in l10n_paths:
l10n_paths[base] = {}
l10n_paths[base][e.name] = e.path
# For chrome and non chrome files or directories, store what langpack path
# corresponds to a package path.
paths = dict((e.path,
l10n_paths[mozpack.path.basedir(e.path, bases)][e.name])
for e in entries if isinstance(e, ManifestEntryWithRelPath))
for path in NON_CHROME:
for p, f in l10n_finder.find(path):
paths[p] = p
# Create a new package, with non localized bits coming from the original
# package, and localized bits coming from the langpack.
packager = SimplePackager(formatter)
for p, f in finder:
if is_manifest(p):
# Remove localized manifest entries.
for e in [e for e in f if e.localized]:
f.remove(e)
base = mozpack.path.basedir(p, paths.keys())
if base:
# If the path is one that needs a locale replacement, use the
# corresponding file from the langpack.
subpath = mozpack.path.relpath(p, base)
path = mozpack.path.normpath(mozpack.path.join(paths[base],
subpath))
files = [f for p, f in l10n_finder.find(path)]
if len(files) == 0 and base in NON_CHROME:
path = path.replace(locale, l10n_locale)
files = [f for p, f in l10n_finder.find(path)]
if len(files) == 0:
if not base in NON_CHROME:
errors.error("Missing file: %s" % os.path.join(l10n, path))
else:
packager.add(path, files[0])
else:
packager.add(p, f)
# Add localized manifest entries from the langpack.
l10n_manifests = []
for base in set(e.base for e in l10n_entries):
m = ManifestFile(base)
for e in l10n_entries:
if e.base == base:
m.add(e)
path = mozpack.path.join(base, 'chrome.%s.manifest' % l10n_locale)
l10n_manifests.append((path, m))
bases = packager.get_bases()
for path, m in l10n_manifests:
base = mozpack.path.basedir(path, bases)
packager.add(path, m)
# Add a "manifest $path" entry in the top manifest under that base.
m = ManifestFile(base)
m.add(Manifest(base, mozpack.path.relpath(path, base)))
packager.add(mozpack.path.join(base, 'chrome.manifest'), m)
packager.close()
# Add any remaining non chrome files.
for base in NON_CHROME:
for p, f in l10n_finder.find(base):
if not formatter.contains(p):
formatter.add(p, f)
# Transplant jar preloading information.
for path, log in finder.jarlogs.iteritems():
assert isinstance(copier[path], Jarrer)
copier[path].preload([l.replace(locale, l10n_locale) for l in log])
copier.copy(source)
generate_precomplete(source)
def remove(self, section, pattern):
assert not self._closed
errors.fatal('Removal is unsupported')
def main():
parser = ArgumentParser()
parser.add_argument('-D', dest='defines', action='append',
metavar="VAR[=VAL]", help='Define a variable')
parser.add_argument('--format', default='omni',
help='Choose the chrome format for packaging ' +
'(omni, jar or flat ; default: %(default)s)')
parser.add_argument('--removals', default=None,
help='removed-files source file')
parser.add_argument('--ignore-errors', action='store_true', default=False,
help='Transform errors into warnings.')
parser.add_argument('--minify', action='store_true', default=False,
help='Make some files more compact while packaging')
parser.add_argument('--minify-js', action='store_true',
help='Minify JavaScript files while packaging.')
parser.add_argument('--js-binary',
help='Path to js binary. This is used to verify '
'minified JavaScript. If this is not defined, '
'minification verification will not be performed.')
parser.add_argument('--jarlog', default='', help='File containing jar ' +
'access logs')
parser.add_argument('--optimizejars', action='store_true', default=False,
help='Enable jar optimizations')
parser.add_argument('--disable-compression', action='store_false',
dest='compress', default=True,
help='Disable jar compression')
parser.add_argument('manifest', default=None, nargs='?',
help='Manifest file name')
parser.add_argument('source', help='Source directory')
parser.add_argument('destination', help='Destination directory')
parser.add_argument('--non-resource', nargs='+', metavar='PATTERN',
default=[],
help='Extra files not to be considered as resources')
args = parser.parse_args()
defines = dict(buildconfig.defines)
if args.ignore_errors:
errors.ignore_errors()
if args.defines:
for name, value in [split_define(d) for d in args.defines]:
defines[name] = value
copier = FileCopier()
if args.format == 'flat':
formatter = FlatFormatter(copier)
elif args.format == 'jar':
formatter = JarFormatter(copier, compress=args.compress, optimize=args.optimizejars)
elif args.format == 'omni':
formatter = OmniJarFormatter(copier,
buildconfig.substs['OMNIJAR_NAME'],
compress=args.compress,
optimize=args.optimizejars,
non_resources=args.non_resource)
else:
errors.fatal('Unknown format: %s' % args.format)
# Adjust defines according to the requested format.
if isinstance(formatter, OmniJarFormatter):
defines['MOZ_OMNIJAR'] = 1
elif 'MOZ_OMNIJAR' in defines:
del defines['MOZ_OMNIJAR']
respath = ''
if 'RESPATH' in defines:
respath = SimpleManifestSink.normalize_path(defines['RESPATH'])
while respath.startswith('/'):
respath = respath[1:]
if not buildconfig.substs['CROSS_COMPILE']:
launcher.tooldir = mozpath.join(buildconfig.topobjdir, 'dist')
with errors.accumulate():
finder_args = dict(
minify=args.minify,
minify_js=args.minify_js,
)
if args.js_binary:
finder_args['minify_js_verify_command'] = [
args.js_binary,
os.path.join(os.path.abspath(os.path.dirname(__file__)),
'js-compare-ast.js')
]
finder = FileFinder(args.source, find_executables=True,
**finder_args)
if 'NO_PKG_FILES' in os.environ:
sinkformatter = NoPkgFilesRemover(formatter,
args.manifest is not None)
else:
sinkformatter = formatter
sink = SimpleManifestSink(finder, sinkformatter)
if args.manifest:
preprocess_manifest(sink, args.manifest, defines)
else:
sink.add(Component(''), 'bin/*')
sink.close(args.manifest is not None)
if args.removals:
removals_in = StringIO(open(args.removals).read())
removals_in.name = args.removals
removals = RemovedFiles(copier)
preprocess(removals_in, removals, defines)
copier.add(mozpath.join(respath, 'removed-files'), removals)
# shlibsign libraries
if launcher.can_launch():
if not mozinfo.isMac and buildconfig.substs.get('COMPILE_ENVIRONMENT'):
for lib in SIGN_LIBS:
libbase = mozpath.join(respath, '%s%s') \
% (buildconfig.substs['DLL_PREFIX'], lib)
libname = '%s%s' % (libbase, buildconfig.substs['DLL_SUFFIX'])
if copier.contains(libname):
copier.add(libbase + '.chk',
LibSignFile(os.path.join(args.destination,
libname)))
# Setup preloading
if args.jarlog and os.path.exists(args.jarlog):
from mozpack.mozjar import JarLog
log = JarLog(args.jarlog)
for p, f in copier:
if not isinstance(f, Jarrer):
continue
key = JarLog.canonicalize(os.path.join(args.destination, p))
if key in log:
f.preload(log[key])
copier.copy(args.destination)
generate_precomplete(os.path.normpath(os.path.join(args.destination,
respath)))
def _repack(app_finder, l10n_finder, copier, formatter, non_chrome=set()):
app = LocaleManifestFinder(app_finder)
l10n = LocaleManifestFinder(l10n_finder)
# The code further below assumes there's only one locale replaced with
# another one.
if len(app.locales) > 1 or len(l10n.locales) > 1:
errors.fatal("Multiple locales aren't supported")
locale = app.locales[0]
l10n_locale = l10n.locales[0]
# For each base directory, store what path a locale chrome package name
# corresponds to.
# e.g., for the following entry under app/chrome:
# locale foo en-US path/to/files
# keep track that the locale path for foo in app is
# app/chrome/path/to/files.
l10n_paths = {}
for e in l10n.entries:
if isinstance(e, ManifestChrome):
base = mozpath.basedir(e.path, app.bases)
l10n_paths.setdefault(base, {})
l10n_paths[base][e.name] = e.path
# For chrome and non chrome files or directories, store what langpack path
# corresponds to a package path.
paths = {}
for e in app.entries:
if isinstance(e, ManifestEntryWithRelPath):
base = mozpath.basedir(e.path, app.bases)
if base not in l10n_paths:
errors.fatal("Locale doesn't contain %s/" % base)
# Allow errors to accumulate
continue
if e.name not in l10n_paths[base]:
errors.fatal("Locale doesn't have a manifest entry for '%s'" %
e.name)
# Allow errors to accumulate
continue
paths[e.path] = l10n_paths[base][e.name]
for pattern in non_chrome:
for base in app.bases:
path = mozpath.join(base, pattern)
left = set(p for p, f in app_finder.find(path))
right = set(p for p, f in l10n_finder.find(path))
for p in right:
paths[p] = p
for p in left - right:
paths[p] = None
# Create a new package, with non localized bits coming from the original
# package, and localized bits coming from the langpack.
packager = SimplePackager(formatter)
for p, f in app_finder:
if is_manifest(p):
# Remove localized manifest entries.
for e in [e for e in f if e.localized]:
f.remove(e)
# If the path is one that needs a locale replacement, use the
# corresponding file from the langpack.
path = None
if p in paths:
path = paths[p]
if not path:
continue
else:
base = mozpath.basedir(p, paths.keys())
if base:
subpath = mozpath.relpath(p, base)
path = mozpath.normpath(mozpath.join(paths[base],
subpath))
if path:
files = [f for p, f in l10n_finder.find(path)]
if not len(files):
if base not in non_chrome:
errors.error("Missing file: %s" %
os.path.join(l10n_finder.base, path))
else:
packager.add(path, files[0])
else:
packager.add(p, f)
# Add localized manifest entries from the langpack.
l10n_manifests = []
for base in set(e.base for e in l10n.entries):
m = ManifestFile(base, [e for e in l10n.entries if e.base == base])
path = mozpath.join(base, 'chrome.%s.manifest' % l10n_locale)
l10n_manifests.append((path, m))
bases = packager.get_bases()
for path, m in l10n_manifests:
base = mozpath.basedir(path, bases)
packager.add(path, m)
# Add a "manifest $path" entry in the top manifest under that base.
m = ManifestFile(base)
m.add(Manifest(base, mozpath.relpath(path, base)))
packager.add(mozpath.join(base, 'chrome.manifest'), m)
packager.close()
# Add any remaining non chrome files.
for pattern in non_chrome:
for base in bases:
for p, f in l10n_finder.find(mozpath.join(base, pattern)):
if not formatter.contains(p):
formatter.add(p, f)
# Transplant jar preloading information.
for path, log in app_finder.jarlogs.iteritems():
assert isinstance(copier[path], Jarrer)
copier[path].preload([l.replace(locale, l10n_locale) for l in log])
def remove(self, component, pattern):
'''
Remove files with the given pattern in the given component.
'''
assert not self._closed
errors.fatal('Removal is unsupported')
def main():
parser = ArgumentParser()
parser.add_argument('-D', dest='defines', action='append',
metavar="VAR[=VAL]", help='Define a variable')
parser.add_argument('--format', default='omni',
help='Choose the chrome format for packaging ' +
'(omni, jar or flat ; default: %(default)s)')
parser.add_argument('--removals', default=None,
help='removed-files source file')
parser.add_argument('--ignore-errors', action='store_true', default=False,
help='Transform errors into warnings.')
parser.add_argument('--minify', action='store_true', default=False,
help='Make some files more compact while packaging')
parser.add_argument('--minify-js', action='store_true',
help='Minify JavaScript files while packaging.')
parser.add_argument('--js-binary',
help='Path to js binary. This is used to verify '
'minified JavaScript. If this is not defined, '
'minification verification will not be performed.')
parser.add_argument('--jarlog', default='', help='File containing jar ' +
'access logs')
parser.add_argument('--optimizejars', action='store_true', default=False,
help='Enable jar optimizations')
parser.add_argument('--unify', default='',
help='Base directory of another build to unify with')
parser.add_argument('manifest', default=None, nargs='?',
help='Manifest file name')
parser.add_argument('source', help='Source directory')
parser.add_argument('destination', help='Destination directory')
parser.add_argument('--non-resource', nargs='+', metavar='PATTERN',
default=[],
help='Extra files not to be considered as resources')
args = parser.parse_args()
defines = dict(buildconfig.defines)
if args.ignore_errors:
errors.ignore_errors()
if args.defines:
for name, value in [split_define(d) for d in args.defines]:
defines[name] = value
copier = FileCopier()
if args.format == 'flat':
formatter = FlatFormatter(copier)
elif args.format == 'jar':
formatter = JarFormatter(copier, optimize=args.optimizejars)
elif args.format == 'omni':
formatter = OmniJarFormatter(copier,
buildconfig.substs['OMNIJAR_NAME'],
optimize=args.optimizejars,
non_resources=args.non_resource)
else:
errors.fatal('Unknown format: %s' % args.format)
# Adjust defines according to the requested format.
if isinstance(formatter, OmniJarFormatter):
defines['MOZ_OMNIJAR'] = 1
elif 'MOZ_OMNIJAR' in defines:
del defines['MOZ_OMNIJAR']
respath = ''
if 'RESPATH' in defines:
respath = SimpleManifestSink.normalize_path(defines['RESPATH'])
while respath.startswith('/'):
respath = respath[1:]
if args.unify:
def is_native(path):
path = os.path.abspath(path)
return platform.machine() in mozpath.split(path)
# Invert args.unify and args.source if args.unify points to the
# native architecture.
args.source, args.unify = sorted([args.source, args.unify],
key=is_native, reverse=True)
if is_native(args.source):
launcher.tooldir = args.source
elif not buildconfig.substs['CROSS_COMPILE']:
launcher.tooldir = buildconfig.substs['LIBXUL_DIST']
with errors.accumulate():
finder_args = dict(
minify=args.minify,
minify_js=args.minify_js,
)
if args.js_binary:
finder_args['minify_js_verify_command'] = [
args.js_binary,
os.path.join(os.path.abspath(os.path.dirname(__file__)),
'js-compare-ast.js')
]
if args.unify:
finder = UnifiedBuildFinder(FileFinder(args.source),
FileFinder(args.unify),
**finder_args)
else:
finder = FileFinder(args.source, **finder_args)
if 'NO_PKG_FILES' in os.environ:
sinkformatter = NoPkgFilesRemover(formatter,
args.manifest is not None)
else:
sinkformatter = formatter
sink = SimpleManifestSink(finder, sinkformatter)
if args.manifest:
preprocess_manifest(sink, args.manifest, defines)
else:
sink.add(Component(''), 'bin/*')
sink.close(args.manifest is not None)
if args.removals:
removals_in = StringIO(open(args.removals).read())
removals_in.name = args.removals
removals = RemovedFiles(copier)
preprocess(removals_in, removals, defines)
copier.add(mozpath.join(respath, 'removed-files'), removals)
# shlibsign libraries
if launcher.can_launch():
if not mozinfo.isMac:
for lib in SIGN_LIBS:
libbase = mozpath.join(respath, '%s%s') \
% (buildconfig.substs['DLL_PREFIX'], lib)
libname = '%s%s' % (libbase, buildconfig.substs['DLL_SUFFIX'])
if copier.contains(libname):
copier.add(libbase + '.chk',
LibSignFile(os.path.join(args.destination,
libname)))
# Setup preloading
if args.jarlog and os.path.exists(args.jarlog):
from mozpack.mozjar import JarLog
log = JarLog(args.jarlog)
for p, f in copier:
if not isinstance(f, Jarrer):
continue
key = JarLog.canonicalize(os.path.join(args.destination, p))
if key in log:
f.preload(log[key])
# Fill startup cache
if isinstance(formatter, OmniJarFormatter) and launcher.can_launch() \
and buildconfig.substs['MOZ_DISABLE_STARTUPCACHE'] != '1':
gre_path = None
def get_bases():
for b in sink.packager.get_bases(addons=False):
for p in (mozpath.join('bin', b), b):
if os.path.exists(os.path.join(args.source, p)):
yield p
break
for base in sorted(get_bases()):
if not gre_path:
gre_path = base
base_path = sink.normalize_path(base)
if base_path in formatter.omnijars:
precompile_cache(formatter.omnijars[base_path],
args.source, gre_path, base)
copier.copy(args.destination)
def main():
parser = ArgumentParser()
parser.add_argument('-D', dest='defines', action='append',
metavar="VAR[=VAL]", help='Define a variable')
parser.add_argument('--format', default='omni',
help='Choose the chrome format for packaging ' +
'(omni, jar or flat ; default: %(default)s)')
parser.add_argument('--removals', default=None,
help='removed-files source file')
parser.add_argument('--ignore-errors', action='store_true', default=False,
help='Transform errors into warnings.')
parser.add_argument('--minify', action='store_true', default=False,
help='Make some files more compact while packaging')
parser.add_argument('--jarlog', default='', help='File containing jar ' +
'access logs')
parser.add_argument('--optimizejars', action='store_true', default=False,
help='Enable jar optimizations')
parser.add_argument('--unify', default='',
help='Base directory of another build to unify with')
parser.add_argument('manifest', default=None, nargs='?',
help='Manifest file name')
parser.add_argument('source', help='Source directory')
parser.add_argument('destination', help='Destination directory')
parser.add_argument('--non-resource', nargs='+', metavar='PATTERN',
default=[],
help='Extra files not to be considered as resources')
args = parser.parse_args()
defines = dict(buildconfig.defines)
if args.ignore_errors:
errors.ignore_errors()
if args.defines:
for name, value in [split_define(d) for d in args.defines]:
defines[name] = value
copier = FileCopier()
if args.format == 'flat':
formatter = FlatFormatter(copier)
elif args.format == 'jar':
formatter = JarFormatter(copier, optimize=args.optimizejars)
elif args.format == 'omni':
formatter = OmniJarFormatter(copier,
buildconfig.substs['OMNIJAR_NAME'],
optimize=args.optimizejars,
non_resources=args.non_resource)
else:
errors.fatal('Unknown format: %s' % args.format)
# Adjust defines according to the requested format.
if isinstance(formatter, OmniJarFormatter):
defines['MOZ_OMNIJAR'] = 1
elif 'MOZ_OMNIJAR' in defines:
del defines['MOZ_OMNIJAR']
binpath = ''
if 'BINPATH' in defines:
binpath = SimpleManifestSink.normalize_path(defines['BINPATH'])
while binpath.startswith('/'):
binpath = binpath[1:]
if args.unify:
def is_native(path):
path = os.path.abspath(path)
return platform.machine() in mozpack.path.split(path)
# Invert args.unify and args.source if args.unify points to the
# native architecture.
args.source, args.unify = sorted([args.source, args.unify],
key=is_native, reverse=True)
if is_native(args.source):
launcher.tooldir = args.source
elif not buildconfig.substs['CROSS_COMPILE']:
launcher.tooldir = buildconfig.substs['LIBXUL_DIST']
with errors.accumulate():
if args.unify:
finder = UnifiedBuildFinder(FileFinder(args.source),
FileFinder(args.unify),
minify=args.minify)
else:
finder = FileFinder(args.source, minify=args.minify)
if 'NO_PKG_FILES' in os.environ:
sinkformatter = NoPkgFilesRemover(formatter,
args.manifest is not None)
else:
sinkformatter = formatter
sink = SimpleManifestSink(finder, sinkformatter)
if args.manifest:
preprocess_manifest(sink, args.manifest, defines)
else:
sink.add(Component(''), 'bin/*')
sink.close(args.manifest is not None)
if args.removals:
lines = [l.lstrip() for l in open(args.removals).readlines()]
removals_in = StringIO(''.join(lines))
removals_in.name = args.removals
removals = RemovedFiles(copier)
preprocess(removals_in, removals, defines)
copier.add(mozpack.path.join(binpath, 'removed-files'), removals)
# shlibsign libraries
if launcher.can_launch():
for lib in SIGN_LIBS:
libbase = mozpack.path.join(binpath, '%s%s') \
% (buildconfig.substs['DLL_PREFIX'], lib)
libname = '%s%s' % (libbase, buildconfig.substs['DLL_SUFFIX'])
if copier.contains(libname):
copier.add(libbase + '.chk',
LibSignFile(os.path.join(args.destination,
libname)))
# Setup preloading
if args.jarlog and os.path.exists(args.jarlog):
from mozpack.mozjar import JarLog
log = JarLog(args.jarlog)
for p, f in copier:
if not isinstance(f, Jarrer):
continue
key = JarLog.canonicalize(os.path.join(args.destination, p))
if key in log:
f.preload(log[key])
# Fill startup cache on Windows and Linux only
# (this currently causes build failure on BSD, so skip on that platfom)
if sys.platform == 'win32' or sys.platform.startswith ('linux'):
if isinstance(formatter, OmniJarFormatter) and launcher.can_launch():
if buildconfig.substs['LIBXUL_SDK']:
gre_path = mozpack.path.join(buildconfig.substs['LIBXUL_DIST'],
'bin')
else:
gre_path = None
for base in sorted([[p for p in [mozpack.path.join('bin', b), b]
if os.path.exists(os.path.join(args.source, p))][0]
for b in sink.packager.get_bases()]):
if not gre_path:
gre_path = base
base_path = sink.normalize_path(base)
if base_path in formatter.omnijars:
precompile_cache(formatter.omnijars[base_path],
args.source, gre_path, base)
copier.copy(args.destination)
generate_precomplete(os.path.normpath(os.path.join(args.destination,
binpath)))
