def test_diff_create(self):
"""Diffs are produced when files are created."""
tmpdir = tempfile.mkdtemp()
try:
path = os.path.join(tmpdir, 'file')
faw = FileAvoidWrite(path, capture_diff=True)
faw.write('new')
faw.close()
diff = '\n'.join(faw.diff)
self.assertIn('+new', diff)
finally:
shutil.rmtree(tmpdir)
def install_from_file(self, filename, distdir):
self.log(logging.INFO, 'artifact',
{'filename': filename},
'Installing from {filename}')
# Do we need to post-process?
processed_filename = filename + PROCESSED_SUFFIX
if self._skip_cache and os.path.exists(processed_filename):
self.log(logging.DEBUG, 'artifact',
{'path': processed_filename},
'Skipping cache: removing cached processed artifact {path}')
os.remove(processed_filename)
if not os.path.exists(processed_filename):
self.log(logging.INFO, 'artifact',
{'filename': filename},
'Processing contents of {filename}')
self.log(logging.INFO, 'artifact',
{'processed_filename': processed_filename},
'Writing processed {processed_filename}')
self._artifact_job.process_artifact(filename, processed_filename)
self._artifact_cache._persist_limit.register_file(processed_filename)
self.log(logging.INFO, 'artifact',
{'processed_filename': processed_filename},
'Installing from processed {processed_filename}')
# Copy all .so files, avoiding modification where possible.
ensureParentDir(mozpath.join(distdir, '.dummy'))
with zipfile.ZipFile(processed_filename) as zf:
for info in zf.infolist():
if info.filename.endswith('.ini'):
continue
n = mozpath.join(distdir, info.filename)
fh = FileAvoidWrite(n, mode='rb')
shutil.copyfileobj(zf.open(info), fh)
file_existed, file_updated = fh.close()
self.log(logging.INFO, 'artifact',
{'updating': 'Updating' if file_updated else 'Not updating', 'filename': n},
'{updating} {filename}')
if not file_existed or file_updated:
# Libraries and binaries may need to be marked executable,
# depending on platform.
perms = info.external_attr >> 16 # See http://stackoverflow.com/a/434689.
perms |= stat.S_IWUSR | stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH # u+w, a+r.
os.chmod(n, perms)
return 0
def gifft_map(output_fd, *args):
probe_type = args[-1]
args = args[DEPS_LEN:-1]
all_objs, options = parse(args)
# Events also need to output maps from event extra enum to strings.
# Sadly we need to generate code for all possible events, not just mirrored.
# Otherwise we won't compile.
if probe_type == "Event":
output_path = Path(os.path.dirname(output_fd.name))
with FileAvoidWrite(output_path / "EventExtraGIFFTMaps.cpp") as cpp_fd:
output_gifft_map(output_fd, probe_type, all_objs, cpp_fd)
else:
output_gifft_map(output_fd, probe_type, all_objs, None)
def main(args):
parser = argparse.ArgumentParser()
parser.add_argument('--verbose',
'-v',
default=False,
action='store_true',
help='be verbose')
parser.add_argument('--silent',
'-s',
default=False,
action='store_true',
help='be silent')
parser.add_argument(
'--srcdir',
metavar='SRCDIR',
action='append',
required=True,
help='directories to read inputs from, in order of priority')
parser.add_argument('output', metavar='OUTPUT', help='output')
opts = parser.parse_args(args)
# Use reversed order so that the first srcdir has higher priority to override keys.
properties = merge_properties('region.properties', reversed(opts.srcdir))
default = properties.get('browser.search.defaultenginename')
engines = properties.get_list('browser.search.order')
if opts.verbose:
print('Read {len} engines: {engines}'.format(len=len(engines),
engines=engines))
print("Default engine is '{default}'.".format(default=default))
browsersearch = {}
browsersearch['default'] = default
browsersearch['engines'] = engines
# FileAvoidWrite creates its parent directories.
output = os.path.abspath(opts.output)
fh = FileAvoidWrite(output)
json.dump(browsersearch, fh)
existed, updated = fh.close()
if not opts.silent:
if updated:
print('{output} updated'.format(output=output))
else:
print('{output} already up-to-date'.format(output=output))
return 0
def main(outdir, compiler, template_file, header_list_file):
if not os.path.isdir(outdir):
os.mkdir(outdir)
template = open(template_file, 'r').read()
for header in open(header_list_file, 'r'):
header = header.rstrip()
if 0 == len(header) or is_comment(header):
continue
path = header_path(header, compiler)
with FileAvoidWrite(os.path.join(outdir, header)) as f:
f.write(string.Template(template).substitute(HEADER=header,
HEADER_PATH=path))
def WriteMakefile(filename, data, build_file, depth, topsrcdir, srcdir, relative_path, common_mk_path, extra_data=None):
if not os.path.isabs(topsrcdir):
topsrcdir = depth + "/" + topsrcdir
if not os.path.isabs(srcdir):
srcdir = depth + "/" + srcdir
#TODO: should compare with the existing file and not overwrite it if the
# contents are the same!
ensure_directory_exists(filename)
with FileAvoidWrite(filename) as f:
f.write(COMMON_HEADER % {'buildfile': build_file,
'depth': depth,
'topsrcdir': topsrcdir,
'srcdir': srcdir})
for k, v in data.iteritems():
f.write("%s = %s\n" % (k, " \\\n ".join([''] + v) if isinstance(v, list) else v))
f.write(COMMON_FOOTER % {'common_mk_path': common_mk_path})
if extra_data:
f.write(extra_data)
def main(argv):
parser = argparse.ArgumentParser('Generate a file from a Python script',
add_help=False)
parser.add_argument('python_script',
metavar='python-script',
type=str,
help='The Python script to run')
parser.add_argument('method_name',
metavar='method-name',
type=str,
help='The method of the script to invoke')
parser.add_argument('output_file',
metavar='output-file',
type=str,
help='The file to generate')
parser.add_argument(
'additional_arguments',
metavar='arg',
nargs='*',
help="Additional arguments to the script's main() method")
args = parser.parse_args(argv)
script = args.python_script
with open(script, 'r') as fh:
module = imp.load_module('script', fh, script,
('.py', 'r', imp.PY_SOURCE))
method = args.method_name
if not hasattr(module, method):
print('Error: script "{0}" is missing a {1} method'.format(
script, method),
file=sys.stderr)
return 1
ret = 1
try:
with FileAvoidWrite(args.output_file) as output:
ret = module.__dict__[method](output, *args.additional_arguments)
except IOError as e:
print('Error opening file "{0}"'.format(e.filename), file=sys.stderr)
traceback.print_exc()
return 1
return ret
def gen_wrappers(unused, outdir, *header_list):
for header in header_list:
with FileAvoidWrite(os.path.join(outdir, header)) as f:
includes = include_next_template.format(header=header)
if header == 'wayland-util.h':
# wayland-util.h in Wayland < 1.12 includes math.h inside an
# extern "C" block, which breaks including the header from C++.
# This was fixed in Wayland 1.12, but for versions earlier than
# that, we work around that by force-including math.h first.
includes = '#include <math.h>\n' + includes
elif header == 'wayland-client.h':
# The system wayland-client.h uses quote includes for
# wayland-util.h, which means wayland-util.h is picked from the
# directory containing wayland-client.h first, and there's no
# way around that with -I, -isystem, or other flags. So, we
# force to include it from our wrapper, before including the
# system header, so that our wayland-util.h wrapper is picked
# first.
includes = '#include "wayland-util.h"\n' + includes
f.write(header_template.format(includes=includes))
def install_from_file(self, filename, distdir):
self.log(logging.INFO, 'artifact', {'filename': filename},
'Installing from {filename}')
# Copy all .so files to dist/bin, avoiding modification where possible.
ensureParentDir(os.path.join(distdir, 'bin', '.dummy'))
with zipfile.ZipFile(filename) as zf:
for info in zf.infolist():
if not info.filename.endswith('.so'):
continue
n = os.path.join(distdir, 'bin',
os.path.basename(info.filename))
fh = FileAvoidWrite(n, mode='r')
shutil.copyfileobj(zf.open(info), fh)
file_existed, file_updated = fh.close()
self.log(
logging.INFO, 'artifact', {
'updating':
'Updating' if file_updated else 'Not updating',
'filename': n
}, '{updating} {filename}')
return 0
def test_diff_not_created_by_default(tmp_path):
file = tmp_path / "file.txt"
faw = FileAvoidWrite(str(file))
faw.write("dummy")
faw.close()
assert faw.diff is None
def gen_wrappers(unused, outdir, *header_list):
for header in header_list:
with FileAvoidWrite(os.path.join(outdir, header)) as f:
f.write(header_template % {
'header': header,
})
def _write_file(self, key, value):
filename = mozpath.join(self._datadir, key)
with FileAvoidWrite(filename) as fh:
json.dump(value, fh, indent=4, encoding=system_encoding)
return filename
def config_status(topobjdir='.', topsrcdir='.', defines=None, substs=None,
source=None, mozconfig=None, args=sys.argv[1:]):
'''Main function, providing config.status functionality.
Contrary to config.status, it doesn't use CONFIG_FILES or CONFIG_HEADERS
variables.
Without the -n option, this program acts as config.status and considers
the current directory as the top object directory, even when config.status
is in a different directory. It will, however, treat the directory
containing config.status as the top object directory with the -n option.
The options to this function are passed when creating the
ConfigEnvironment. These lists, as well as the actual wrapper script
around this function, are meant to be generated by configure.
See build/autoconf/config.status.m4.
'''
if 'CONFIG_FILES' in os.environ:
raise Exception('Using the CONFIG_FILES environment variable is not '
'supported.')
if 'CONFIG_HEADERS' in os.environ:
raise Exception('Using the CONFIG_HEADERS environment variable is not '
'supported.')
if not os.path.isabs(topsrcdir):
raise Exception('topsrcdir must be defined as an absolute directory: '
'%s' % topsrcdir)
default_backends = ['RecursiveMake']
default_backends = (substs or {}).get('BUILD_BACKENDS', ['RecursiveMake'])
parser = ArgumentParser()
parser.add_argument('-v', '--verbose', dest='verbose', action='store_true',
help='display verbose output')
parser.add_argument('-n', dest='not_topobjdir', action='store_true',
help='do not consider current directory as top object directory')
parser.add_argument('-d', '--diff', action='store_true',
help='print diffs of changed files.')
parser.add_argument('-b', '--backend', nargs='+', choices=sorted(backends),
default=default_backends,
help='what backend to build (default: %s).' %
' '.join(default_backends))
parser.add_argument('--dry-run', action='store_true',
help='do everything except writing files out.')
options = parser.parse_args(args)
# Without -n, the current directory is meant to be the top object directory
if not options.not_topobjdir:
topobjdir = os.path.realpath('.')
env = ConfigEnvironment(topsrcdir, topobjdir, defines=defines, substs=substs,
source=source, mozconfig=mozconfig)
with FileAvoidWrite(os.path.join(topobjdir, 'mozinfo.json')) as f:
write_mozinfo(f, env, os.environ)
cpu_start = process_time()
time_start = time.time()
# Make appropriate backend instances, defaulting to RecursiveMakeBackend,
# or what is in BUILD_BACKENDS.
selected_backends = [get_backend_class(b)(env) for b in options.backend]
if options.dry_run:
for b in selected_backends:
b.dry_run = True
reader = BuildReader(env)
emitter = TreeMetadataEmitter(env)
# This won't actually do anything because of the magic of generators.
definitions = emitter.emit(reader.read_topsrcdir())
log_level = logging.DEBUG if options.verbose else logging.INFO
log_manager.add_terminal_logging(level=log_level)
log_manager.enable_unstructured()
print('Reticulating splines...', file=sys.stderr)
if len(selected_backends) > 1:
definitions = list(definitions)
for the_backend in selected_backends:
the_backend.consume(definitions)
execution_time = 0.0
for obj in chain((reader, emitter), selected_backends):
summary = obj.summary()
print(summary, file=sys.stderr)
execution_time += summary.execution_time
if hasattr(obj, 'gyp_summary'):
summary = obj.gyp_summary()
print(summary, file=sys.stderr)
cpu_time = process_time() - cpu_start
wall_time = time.time() - time_start
efficiency = cpu_time / wall_time if wall_time else 100
untracked = wall_time - execution_time
print(
'Total wall time: {:.2f}s; CPU time: {:.2f}s; Efficiency: '
'{:.0%}; Untracked: {:.2f}s'.format(
wall_time, cpu_time, efficiency, untracked),
file=sys.stderr
)
if options.diff:
for the_backend in selected_backends:
for path, diff in sorted(the_backend.file_diffs.items()):
print('\n'.join(diff))
# Advertise Visual Studio if appropriate.
if os.name == 'nt' and 'VisualStudio' not in options.backend:
print(VISUAL_STUDIO_ADVERTISEMENT)
# Advertise Android Studio if it is appropriate.
if MachCommandConditions.is_android(env):
print(ANDROID_IDE_ADVERTISEMENT)
def _write_file(self, key, value):
encoding = 'mbcs' if sys.platform == 'win32' else 'utf-8'
filename = mozpath.join(self._datadir, key)
with FileAvoidWrite(filename) as fh:
json.dump(value, fh, indent=4, encoding=encoding)
return filename
def process(input_dirs, inc_paths, bindings_conf, header_dir, xpcrs_dir,
xpt_dir, deps_dir, module, idl_files):
p = IDLParser()
xpts = []
mk = Makefile()
rule = mk.create_rule()
glbl = {}
exec(open(bindings_conf, encoding='utf-8').read(), glbl)
webidlconfig = glbl['DOMInterfaces']
# Write out dependencies for Python modules we import. If this list isn't
# up to date, we will not re-process XPIDL files if the processor changes.
rule.add_dependencies(
six.ensure_text(s) for s in iter_modules_in_path(topsrcdir))
for path in idl_files:
basename = os.path.basename(path)
stem, _ = os.path.splitext(basename)
idl_data = open(path, encoding='utf-8').read()
idl = p.parse(idl_data, filename=path)
idl.resolve(inc_paths, p, webidlconfig)
header_path = os.path.join(header_dir, '%s.h' % stem)
rs_rt_path = os.path.join(xpcrs_dir, 'rt', '%s.rs' % stem)
rs_bt_path = os.path.join(xpcrs_dir, 'bt', '%s.rs' % stem)
xpts.append(jsonxpt.build_typelib(idl))
rule.add_dependencies(six.ensure_text(s) for s in idl.deps)
# The print_* functions don't actually do anything with the
# passed-in path other than writing it into the file to let people
# know where the original source was. This script receives
# absolute paths, which are not so great to embed in header files
# (they mess with deterministic generation of files on different
# machines, Searchfox logic, shared compilation caches, etc.), so
# we pass in fake paths that are the same across compilations, but
# should still enable people to figure out where to go.
relpath = mozpath.relpath(path, topsrcdir)
with FileAvoidWrite(header_path) as fh:
print_header(idl, fh, path, relpath)
with FileAvoidWrite(rs_rt_path) as fh:
print_rust_bindings(idl, fh, relpath)
with FileAvoidWrite(rs_bt_path) as fh:
print_rust_macros_bindings(idl, fh, relpath)
# NOTE: We don't use FileAvoidWrite here as we may re-run this code due to a
# number of different changes in the code, which may not cause the .xpt
# files to be changed in any way. This means that make will re-run us every
# time a build is run whether or not anything changed. To fix this we
# unconditionally write out the file.
xpt_path = os.path.join(xpt_dir, '%s.xpt' % module)
with open(xpt_path, 'w', encoding='utf-8', newline='\n') as fh:
jsonxpt.write(jsonxpt.link(xpts), fh)
rule.add_targets([six.ensure_text(xpt_path)])
if deps_dir:
deps_path = os.path.join(deps_dir, '%s.pp' % module)
with FileAvoidWrite(deps_path) as fh:
mk.dump(fh)
def WriteCommonMk(path, build_files, scriptname, commandline):
with FileAvoidWrite(path) as f:
f.write(COMMON_MK % {'input_gypfiles': ' '.join(build_files),
'generator': scriptname,
'commandline': ' '.join(commandline)})
def open_output(filename):
return FileAvoidWrite(os.path.join(os.path.dirname(fd.name), filename))
def gradle_install(self, quiet=False):
import mozpack.manifests
m = mozpack.manifests.InstallManifest()
def srcdir(dst, src):
m.add_symlink(os.path.join(self.topsrcdir, src), dst)
srcdir('build.gradle', 'mobile/android/gradle/build.gradle')
srcdir('settings.gradle', 'mobile/android/gradle/settings.gradle')
m.add_pattern_copy(os.path.join(self.topsrcdir, 'mobile/android/gradle/gradle/wrapper'), '**', 'gradle/wrapper')
m.add_copy(os.path.join(self.topsrcdir, 'mobile/android/gradle/gradlew'), 'gradlew')
defines = {
'topsrcdir': self.topsrcdir,
'topobjdir': self.topobjdir,
'ANDROID_SDK_ROOT': self.substs['ANDROID_SDK_ROOT'],
}
m.add_preprocess(os.path.join(self.topsrcdir, 'mobile/android/gradle/gradle.properties.in'),
'gradle.properties',
defines=defines,
deps=os.path.join(self.topobjdir, 'mobile/android/gradle/.deps/gradle.properties.pp'))
m.add_preprocess(os.path.join(self.topsrcdir, 'mobile/android/gradle/local.properties.in'),
'local.properties',
defines=defines,
deps=os.path.join(self.topobjdir, 'mobile/android/gradle/.deps/local.properties.pp'))
srcdir('thirdparty/build.gradle', 'mobile/android/gradle/thirdparty/build.gradle')
srcdir('thirdparty/src/main/AndroidManifest.xml', 'mobile/android/gradle/thirdparty/AndroidManifest.xml')
srcdir('thirdparty/src/main/java', 'mobile/android/thirdparty')
srcdir('omnijar/build.gradle', 'mobile/android/gradle/omnijar/build.gradle')
srcdir('omnijar/src/main/java/locales', 'mobile/android/locales')
srcdir('omnijar/src/main/java/chrome', 'mobile/android/chrome')
srcdir('omnijar/src/main/java/components', 'mobile/android/components')
srcdir('omnijar/src/main/java/modules', 'mobile/android/modules')
srcdir('omnijar/src/main/java/themes', 'mobile/android/themes')
srcdir('app/build.gradle', 'mobile/android/gradle/app/build.gradle')
srcdir('app/src/androidTest/res', 'build/mobile/robocop/res')
srcdir('app/src/androidTest/assets', 'mobile/android/tests/browser/robocop/assets')
# Test code.
srcdir('app/src/robocop', 'mobile/android/tests/browser/robocop/src')
srcdir('app/src/background', 'mobile/android/tests/background/junit3/src')
srcdir('app/src/browser', 'mobile/android/tests/browser/junit3/src')
srcdir('app/src/javaaddons', 'mobile/android/tests/javaaddons/src')
# Test libraries.
srcdir('app/libs', 'build/mobile/robocop')
srcdir('base/build.gradle', 'mobile/android/gradle/base/build.gradle')
srcdir('base/lint.xml', 'mobile/android/gradle/base/lint.xml')
srcdir('base/src/main/AndroidManifest.xml', 'mobile/android/gradle/base/AndroidManifest.xml')
srcdir('base/src/main/java/org/mozilla/gecko', 'mobile/android/base')
srcdir('base/src/main/java/org/mozilla/mozstumbler', 'mobile/android/stumbler/java/org/mozilla/mozstumbler')
srcdir('base/src/main/java/org/mozilla/search', 'mobile/android/search/java/org/mozilla/search')
srcdir('base/src/main/java/org/mozilla/javaaddons', 'mobile/android/javaaddons/java/org/mozilla/javaaddons')
srcdir('base/src/webrtc_audio_device/java', 'media/webrtc/trunk/webrtc/modules/audio_device/android/java/src')
srcdir('base/src/webrtc_video_capture/java', 'media/webrtc/trunk/webrtc/modules/video_capture/android/java/src')
srcdir('base/src/webrtc_video_render/java', 'media/webrtc/trunk/webrtc/modules/video_render/android/java/src')
srcdir('base/src/main/res', 'mobile/android/base/resources')
srcdir('base/src/main/assets', 'mobile/android/app/assets')
srcdir('base/src/crashreporter/res', 'mobile/android/base/crashreporter/res')
srcdir('base/src/branding/res', os.path.join(self.substs['MOZ_BRANDING_DIRECTORY'], 'res'))
# JUnit 4 test code.
srcdir('base/src/background_junit4', 'mobile/android/tests/background/junit4/src')
srcdir('base/resources/background_junit4', 'mobile/android/tests/background/junit4/resources')
manifest_path = os.path.join(self.topobjdir, 'mobile', 'android', 'gradle.manifest')
with FileAvoidWrite(manifest_path) as f:
m.write(fileobj=f)
self.virtualenv_manager.ensure()
code = self.run_process([
self.virtualenv_manager.python_path,
os.path.join(self.topsrcdir, 'python/mozbuild/mozbuild/action/process_install_manifest.py'),
'--no-remove',
'--no-remove-all-directory-symlinks',
'--no-remove-empty-directories',
os.path.join(self.topobjdir, 'mobile', 'android', 'gradle'),
manifest_path],
pass_thru=True, # Allow user to run gradle interactively.
ensure_exit_code=False, # Don't throw on non-zero exit code.
cwd=mozpath.join(self.topsrcdir, 'mobile', 'android'))
if not quiet:
if not code:
print(SUCCESS.format(topobjdir=self.topobjdir))
return code
def gen_wrappers(unused, outdir, compiler, template_file, *header_list):
template = open(template_file, "r").read()
for header in header_list:
with FileAvoidWrite(os.path.join(outdir, header)) as f:
f.write(string.Template(template).substitute(HEADER=header))
def main(args):
parser = argparse.ArgumentParser()
parser.add_argument('--verbose',
'-v',
default=False,
action='store_true',
help='be verbose')
parser.add_argument('--silent',
'-s',
default=False,
action='store_true',
help='be silent')
parser.add_argument(
'--srcdir',
metavar='SRCDIR',
action='append',
required=True,
help='directories to read inputs from, in order of priority')
parser.add_argument('output', metavar='OUTPUT', help='output')
opts = parser.parse_args(args)
# Use reversed order so that the first srcdir has higher priority to override keys.
properties = merge_properties('region.properties', reversed(opts.srcdir))
# Default, not region-specific.
default = properties.get('browser.search.defaultenginename')
engines = properties.get_list('browser.search.order')
writer = codecs.getwriter('utf-8')(sys.stdout)
if opts.verbose:
print('Read {len} engines: {engines}'.format(len=len(engines),
engines=engines),
file=writer)
print("Default engine is '{default}'.".format(default=default),
file=writer)
browsersearch = {}
browsersearch['default'] = default
browsersearch['engines'] = engines
# This gets defaults, yes; but it also gets the list of regions known.
regions = properties.get_dict('browser.search.defaultenginename')
browsersearch['regions'] = {}
for region in regions.keys():
region_default = regions[region]
region_engines = properties.get_list(
'browser.search.order.{region}'.format(region=region))
if opts.verbose:
print("Region '{region}': Read {len} engines: {region_engines}".
format(len=len(region_engines),
region=region,
region_engines=region_engines),
file=writer)
print("Region '{region}': Default engine is '{region_default}'.".
format(region=region, region_default=region_default),
file=writer)
browsersearch['regions'][region] = {
'default': region_default,
'engines': region_engines,
}
# FileAvoidWrite creates its parent directories.
output = os.path.abspath(opts.output)
fh = FileAvoidWrite(output)
json.dump(browsersearch, fh)
existed, updated = fh.close()
if not opts.silent:
if updated:
print('{output} updated'.format(output=output))
else:
print('{output} already up-to-date'.format(output=output))
return 0
def main(args):
parser = argparse.ArgumentParser()
parser.add_argument('--verbose',
'-v',
default=False,
action='store_true',
help='be verbose')
parser.add_argument('--silent',
'-s',
default=False,
action='store_true',
help='be silent')
parser.add_argument('--android-package-name',
metavar='NAME',
required=True,
help='Android package name')
parser.add_argument(
'--resources',
metavar='RESOURCES',
default=None,
help='optional Android resource directory to find drawables in')
parser.add_argument(
'--srcdir',
metavar='SRCDIR',
action='append',
required=True,
help='directories to read inputs from, in order of priority')
parser.add_argument('output', metavar='OUTPUT', help='output')
opts = parser.parse_args(args)
# Use reversed order so that the first srcdir has higher priority to override keys.
properties = merge_properties('region.properties', reversed(opts.srcdir))
# Keep these two in sync.
image_url_template = 'android.resource://%s/drawable/suggestedsites_{name}' % opts.android_package_name
drawables_template = 'drawable*/suggestedsites_{name}.*'
# Load properties corresponding to each site name and define their
# respective image URL.
sites = []
def add_names(names, defaults={}):
for name in names:
site = copy.deepcopy(defaults)
site.update(
properties.get_dict(
'browser.suggestedsites.{name}'.format(name=name),
required_keys=('title', 'url', 'bgcolor')))
site['imageurl'] = image_url_template.format(name=name)
sites.append(site)
# Now check for existence of an appropriately named drawable. If none
# exists, throw. This stops a locale discovering, at runtime, that the
# corresponding drawable was not added to en-US.
if not opts.resources:
continue
resources = os.path.abspath(opts.resources)
finder = FileFinder(resources)
matches = [
p for p, _ in finder.find(drawables_template.format(name=name))
]
if not matches:
raise Exception(
"Could not find drawable in '{resources}' for '{name}'".
format(resources=resources, name=name))
else:
if opts.verbose:
print(
"Found {len} drawables in '{resources}' for '{name}': {matches}"
.format(len=len(matches),
resources=resources,
name=name,
matches=matches))
# We want the lists to be ordered for reproducibility. Each list has a
# "default" JSON list item which will be extended by the properties read.
lists = [
('browser.suggestedsites.list', {}),
('browser.suggestedsites.restricted.list', {
'restricted': True
}),
]
if opts.verbose:
print('Reading {len} suggested site lists: {lists}'.format(
len=len(lists), lists=[list_name for list_name, _ in lists]))
for (list_name, list_item_defaults) in lists:
names = properties.get_list(list_name)
if opts.verbose:
print('Reading {len} suggested sites from {list}: {names}'.format(
len=len(names), list=list_name, names=names))
add_names(names, list_item_defaults)
# FileAvoidWrite creates its parent directories.
output = os.path.abspath(opts.output)
fh = FileAvoidWrite(output)
json.dump(sites, fh)
existed, updated = fh.close()
if not opts.silent:
if updated:
print('{output} updated'.format(output=output))
else:
print('{output} already up-to-date'.format(output=output))
return 0
def _write_file(self, key, value):
filename = mozpath.join(self._datadir, key)
with FileAvoidWrite(filename) as fh:
to_write = json.dumps(value, indent=4)
fh.write(to_write.encode(system_encoding))
return filename
def main(argv):
parser = argparse.ArgumentParser('Generate a file from a Python script',
add_help=False)
parser.add_argument('--locale', metavar='locale', type=str,
help='The locale in use.')
parser.add_argument('python_script', metavar='python-script', type=str,
help='The Python script to run')
parser.add_argument('method_name', metavar='method-name', type=str,
help='The method of the script to invoke')
parser.add_argument('output_file', metavar='output-file', type=str,
help='The file to generate')
parser.add_argument('dep_file', metavar='dep-file', type=str,
help='File to write any additional make dependencies to')
parser.add_argument('dep_target', metavar='dep-target', type=str,
help='Make target to use in the dependencies file')
parser.add_argument('additional_arguments', metavar='arg',
nargs=argparse.REMAINDER,
help="Additional arguments to the script's main() method")
args = parser.parse_args(argv)
kwargs = {}
if args.locale:
kwargs['locale'] = args.locale
script = args.python_script
# Permit the script to import modules from the same directory in which it
# resides. The justification for doing this is that if we were invoking
# the script as:
#
# python script arg1...
#
# then importing modules from the script's directory would come for free.
# Since we're invoking the script in a roundabout way, we provide this
# bit of convenience.
sys.path.append(os.path.dirname(script))
with open(script, 'r') as fh:
module = imp.load_module('script', fh, script,
('.py', 'r', imp.PY_SOURCE))
method = args.method_name
if not hasattr(module, method):
print('Error: script "{0}" is missing a {1} method'.format(script, method),
file=sys.stderr)
return 1
ret = 1
try:
with FileAvoidWrite(args.output_file, mode='rb') as output:
try:
ret = module.__dict__[method](output, *args.additional_arguments, **kwargs)
except Exception:
# Ensure that we don't overwrite the file if the script failed.
output.avoid_writing_to_file()
raise
# The following values indicate a statement of success:
# - a set() (see below)
# - 0
# - False
# - None
#
# Everything else is an error (so scripts can conveniently |return
# 1| or similar). If a set is returned, the elements of the set
# indicate additional dependencies that will be listed in the deps
# file. Python module imports are automatically included as
# dependencies.
if isinstance(ret, set):
deps = ret
# The script succeeded, so reset |ret| to indicate that.
ret = None
else:
deps = set()
# Only write out the dependencies if the script was successful
if not ret:
# Add dependencies on any python modules that were imported by
# the script.
deps |= set(iter_modules_in_path(buildconfig.topsrcdir,
buildconfig.topobjdir))
# Add dependencies on any buildconfig items that were accessed
# by the script.
deps |= set(buildconfig.get_dependencies())
mk = Makefile()
mk.create_rule([args.dep_target]).add_dependencies(deps)
with FileAvoidWrite(args.dep_file) as dep_file:
mk.dump(dep_file)
else:
# Ensure that we don't overwrite the file if the script failed.
output.avoid_writing_to_file()
except IOError as e:
print('Error opening file "{0}"'.format(e.filename), file=sys.stderr)
traceback.print_exc()
return 1
return ret
def generate_build_files(self):
"""Generate files required for the build.
This function is in charge of generating all the .h/.cpp files derived
from input .webidl files. Please note that there are build actions
required to produce .webidl files and these build actions are
explicitly not captured here: this function assumes all .webidl files
are present and up to date.
This routine is called as part of the build to ensure files that need
to exist are present and up to date. This routine may not be called if
the build dependencies (generated as a result of calling this the first
time) say everything is up to date.
Because reprocessing outputs for every .webidl on every invocation
is expensive, we only regenerate the minimal set of files on every
invocation. The rules for deciding what needs done are roughly as
follows:
1. If any .webidl changes, reparse all .webidl files and regenerate
the global derived files. Only regenerate output files (.h/.cpp)
impacted by the modified .webidl files.
2. If an non-.webidl dependency (Python files, config file) changes,
assume everything is out of date and regenerate the world. This
is because changes in those could globally impact every output
file.
3. If an output file is missing, ensure it is present by performing
necessary regeneration.
"""
# Despite #1 above, we assume the build system is smart enough to not
# invoke us if nothing has changed. Therefore, any invocation means
# something has changed. And, if anything has changed, we need to
# parse the WebIDL.
self._parse_webidl()
result = BuildResult()
# If we parse, we always update globals - they are cheap and it is
# easier that way.
created, updated, unchanged = self._write_global_derived()
result.created |= created
result.updated |= updated
result.unchanged |= unchanged
# If any of the extra dependencies changed, regenerate the world.
global_changed, global_hashes = self._global_dependencies_changed()
if global_changed:
# Make a copy because we may modify.
changed_inputs = set(self._input_paths)
else:
changed_inputs = self._compute_changed_inputs()
self._state['global_depends'] = global_hashes
# Generate bindings from .webidl files.
for filename in sorted(changed_inputs):
basename = mozpath.basename(filename)
result.inputs.add(filename)
written, deps = self._generate_build_files_for_webidl(filename)
result.created |= written[0]
result.updated |= written[1]
result.unchanged |= written[2]
self._state['webidls'][basename] = dict(
filename=filename,
outputs=written[0] | written[1] | written[2],
inputs=set(deps),
sha1=self._input_hashes[filename],
)
# Process some special interfaces required for testing.
for interface in self._example_interfaces:
written = self.generate_example_files(interface)
result.created |= written[0]
result.updated |= written[1]
result.unchanged |= written[2]
# Generate a make dependency file.
if self._make_deps_path:
mk = Makefile()
codegen_rule = mk.create_rule([self._make_deps_target])
codegen_rule.add_dependencies(global_hashes.keys())
codegen_rule.add_dependencies(self._input_paths)
with FileAvoidWrite(self._make_deps_path) as fh:
mk.dump(fh)
self._save_state()
return result
def main(argv):
parser = argparse.ArgumentParser('Generate a file from a Python script',
add_help=False)
parser.add_argument('python_script',
metavar='python-script',
type=str,
help='The Python script to run')
parser.add_argument('method_name',
metavar='method-name',
type=str,
help='The method of the script to invoke')
parser.add_argument('output_file',
metavar='output-file',
type=str,
help='The file to generate')
parser.add_argument(
'dep_file',
metavar='dep-file',
type=str,
help='File to write any additional make dependencies to')
parser.add_argument(
'additional_arguments',
metavar='arg',
nargs='*',
help="Additional arguments to the script's main() method")
args = parser.parse_args(argv)
script = args.python_script
# Permit the script to import modules from the same directory in which it
# resides. The justification for doing this is that if we were invoking
# the script as:
#
# python script arg1...
#
# then importing modules from the script's directory would come for free.
# Since we're invoking the script in a roundabout way, we provide this
# bit of convenience.
sys.path.append(os.path.dirname(script))
with open(script, 'r') as fh:
module = imp.load_module('script', fh, script,
('.py', 'r', imp.PY_SOURCE))
method = args.method_name
if not hasattr(module, method):
print('Error: script "{0}" is missing a {1} method'.format(
script, method),
file=sys.stderr)
return 1
ret = 1
try:
with FileAvoidWrite(args.output_file) as output:
ret = module.__dict__[method](output, *args.additional_arguments)
# We treat sets as a statement of success. Everything else
# is an error (so scripts can conveniently |return 1| or
# similar).
if isinstance(ret, set) and ret:
mk = Makefile()
mk.create_rule([args.output_file]).add_dependencies(ret)
with FileAvoidWrite(args.dep_file) as dep_file:
mk.dump(dep_file)
# The script succeeded, so reset |ret| to indicate that.
ret = None
except IOError as e:
print('Error opening file "{0}"'.format(e.filename), file=sys.stderr)
traceback.print_exc()
return 1
return ret
