def main():
parser = optparse.OptionParser()
parser.add_option('--jars', help='The jars to merge.')
parser.add_option('--jar-path', help='The output merged jar file.')
options, _ = parser.parse_args()
with build_utils.TempDir() as temp_dir:
for jar_file in build_utils.ParseGypList(options.jars):
build_utils.ExtractAll(jar_file, path=temp_dir, pattern='*.class')
jar.JarDirectory(temp_dir, [], options.jar_path)
def _CreateJarFile(jar_path, provider_configurations, additional_jar_files,
classes_dir):
logging.info('Start creating jar file: %s', jar_path)
with build_utils.AtomicOutput(jar_path) as f:
jar.JarDirectory(
classes_dir,
f.name,
# Avoid putting generated java files into the jar since
# _MoveGeneratedJavaFilesToGenDir has not completed yet
predicate=lambda name: not name.endswith('.java'),
provider_configurations=provider_configurations,
additional_files=additional_jar_files)
logging.info('Completed jar file: %s', jar_path)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('jars',
nargs=argparse.REMAINDER,
help='The jars to merge.')
parser.add_argument('--output-jar', help='Name of the merged JAR file.')
parser.add_argument(
'--validate-skipped-jars-list',
action='store_true',
help='Whether to validate KNOWN_SKIPPED_JARS by making '
'sure it matches all the jars passed in --jars that are '
'being skipped.')
options = parser.parse_args(build_utils.ExpandFileArgs(sys.argv[1:]))
jars = []
for j in options.jars:
jars.extend(build_utils.ParseGypList(j))
options.jars = jars
if options.validate_skipped_jars_list:
extra, missing = ValidateKnownSkippedJars(options.jars)
# It is fine for |extra| not to be empty: different build options may
# include fewer JARs. |missing| being non-empty is fatal, though, as it
# means there will be problems for users since we are not skipping files
# that we should.
if extra:
print
print 'merge_jars.py: The following JARs in KNOWN_SKIPPED_JARS were ' \
'not used:'
print ' %s' % ', '.join(sorted(extra))
print
if missing:
print
print 'merge_jars.py: The following JARs are not mergeable but are ' \
'not part of KNOWN_SKIPPED_JARS:'
print ' %s' % ', '.join(sorted(missing))
print
return 1
with build_utils.TempDir() as temp_dir:
for jar_file in options.jars:
# If a JAR has classes outside our allowed namespaces (mostly
# org.chromium and org.xwalk), we need to skip it otherwise there can be
# build issues when a user builds an app with Crosswalk as well as
# another package with one of these non-allowed namespaces (see
# XWALK-5092, XWALK-6597).
if not IsMergeableJar(jar_file):
continue
build_utils.ExtractAll(jar_file, path=temp_dir, pattern='*.class')
jar.JarDirectory(temp_dir, options.output_jar)
def main():
parser = optparse.OptionParser()
parser.add_option('--build-dir',
help='Base build directory, such as out/Release. JARs '
'outside this directory will be skipped.')
parser.add_option('--jars', help='The jars to merge.')
parser.add_option('--output-jar', help='Name of the merged JAR file.')
options, _ = parser.parse_args()
build_dir = os.path.abspath(options.build_dir)
with build_utils.TempDir() as temp_dir:
for jar_file in build_utils.ParseGypList(options.jars):
if not os.path.abspath(jar_file).startswith(build_dir):
continue
build_utils.ExtractAll(jar_file, path=temp_dir, pattern='*.class')
jar.JarDirectory(temp_dir, [], options.output_jar)
def _OnStaleMd5(changes, options, javac_cmd, java_files, classpath_inputs,
classpath):
# Don't bother enabling incremental compilation for non-chromium code.
incremental = options.incremental and options.chromium_code
with build_utils.TempDir() as temp_dir:
srcjars = options.java_srcjars
classes_dir = os.path.join(temp_dir, 'classes')
os.makedirs(classes_dir)
changed_paths = None
# jmake can handle deleted files, but it's a rare case and it would
# complicate this script's logic.
if incremental and changes.AddedOrModifiedOnly():
changed_paths = set(changes.IterChangedPaths())
# Do a full compile if classpath has changed.
# jmake doesn't seem to do this on its own... Might be that ijars mess up
# its change-detection logic.
if any(p in changed_paths for p in classpath_inputs):
changed_paths = None
if options.incremental:
pdb_path = options.jar_path + '.pdb'
if incremental:
# jmake is a compiler wrapper that figures out the minimal set of .java
# files that need to be rebuilt given a set of .java files that have
# changed.
# jmake determines what files are stale based on timestamps between .java
# and .class files. Since we use .jars, .srcjars, and md5 checks,
# timestamp info isn't accurate for this purpose. Rather than use jmake's
# programatic interface (like we eventually should), we ensure that all
# .class files are newer than their .java files, and convey to jmake which
# sources are stale by having their .class files be missing entirely
# (by not extracting them).
javac_cmd = _ConvertToJMakeArgs(javac_cmd, pdb_path)
if srcjars:
_FixTempPathsInIncrementalMetadata(pdb_path, temp_dir)
srcjar_files = dict()
if srcjars:
java_dir = os.path.join(temp_dir, 'java')
os.makedirs(java_dir)
for srcjar in options.java_srcjars:
if changed_paths:
changed_paths.update(os.path.join(java_dir, f)
for f in changes.IterChangedSubpaths(srcjar))
extracted_files = build_utils.ExtractAll(
srcjar, path=java_dir, pattern='*.java')
for path in extracted_files:
srcjar_files[path] = srcjar
jar_srcs = build_utils.FindInDirectory(java_dir, '*.java')
java_files.extend(jar_srcs)
if changed_paths:
# Set the mtime of all sources to 0 since we use the absence of .class
# files to tell jmake which files are stale.
for path in jar_srcs:
os.utime(path, (0, 0))
_CreateInfoFile(java_files, options, srcjar_files)
if java_files:
if changed_paths:
changed_java_files = [p for p in java_files if p in changed_paths]
if os.path.exists(options.jar_path):
_ExtractClassFiles(options.jar_path, classes_dir, changed_java_files)
# Add the extracted files to the classpath. This is required because
# when compiling only a subset of files, classes that haven't changed
# need to be findable.
classpath.append(classes_dir)
# Can happen when a target goes from having no sources, to having sources.
# It's created by the call to build_utils.Touch() below.
if incremental:
if os.path.exists(pdb_path) and not os.path.getsize(pdb_path):
os.unlink(pdb_path)
# Don't include the output directory in the initial set of args since it
# being in a temp dir makes it unstable (breaks md5 stamping).
cmd = javac_cmd + ['-d', classes_dir]
# Pass classpath and source paths as response files to avoid extremely
# long command lines that are tedius to debug.
if classpath:
cmd += ['-classpath', ':'.join(classpath)]
java_files_rsp_path = os.path.join(temp_dir, 'files_list.txt')
with open(java_files_rsp_path, 'w') as f:
f.write(' '.join(java_files))
cmd += ['@' + java_files_rsp_path]
# JMake prints out some diagnostic logs that we want to ignore.
# This assumes that all compiler output goes through stderr.
stdout_filter = lambda s: ''
if md5_check.PRINT_EXPLANATIONS:
stdout_filter = None
attempt_build = lambda: build_utils.CheckOutput(
cmd,
print_stdout=options.chromium_code,
stdout_filter=stdout_filter,
stderr_filter=ProcessJavacOutput)
try:
attempt_build()
except build_utils.CalledProcessError as e:
# Work-around for a bug in jmake (http://crbug.com/551449).
if 'project database corrupted' not in e.output:
raise
print ('Applying work-around for jmake project database corrupted '
'(http://crbug.com/551449).')
os.unlink(pdb_path)
attempt_build()
if options.incremental and (not java_files or not incremental):
# Make sure output exists.
build_utils.Touch(pdb_path)
jar.JarDirectory(classes_dir,
options.jar_path,
provider_configurations=options.provider_configurations,
additional_files=options.additional_jar_files)
def main(argv):
colorama.init()
argv = build_utils.ExpandFileArgs(argv)
parser = optparse.OptionParser()
build_utils.AddDepfileOption(parser)
parser.add_option(
'--src-gendirs',
help='Directories containing generated java files.')
parser.add_option(
'--java-srcjars',
action='append',
default=[],
help='List of srcjars to include in compilation.')
parser.add_option(
'--bootclasspath',
action='append',
default=[],
help='Boot classpath for javac. If this is specified multiple times, '
'they will all be appended to construct the classpath.')
parser.add_option(
'--classpath',
action='append',
help='Classpath for javac. If this is specified multiple times, they '
'will all be appended to construct the classpath.')
parser.add_option(
'--javac-includes',
help='A list of file patterns. If provided, only java files that match'
'one of the patterns will be compiled.')
parser.add_option(
'--jar-excluded-classes',
default='',
help='List of .class file patterns to exclude from the jar.')
parser.add_option(
'--chromium-code',
type='int',
help='Whether code being compiled should be built with stricter '
'warnings for chromium code.')
parser.add_option(
'--use-errorprone-path',
help='Use the Errorprone compiler at this path.')
parser.add_option(
'--classes-dir',
help='Directory for compiled .class files.')
parser.add_option('--jar-path', help='Jar output path.')
parser.add_option(
'--main-class',
help='The class containing the main method.')
parser.add_option(
'--manifest-entry',
action='append',
help='Key:value pairs to add to the .jar manifest.')
parser.add_option('--stamp', help='Path to touch on success.')
options, args = parser.parse_args(argv)
if options.main_class and not options.jar_path:
parser.error('--main-class requires --jar-path')
bootclasspath = []
for arg in options.bootclasspath:
bootclasspath += build_utils.ParseGypList(arg)
classpath = []
for arg in options.classpath:
classpath += build_utils.ParseGypList(arg)
java_srcjars = []
for arg in options.java_srcjars:
java_srcjars += build_utils.ParseGypList(arg)
java_files = args
if options.src_gendirs:
src_gendirs = build_utils.ParseGypList(options.src_gendirs)
java_files += build_utils.FindInDirectories(src_gendirs, '*.java')
input_files = bootclasspath + classpath + java_srcjars + java_files
with build_utils.TempDir() as temp_dir:
classes_dir = os.path.join(temp_dir, 'classes')
os.makedirs(classes_dir)
if java_srcjars:
java_dir = os.path.join(temp_dir, 'java')
os.makedirs(java_dir)
for srcjar in java_srcjars:
build_utils.ExtractAll(srcjar, path=java_dir, pattern='*.java')
java_files += build_utils.FindInDirectory(java_dir, '*.java')
if options.javac_includes:
javac_includes = build_utils.ParseGypList(options.javac_includes)
filtered_java_files = []
for f in java_files:
for include in javac_includes:
if fnmatch.fnmatch(f, include):
filtered_java_files.append(f)
break
java_files = filtered_java_files
if len(java_files) != 0:
DoJavac(
bootclasspath,
classpath,
classes_dir,
options.chromium_code,
options.use_errorprone_path,
java_files)
if options.jar_path:
if options.main_class or options.manifest_entry:
if options.manifest_entry:
entries = map(lambda e: e.split(":"), options.manifest_entry)
else:
entries = []
manifest_file = os.path.join(temp_dir, 'manifest')
CreateManifest(manifest_file, classpath, options.main_class, entries)
else:
manifest_file = None
jar.JarDirectory(classes_dir,
build_utils.ParseGypList(options.jar_excluded_classes),
options.jar_path,
manifest_file=manifest_file)
if options.classes_dir:
# Delete the old classes directory. This ensures that all .class files in
# the output are actually from the input .java files. For example, if a
# .java file is deleted or an inner class is removed, the classes
# directory should not contain the corresponding old .class file after
# running this action.
build_utils.DeleteDirectory(options.classes_dir)
shutil.copytree(classes_dir, options.classes_dir)
if options.depfile:
build_utils.WriteDepfile(
options.depfile,
input_files + build_utils.GetPythonDependencies())
if options.stamp:
build_utils.Touch(options.stamp)
def _OnStaleMd5(changes, options, javac_cmd, java_files, classpath_inputs,
runtime_classpath):
with build_utils.TempDir() as temp_dir:
srcjars = options.java_srcjars
# The .excluded.jar contains .class files excluded from the main jar.
# It is used for incremental compiles.
excluded_jar_path = options.jar_path.replace('.jar', '.excluded.jar')
classes_dir = os.path.join(temp_dir, 'classes')
os.makedirs(classes_dir)
changed_paths = None
# jmake can handle deleted files, but it's a rare case and it would
# complicate this script's logic.
if options.incremental and changes.AddedOrModifiedOnly():
changed_paths = set(changes.IterChangedPaths())
# Do a full compile if classpath has changed.
# jmake doesn't seem to do this on its own... Might be that ijars mess up
# its change-detection logic.
if any(p in changed_paths for p in classpath_inputs):
changed_paths = None
if options.incremental:
# jmake is a compiler wrapper that figures out the minimal set of .java
# files that need to be rebuilt given a set of .java files that have
# changed.
# jmake determines what files are stale based on timestamps between .java
# and .class files. Since we use .jars, .srcjars, and md5 checks,
# timestamp info isn't accurate for this purpose. Rather than use jmake's
# programatic interface (like we eventually should), we ensure that all
# .class files are newer than their .java files, and convey to jmake which
# sources are stale by having their .class files be missing entirely
# (by not extracting them).
pdb_path = options.jar_path + '.pdb'
javac_cmd = _ConvertToJMakeArgs(javac_cmd, pdb_path)
if srcjars:
_FixTempPathsInIncrementalMetadata(pdb_path, temp_dir)
if srcjars:
java_dir = os.path.join(temp_dir, 'java')
os.makedirs(java_dir)
for srcjar in options.java_srcjars:
if changed_paths:
changed_paths.update(os.path.join(java_dir, f)
for f in changes.IterChangedSubpaths(srcjar))
build_utils.ExtractAll(srcjar, path=java_dir, pattern='*.java')
jar_srcs = build_utils.FindInDirectory(java_dir, '*.java')
jar_srcs = _FilterJavaFiles(jar_srcs, options.javac_includes)
java_files.extend(jar_srcs)
if changed_paths:
# Set the mtime of all sources to 0 since we use the absense of .class
# files to tell jmake which files are stale.
for path in jar_srcs:
os.utime(path, (0, 0))
if java_files:
if changed_paths:
changed_java_files = [p for p in java_files if p in changed_paths]
if os.path.exists(options.jar_path):
_ExtractClassFiles(options.jar_path, classes_dir, changed_java_files)
if os.path.exists(excluded_jar_path):
_ExtractClassFiles(excluded_jar_path, classes_dir, changed_java_files)
# Add the extracted files to the classpath. This is required because
# when compiling only a subset of files, classes that haven't changed
# need to be findable.
classpath_idx = javac_cmd.index('-classpath')
javac_cmd[classpath_idx + 1] += ':' + classes_dir
# Don't include the output directory in the initial set of args since it
# being in a temp dir makes it unstable (breaks md5 stamping).
cmd = javac_cmd + ['-d', classes_dir] + java_files
# JMake prints out some diagnostic logs that we want to ignore.
# This assumes that all compiler output goes through stderr.
stdout_filter = lambda s: ''
if md5_check.PRINT_EXPLANATIONS:
stdout_filter = None
build_utils.CheckOutput(
cmd,
print_stdout=options.chromium_code,
stdout_filter=stdout_filter,
stderr_filter=ColorJavacOutput)
if options.main_class or options.manifest_entry:
entries = []
if options.manifest_entry:
entries = [e.split(':') for e in options.manifest_entry]
manifest_file = os.path.join(temp_dir, 'manifest')
_CreateManifest(manifest_file, runtime_classpath, options.main_class,
entries)
else:
manifest_file = None
glob = options.jar_excluded_classes
inclusion_predicate = lambda f: not build_utils.MatchesGlob(f, glob)
exclusion_predicate = lambda f: not inclusion_predicate(f)
jar.JarDirectory(classes_dir,
options.jar_path,
manifest_file=manifest_file,
predicate=inclusion_predicate)
jar.JarDirectory(classes_dir,
excluded_jar_path,
predicate=exclusion_predicate)
def _OnStaleMd5(changes, options, javac_cmd, java_files, classpath_inputs):
incremental = options.incremental
# Don't bother enabling incremental compilation for third_party code, since
# _CheckPathMatchesClassName() fails on some of it, and it's not really much
# benefit.
for java_file in java_files:
if 'third_party' in java_file:
incremental = False
else:
_CheckPathMatchesClassName(java_file)
with build_utils.TempDir() as temp_dir:
srcjars = options.java_srcjars
# The .excluded.jar contains .class files excluded from the main jar.
# It is used for incremental compiles.
excluded_jar_path = options.jar_path.replace('.jar', '.excluded.jar')
classes_dir = os.path.join(temp_dir, 'classes')
os.makedirs(classes_dir)
changed_paths = None
# jmake can handle deleted files, but it's a rare case and it would
# complicate this script's logic.
if incremental and changes.AddedOrModifiedOnly():
changed_paths = set(changes.IterChangedPaths())
# Do a full compile if classpath has changed.
# jmake doesn't seem to do this on its own... Might be that ijars mess up
# its change-detection logic.
if any(p in changed_paths for p in classpath_inputs):
changed_paths = None
if incremental:
# jmake is a compiler wrapper that figures out the minimal set of .java
# files that need to be rebuilt given a set of .java files that have
# changed.
# jmake determines what files are stale based on timestamps between .java
# and .class files. Since we use .jars, .srcjars, and md5 checks,
# timestamp info isn't accurate for this purpose. Rather than use jmake's
# programatic interface (like we eventually should), we ensure that all
# .class files are newer than their .java files, and convey to jmake which
# sources are stale by having their .class files be missing entirely
# (by not extracting them).
pdb_path = options.jar_path + '.pdb'
javac_cmd = _ConvertToJMakeArgs(javac_cmd, pdb_path)
if srcjars:
_FixTempPathsInIncrementalMetadata(pdb_path, temp_dir)
if srcjars:
java_dir = os.path.join(temp_dir, 'java')
os.makedirs(java_dir)
for srcjar in options.java_srcjars:
if changed_paths:
changed_paths.update(
os.path.join(java_dir, f)
for f in changes.IterChangedSubpaths(srcjar))
build_utils.ExtractAll(srcjar, path=java_dir, pattern='*.java')
jar_srcs = build_utils.FindInDirectory(java_dir, '*.java')
jar_srcs = _FilterJavaFiles(jar_srcs, options.javac_includes)
java_files.extend(jar_srcs)
if changed_paths:
# Set the mtime of all sources to 0 since we use the absense of .class
# files to tell jmake which files are stale.
for path in jar_srcs:
os.utime(path, (0, 0))
if java_files:
if changed_paths:
changed_java_files = [
p for p in java_files if p in changed_paths
]
if os.path.exists(options.jar_path):
_ExtractClassFiles(options.jar_path, classes_dir,
changed_java_files)
if os.path.exists(excluded_jar_path):
_ExtractClassFiles(excluded_jar_path, classes_dir,
changed_java_files)
# Add the extracted files to the classpath. This is required because
# when compiling only a subset of files, classes that haven't changed
# need to be findable.
classpath_idx = javac_cmd.index('-classpath')
javac_cmd[classpath_idx + 1] += ':' + classes_dir
# Can happen when a target goes from having no sources, to having sources.
# It's created by the call to build_utils.Touch() below.
if incremental:
if os.path.exists(pdb_path) and not os.path.getsize(pdb_path):
os.unlink(pdb_path)
# Don't include the output directory in the initial set of args since it
# being in a temp dir makes it unstable (breaks md5 stamping).
cmd = javac_cmd + ['-d', classes_dir] + java_files
# JMake prints out some diagnostic logs that we want to ignore.
# This assumes that all compiler output goes through stderr.
stdout_filter = lambda s: ''
if md5_check.PRINT_EXPLANATIONS:
stdout_filter = None
attempt_build = lambda: build_utils.CheckOutput(
cmd,
print_stdout=options.chromium_code,
stdout_filter=stdout_filter,
stderr_filter=ColorJavacOutput)
try:
attempt_build()
except build_utils.CalledProcessError as e:
# Work-around for a bug in jmake (http://crbug.com/551449).
if 'project database corrupted' not in e.output:
raise
print(
'Applying work-around for jmake project database corrupted '
'(http://crbug.com/551449).')
os.unlink(pdb_path)
attempt_build()
elif incremental:
# Make sure output exists.
build_utils.Touch(pdb_path)
glob = options.jar_excluded_classes
inclusion_predicate = lambda f: not build_utils.MatchesGlob(f, glob)
exclusion_predicate = lambda f: not inclusion_predicate(f)
jar.JarDirectory(
classes_dir,
options.jar_path,
predicate=inclusion_predicate,
provider_configurations=options.provider_configurations,
additional_files=options.additional_jar_files)
jar.JarDirectory(
classes_dir,
excluded_jar_path,
predicate=exclusion_predicate,
provider_configurations=options.provider_configurations,
additional_files=options.additional_jar_files)
def _OnStaleMd5(changes, options, javac_cmd, java_files, classpath_inputs,
runtime_classpath):
with build_utils.TempDir() as temp_dir:
srcjars = options.java_srcjars
# The .excluded.jar contains .class files excluded from the main jar.
# It is used for incremental compiles.
excluded_jar_path = options.jar_path.replace('.jar', '.excluded.jar')
classes_dir = os.path.join(temp_dir, 'classes')
os.makedirs(classes_dir)
changed_paths = None
if options.incremental and changes.AddedOrModifiedOnly():
changed_paths = set(changes.IterChangedPaths())
# Do a full compile if classpath has changed.
if any(p in changed_paths for p in classpath_inputs):
changed_paths = None
else:
java_files = [p for p in java_files if p in changed_paths]
srcjars = [p for p in srcjars if p in changed_paths]
if srcjars:
java_dir = os.path.join(temp_dir, 'java')
os.makedirs(java_dir)
for srcjar in options.java_srcjars:
extract_predicate = None
if changed_paths:
changed_subpaths = set(changes.IterChangedSubpaths(srcjar))
extract_predicate = lambda p: p in changed_subpaths
build_utils.ExtractAll(srcjar,
path=java_dir,
pattern='*.java',
predicate=extract_predicate)
jar_srcs = build_utils.FindInDirectory(java_dir, '*.java')
java_files.extend(
_FilterJavaFiles(jar_srcs, options.javac_includes))
if java_files:
if changed_paths:
# When no files have been removed and the output jar already
# exists, reuse .class files from the existing jar.
_ExtractClassFiles(options.jar_path, classes_dir, java_files)
_ExtractClassFiles(excluded_jar_path, classes_dir, java_files)
# Add the extracted files to the classpath.
classpath_idx = javac_cmd.index('-classpath')
javac_cmd[classpath_idx + 1] += ':' + classes_dir
# Don't include the output directory in the initial set of args since it
# being in a temp dir makes it unstable (breaks md5 stamping).
cmd = javac_cmd + ['-d', classes_dir] + java_files
build_utils.CheckOutput(cmd,
print_stdout=options.chromium_code,
stderr_filter=ColorJavacOutput)
if options.main_class or options.manifest_entry:
entries = []
if options.manifest_entry:
entries = [e.split(':') for e in options.manifest_entry]
manifest_file = os.path.join(temp_dir, 'manifest')
_CreateManifest(manifest_file, runtime_classpath,
options.main_class, entries)
else:
manifest_file = None
glob = options.jar_excluded_classes
inclusion_predicate = lambda f: not build_utils.MatchesGlob(f, glob)
exclusion_predicate = lambda f: not inclusion_predicate(f)
jar.JarDirectory(classes_dir,
options.jar_path,
manifest_file=manifest_file,
predicate=inclusion_predicate)
jar.JarDirectory(classes_dir,
excluded_jar_path,
predicate=exclusion_predicate)
