def save(self):
"""
Pickles this module descriptor to a file if it corresponds to a distribution.
Otherwise, does nothing.
:return: the path to which this module descriptor was pickled or None
"""
dist = self.dist
if not dist:
# Don't pickle a JDK module
return None
_, moduleDir, _ = get_java_module_info(dist, fatalIfNotModule=True) # pylint: disable=unpacking-non-sequence
path = moduleDir + '.pickled'
modulepath = self.modulepath
jarpath = self.jarpath
self.modulepath = [
m.name if not m.dist else 'dist:' + m.dist.name for m in modulepath
]
self.dist = dist.name
self.jarpath = os.path.relpath(jarpath, dirname(path))
try:
with mx.SafeFileCreation(path) as sfc, open(sfc.tmpPath,
'wb') as f:
pickle.dump(self, f)
finally:
self.modulepath = modulepath
self.dist = dist
self.jarpath = jarpath
def _tck(args):
"""runs TCK tests"""
parser = ArgumentParser(prog="mx tck", description="run the TCK tests", formatter_class=RawDescriptionHelpFormatter, epilog=_debuggertestHelpSuffix)
parser.add_argument("--tck-configuration", help="TCK configuration", choices=["compile", "debugger", "default"], default="default")
parsed_args, args = parser.parse_known_args(args)
tckConfiguration = parsed_args.tck_configuration
index = len(args)
for arg in reversed(args):
if arg.startswith("-"):
break
index = index - 1
args_no_tests = args[0:index]
tests = args[index:len(args)]
if len(tests) == 0:
tests = ["com.oracle.truffle.tck.tests"]
index = len(args_no_tests)
has_separator_arg = False
for arg in reversed(args_no_tests):
if arg.startswith("--"):
if arg == "--":
has_separator_arg = True
break
index = index - 1
unitTestOptions = args_no_tests[0:max(index - (1 if has_separator_arg else 0), 0)]
jvmOptions = args_no_tests[index:len(args_no_tests)]
if tckConfiguration == "default":
unittest(unitTestOptions + ["--"] + jvmOptions + tests)
elif tckConfiguration == "debugger":
with mx.SafeFileCreation(os.path.join(tempfile.gettempdir(), "debugalot")) as sfc:
_execute_debugger_test(tests, sfc.tmpPath, False, unitTestOptions, jvmOptions)
elif tckConfiguration == "compile":
if not _is_graalvm(mx.get_jdk()):
mx.abort("The 'compile' TCK configuration requires graalvm execution, run with --java-home=<path_to_graalvm>.")
unittest(unitTestOptions + ["--"] + jvmOptions + ["-Dgraal.TruffleCompileImmediately=true", "-Dgraal.TruffleCompilationExceptionsAreThrown=true"] + tests)
def _tck(args):
"""runs TCK tests"""
parser = ArgumentParser(prog="mx tck", description="run the TCK tests", formatter_class=RawDescriptionHelpFormatter, epilog=_debuggertestHelpSuffix)
parser.add_argument("--tck-configuration", help="TCK configuration", choices=["default", "debugger"], default="default")
parsed_args, args = parser.parse_known_args(args)
tckConfiguration = parsed_args.tck_configuration
index = len(args)
for arg in reversed(args):
if arg.startswith("-"):
break
index = index - 1
args_no_tests = args[0:index]
tests = args[index:len(args)]
if len(tests) == 0:
tests = ["com.oracle.truffle.tck.tests"]
index = len(args_no_tests)
for arg in reversed(args_no_tests):
if arg.startswith("--"):
break
index = index - 1
unitTestOptions = args_no_tests[0:max(index-1, 0)]
jvmOptions = args_no_tests[index:len(args_no_tests)]
if tckConfiguration == "default":
unittest(unitTestOptions + ["--"] + jvmOptions + tests)
elif tckConfiguration == "debugger":
with mx.SafeFileCreation(os.path.join(tempfile.gettempdir(), "debugalot")) as sfc:
_execute_debugger_test(tests, sfc.tmpPath, False, unitTestOptions, jvmOptions)
def getTestFile(self):
if not hasattr(self, '_testfile'):
self._testfile = os.path.join(self.out_dir, 'tests.cache')
with mx.SafeFileCreation(self._testfile) as sfc, open(sfc.tmpPath, "w") as f:
mx.logv("Writing test file: " + self._testfile)
f.write('set(SULONG_TESTS {} CACHE FILEPATH "test files")'.format(';'.join(self.getTests())))
return self._testfile
def _zip_files(files, baseDir, zipPath):
with mx.SafeFileCreation(zipPath) as sfc:
zf = zipfile.ZipFile(sfc.tmpPath, 'w')
for f in sorted(set(files)):
relpath = os.path.relpath(f, baseDir)
arcname = relpath.replace(os.sep, '/')
zf.write(f, arcname)
zf.close()
def build(self):
if not mx.exists(self._manifest) \
or self._reason is None \
or mx.basename(self._manifest) in self._reason \
or 'phony' in self._reason:
with mx.SafeFileCreation(self._manifest) as sfc:
self.subject.generate_manifest(sfc.tmpPath)
if mx.exists(self._manifest) \
and not filecmp.cmp(self._manifest, sfc.tmpPath, shallow=False):
self.ninja.clean()
self.ninja.build()
def _write_guard(self, source_dir, cmake_config):
with mx.SafeFileCreation(self.guard_file()) as sfc:
with open(sfc.tmpPath, 'w') as fp:
fp.write(self._guard_data(source_dir, cmake_config))
def make_java_module(dist, jdk):
"""
Creates a Java module from a distribution.
This updates the JAR by adding `module-info` classes.
The `META-INF` directory can not be versioned. However, we make an exception here for `META-INF/services`:
if different versions should have different service providers, a `META-INF/_versions/<version>/META-INF/services`
directory can be used (note the `_` before `versions`).
These service provider declarations will be used to build the versioned module-info files and the
`META-INF/_versions/<version>` directories will be removed from the archive.
This is done using a separate versioning directory so that the JAR can be a valid multi-release JAR before this
transformation.
input:
com/foo/MyProvider.class # JDK 8 or earlier specific provider
META-INF/services/com.foo.MyService # Contains: com.foo.MyProvider
META-INF/_versions/9/META-INF/services/com.foo.MyService # Contains: com.foo.MyProvider
META-INF/versions/9/com/foo/MyProvider.class # JDK 9 and 10 specific provider
META-INF/_versions/11/META-INF/services/com.foo.MyService # Contains: provides com.foo.MyService with com.foo.MyProvider
META-INF/versions/11/com/foo/MyProvider.class # JDK 11 and later specific provider
output:
com/foo/MyProvider.class # JDK 8 or earlier specific provider
META-INF/services/com.foo.MyService # Contains: com.foo.MyProvider
META-INF/versions/9/module-info.class # Contains: provides com.foo.MyService with com.foo.MyProvider
META-INF/versions/9/com/foo/MyProvider.class # JDK 9 and 10 specific provider
META-INF/versions/11/module-info.class # Contains: provides com.foo.MyService with com.foo.MyProvider
META-INF/versions/11/com/foo/MyProvider.class # JDK 11 and later specific provider
:param JARDistribution dist: the distribution from which to create a module
:param JDKConfig jdk: a JDK with a version >= 9 that can be used to compile the module-info class
:return: the `JavaModuleDescriptor` for the created Java module
"""
info = get_java_module_info(dist)
if info is None:
return None
moduleName, _, moduleJar = info # pylint: disable=unpacking-non-sequence
mx.log('Building Java module ' + moduleName + ' from ' + dist.name)
exports = {}
requires = {}
concealedRequires = {}
base_uses = set()
modulepath = list()
usedModules = set()
if dist.suite.getMxCompatibility().moduleDepsEqualDistDeps():
moduledeps = dist.archived_deps()
for dep in mx.classpath_entries(dist, includeSelf=False):
if dep.isJARDistribution():
jmd = as_java_module(dep, jdk)
modulepath.append(jmd)
requires[jmd.name] = {jdk.get_transitive_requires_keyword()}
elif (dep.isJdkLibrary()
or dep.isJreLibrary()) and dep.is_provided_by(jdk):
pass
elif dep.isLibrary():
jmd = get_library_as_module(dep, jdk)
modulepath.append(jmd)
requires[jmd.name] = set()
else:
mx.abort(dist.name + ' cannot depend on ' + dep.name +
' as it does not define a module')
else:
moduledeps = get_module_deps(dist)
# Append JDK modules to module path
jdkModules = jdk.get_modules()
if not isinstance(jdkModules, list):
jdkModules = list(jdkModules)
allmodules = modulepath + jdkModules
javaprojects = [d for d in moduledeps if d.isJavaProject()]
# Collect packages in the module first
packages = set()
for dep in javaprojects:
packages.update(dep.defined_java_packages())
for dep in javaprojects:
base_uses.update(getattr(dep, 'uses', []))
for pkg in getattr(dep, 'runtimeDeps', []):
requires.setdefault(pkg, {'static'})
for pkg in itertools.chain(
dep.imported_java_packages(projectDepsOnly=False),
getattr(dep, 'imports', [])):
# Only consider packages not defined by the module we're creating. This handles the
# case where we're creating a module that will upgrade an existing upgradeable
# module in the JDK such as jdk.internal.vm.compiler.
if pkg not in packages:
depModule, visibility = lookup_package(allmodules, pkg,
moduleName)
if depModule and depModule.name != moduleName:
requires.setdefault(depModule.name, set())
if visibility == 'exported':
# A distribution based module does not re-export its imported JDK packages
usedModules.add(depModule)
else:
assert visibility == 'concealed'
concealedRequires.setdefault(depModule.name,
set()).add(pkg)
usedModules.add(depModule)
# If an "exports" attribute is not present, all packages are exported
for package in _expand_package_info(
dep, getattr(dep, 'exports', dep.defined_java_packages())):
if ' to ' in package:
splitpackage = package.split(' to ')
package = splitpackage[0].strip()
if not package:
mx.abort(
'exports attribute cannot have empty package value',
context=dist)
targets = [n.strip() for n in splitpackage[1].split(',')]
if not targets:
mx.abort(
'exports attribute must have at least one target for qualified export',
context=dist)
exports.setdefault(package, targets)
else:
exports.setdefault(package, [])
work_directory = mkdtemp()
try:
files_to_remove = set()
# To compile module-info.java, all classes it references must either be given
# as Java source files or already exist as class files in the output directory.
# As such, the jar file for each constituent distribution must be unpacked
# in the output directory.
versions = {}
for d in [dist] + [md for md in moduledeps if md.isJARDistribution()]:
if d.isJARDistribution():
with zipfile.ZipFile(d.original_path(), 'r') as zf:
for arcname in sorted(zf.namelist()):
m = _versioned_re.match(arcname)
if m:
version = m.group(1)
unversioned_name = m.group(2)
if version <= jdk.javaCompliance:
versions.setdefault(
version, {})[unversioned_name] = arcname
else:
# Ignore resource whose version is too high
pass
if unversioned_name.startswith(
'META-INF/services/'):
files_to_remove.add(arcname)
elif unversioned_name.startswith('META-INF/'):
mx.abort(
"META-INF resources can not be versioned and will make modules fail to load ({})."
.format(arcname))
default_jmd = None
all_versions = set(versions.keys())
if '9' not in all_versions:
# 9 is the first version that supports modules and can be versioned in the JAR:
# if there is no `META-INF/versions/9` then we should add a `module-info.class` to the root of the JAR
# so that the module works on JDK 9.
all_versions = all_versions | {'common'}
default_version = 'common'
else:
default_version = str(max((int(v) for v in all_versions)))
for version in all_versions:
uses = base_uses.copy()
provides = {}
dest_dir = join(work_directory, version)
int_version = int(version) if version != 'common' else -1
for d in [dist
] + [md for md in moduledeps if md.isJARDistribution()]:
if d.isJARDistribution():
with zipfile.ZipFile(d.original_path(), 'r') as zf:
for name in zf.namelist():
m = _versioned_re.match(name)
if m:
file_version = int(m.group(1))
if file_version > int_version:
continue
unversioned_name = m.group(2)
if name.startswith(_special_versioned_prefix):
if not unversioned_name.startswith(
'META-INF/services'):
raise mx.abort(
"The special versioned directory ({}) is only supported for META-INF/services files. Got {}"
.format(_special_versioned_prefix,
name))
if unversioned_name:
dst = join(dest_dir, unversioned_name)
parent = dirname(dst)
if parent and not exists(parent):
os.makedirs(parent)
with open(dst, 'wb') as fp:
fp.write(zf.read(name))
else:
zf.extract(name, dest_dir)
servicesDir = join(dest_dir, 'META-INF', 'services')
if exists(servicesDir):
for servicePathName in os.listdir(servicesDir):
# While a META-INF provider configuration file must use a fully qualified binary
# name[1] of the service, a provides directive in a module descriptor must use
# the fully qualified non-binary name[2] of the service.
#
# [1] https://docs.oracle.com/javase/9/docs/api/java/util/ServiceLoader.html
# [2] https://docs.oracle.com/javase/9/docs/api/java/lang/module/ModuleDescriptor.Provides.html#service--
service = servicePathName.replace('$', '.')
assert '/' not in service
with open(join(servicesDir,
servicePathName)) as fp:
serviceContent = fp.read()
provides.setdefault(service, set()).update(
serviceContent.splitlines())
# Service types defined in the module are assumed to be used by the module
serviceClassfile = service.replace(
'.', '/') + '.class'
if exists(join(dest_dir, serviceClassfile)):
uses.add(service)
jmd = JavaModuleDescriptor(moduleName,
exports,
requires,
uses,
provides,
packages=packages,
concealedRequires=concealedRequires,
jarpath=moduleJar,
dist=dist,
modulepath=modulepath)
# Compile module-info.class
module_info_java = join(dest_dir, 'module-info.java')
with open(module_info_java, 'w') as fp:
print(jmd.as_module_info(), file=fp)
javacCmd = [jdk.javac, '-d', dest_dir]
jdkModuleNames = [m.name for m in jdkModules]
modulepathJars = [
m.jarpath for m in jmd.modulepath
if m.jarpath and m.name not in jdkModuleNames
]
upgrademodulepathJars = [
m.jarpath for m in jmd.modulepath
if m.jarpath and m.name in jdkModuleNames
]
# TODO we should rather use the right JDK
javacCmd += [
'-target', version if version != 'common' else '9', '-source',
version if version != 'common' else '9'
]
if modulepathJars:
javacCmd.append('--module-path')
javacCmd.append(os.pathsep.join(modulepathJars))
if upgrademodulepathJars:
javacCmd.append('--upgrade-module-path')
javacCmd.append(os.pathsep.join(upgrademodulepathJars))
if concealedRequires:
for module, packages_ in concealedRequires.items():
for package in packages_:
javacCmd.append('--add-exports=' + module + '/' +
package + '=' + moduleName)
# https://blogs.oracle.com/darcy/new-javac-warning-for-setting-an-older-source-without-bootclasspath
# Disable the "bootstrap class path not set in conjunction with -source N" warning
# as we're relying on the Java compliance of project to correctly specify a JDK range
# providing the API required by the project. Also disable the warning about unknown
# modules in qualified exports (not sure how to avoid these since we build modules
# separately).
javacCmd.append('-Xlint:-options,-module')
javacCmd.append(module_info_java)
mx.run(javacCmd)
module_info_class = join(dest_dir, 'module-info.class')
# Append the module-info.class
module_info_arc_dir = ''
if version != 'common':
module_info_arc_dir = _versioned_prefix + version + '/'
if version == default_version:
default_jmd = jmd
with ZipFile(moduleJar, 'a') as zf:
zf.write(module_info_class,
module_info_arc_dir + basename(module_info_class))
zf.write(module_info_java,
module_info_arc_dir + basename(module_info_java))
if files_to_remove:
with mx.SafeFileCreation(moduleJar) as sfc:
with ZipFile(moduleJar,
'r') as inzf, ZipFile(sfc.tmpPath, 'w',
inzf.compression) as outzf:
for info in inzf.infolist():
if info.filename not in files_to_remove:
outzf.writestr(info, inzf.read(info))
finally:
shutil.rmtree(work_directory)
default_jmd.save()
return default_jmd
