def get_add_exports(self):
distributions = self.jar_distributions
distributions_transitive = mx.classpath_entries(distributions)
required_exports = mx_javamodules.requiredExports(
distributions_transitive, base_jdk())
return ' '.join(
AbstractNativeImageConfig.get_add_exports_list(required_exports))
def suite_collector(suite, predicate, collector, javaProperties,
seenSuites):
if suite.name in seenSuites:
return
seenSuites.add(suite.name)
suite.visit_imports(import_visitor,
predicate=predicate,
collector=collector,
javaProperties=javaProperties,
seenSuites=seenSuites)
for dist in suite.dists:
if predicate(dist):
for distCpEntry in mx.classpath_entries(dist):
if hasattr(distCpEntry, "getJavaProperties"):
for key, value in distCpEntry.getJavaProperties(
).items():
javaProperties[key] = value
if distCpEntry.isJdkLibrary() or distCpEntry.isJreLibrary(
):
cpPath = distCpEntry.classpath_repr(mx.get_jdk(),
resolve=True)
else:
cpPath = distCpEntry.classpath_repr(resolve=True)
if cpPath:
collector[cpPath] = None
def get_add_exports(self, missing_jars):
if self.use_modules is None:
return ''
distributions = self.jar_distributions
distributions_transitive = mx.classpath_entries(distributions)
distributions_transitive_clean = [entry for entry in distributions_transitive if str(entry) not in missing_jars]
required_exports = mx_javamodules.requiredExports(distributions_transitive_clean, base_jdk())
return ' '.join(AbstractNativeImageConfig.get_add_exports_list(required_exports))
def make_dist_rule(dist, mf):
def path_dist_relative(p):
return os.path.relpath(p, dist.suite.dir)
jdkDeployedDists = get_jdk_deployed_dists()
jarName = os.path.basename(dist.path)
sourcesVariableName = dist.name + "_SRC"
depJarVariableName = dist.name + "_DEP_JARS"
sources = []
resources = []
projects = [p for p in dist.archived_deps() if p.isJavaProject()]
targetPathPrefix = "$(TARGET)/"
annotationProcessorDeps = set()
classPath = [targetPathPrefix + os.path.basename(e.path) for e in mx.classpath_entries(dist, includeSelf=False, preferProjects=False)]
for p in projects:
projectDir = path_dist_relative(p.dir)
annotationProcessorDeps.update(p.declaredAnnotationProcessors)
depCheck = _get_dependency_check(p)
if depCheck:
sources.append(depCheck)
for src in [projectDir + '/' + d for d in p.srcDirs]:
sources.append(sourcesVariableName + " += $(shell find {} -type f 2> /dev/null)".format(src))
metaInf = src + "/META-INF"
if os.path.exists(os.path.join(dist.suite.dir, metaInf)):
resources.append(metaInf)
if depCheck:
sources.append("endif")
mf.add_definition("\n".join(sources))
apDistNames = []
apDistVariableNames = []
apDependencies = []
for apd in sorted(annotationProcessorDeps):
apDistNames.append(apd.name)
apDistVariableNames.append("$(" + apd.name + "_JAR)")
apDependencies.append("$(subst $(space),:,$(" + apd.name + "_DEP_JARS))")
shouldExport = dist in jdkDeployedDists
props = {
"name": dist.name,
"jarName": targetPathPrefix + jarName,
"depJarsVariableAccess": "$(" + depJarVariableName + ")" if len(classPath) > 0 else "",
"depJarsVariable": depJarVariableName,
"sourcesVariableName": sourcesVariableName,
"annotationProcessors": " ".join(apDistVariableNames),
"cpAnnotationProcessors": ":".join(apDistVariableNames + apDependencies),
"jarDeps": " ".join(classPath),
"copyResources": " ".join(resources)
}
mf.add_definition("{name}_JAR = {jarName}".format(**props))
if len(classPath) > 0: mf.add_definition("{depJarsVariable} = {jarDeps}".format(**props))
if shouldExport: mf.add_definition("EXPORTED_FILES += $({name}_JAR)".format(**props))
mf.add_rule("""$({name}_JAR): $({sourcesVariableName}) {annotationProcessors} {depJarsVariableAccess}
\t$(call build_and_jar,{cpAnnotationProcessors},$(subst $(space),:,{depJarsVariableAccess}),{copyResources},$({name}_JAR))
""".format(**props))
return
def suite_collector(suite, predicate, collector, javaProperties, seenSuites):
if suite.name in seenSuites:
return
seenSuites.add(suite.name)
suite.visit_imports(import_visitor, predicate=predicate, collector=collector, javaProperties=javaProperties, seenSuites=seenSuites)
for dist in suite.dists:
if predicate(dist):
for distCpEntry in mx.classpath_entries(dist):
if hasattr(distCpEntry, "getJavaProperties"):
for key, value in dist.getJavaProperties().items():
javaProperties[key] = value
if distCpEntry.isJdkLibrary() or distCpEntry.isJreLibrary():
cpPath = distCpEntry.classpath_repr(mx.get_jdk(), resolve=True)
else:
cpPath = distCpEntry.classpath_repr(resolve=True)
if cpPath:
collector[cpPath] = None
def make_java_module(dist, jdk):
"""
Creates a Java module from a distribution.
:param JARDistribution dist: the distribution from which to create a module
:param JDKConfig jdk: a JDK with a version >= 9 that can be used to compile the module-info class
:return: the `JavaModuleDescriptor` for the created Java module
"""
info = get_java_module_info(dist)
if info is None:
return None
moduleName, moduleDir, moduleJar = info # pylint: disable=unpacking-non-sequence
mx.log('Building Java module ' + moduleName + ' from ' + dist.name)
exports = {}
requires = {}
concealedRequires = {}
addExports = set()
uses = set()
modulepath = list()
usedModules = set()
if dist.suite.getMxCompatibility().moduleDepsEqualDistDeps():
moduledeps = dist.archived_deps()
for dep in mx.classpath_entries(dist, includeSelf=False):
jmd = make_java_module(dep, jdk) if dep.isJARDistribution() else None
if jmd:
modulepath.append(jmd)
requires[jmd.name] = set(['public'])
elif (dep.isJdkLibrary() or dep.isJreLibrary()) and dep.is_provided_by(jdk):
pass
else:
mx.abort(dist.name + ' cannot depend on ' + dep.name + ' as it does not define a module')
else:
moduledeps = get_module_deps(dist)
# Prepend JDK modules to module path
allmodules = list(jdk.get_modules()) + modulepath
javaprojects = [d for d in moduledeps if d.isJavaProject()]
packages = []
for dep in javaprojects:
uses.update(getattr(dep, 'uses', []))
for pkg in itertools.chain(dep.imported_java_packages(projectDepsOnly=False), getattr(dep, 'imports', [])):
depModule, visibility = lookup_package(allmodules, pkg, moduleName)
if depModule:
requires.setdefault(depModule.name, set())
if visibility == 'exported':
# A distribution based module does not re-export its imported JDK packages
usedModules.add(depModule)
else:
assert visibility == 'concealed'
concealedRequires.setdefault(depModule.name, set()).add(pkg)
usedModules.add(depModule)
addExports.add('--add-exports=' + depModule.name + '/' + pkg + '=' + moduleName)
# If an "exports" attribute is not present, all packages are exported
for package in _expand_package_info(dep, getattr(dep, 'exports', dep.defined_java_packages())):
exports.setdefault(package, [])
packages.extend(dep.defined_java_packages())
provides = {}
if exists(moduleDir):
shutil.rmtree(moduleDir)
for d in [dist] + [md for md in moduledeps if md.isJARDistribution()]:
if d.isJARDistribution():
with zipfile.ZipFile(d.path, 'r') as zf:
# To compile module-info.java, all classes it references must either be given
# as Java source files or already exist as class files in the output directory.
# As such, the jar file for each constituent distribution must be unpacked
# in the output directory.
zf.extractall(path=moduleDir)
names = frozenset(zf.namelist())
for arcname in names:
if arcname.startswith('META-INF/services/') and not arcname == 'META-INF/services/':
service = arcname[len('META-INF/services/'):]
assert '/' not in service
provides.setdefault(service, set()).update(zf.read(arcname).splitlines())
# Service types defined in the module are assumed to be used by the module
serviceClass = service.replace('.', '/') + '.class'
if serviceClass in names:
uses.add(service)
jmd = JavaModuleDescriptor(moduleName, exports, requires, uses, provides, packages=packages, concealedRequires=concealedRequires,
jarpath=moduleJar, dist=dist, modulepath=modulepath)
# Compile module-info.class
moduleInfo = join(moduleDir, 'module-info.java')
with open(moduleInfo, 'w') as fp:
print >> fp, jmd.as_module_info()
javacCmd = [jdk.javac, '-d', moduleDir]
modulepathJars = [m.jarpath for m in jmd.modulepath if m.jarpath]
if modulepathJars:
javacCmd.append('--module-path')
javacCmd.append(os.pathsep.join(modulepathJars))
javacCmd.extend(addExports)
javacCmd.append(moduleInfo)
mx.run(javacCmd)
# Create the module jar
shutil.make_archive(moduleJar, 'zip', moduleDir)
os.rename(moduleJar + '.zip', moduleJar)
jmd.save()
return jmd
def build(self):
if 'FASTR_RELEASE' not in os.environ:
mx.log('FastR: set FASTR_RELEASE to update release project')
return
# copy the release directories
output_dir = self.subject.dir
fastr_dir = _fastr_suite.dir
for d in ['bin', 'include', 'library', 'etc', 'share', 'doc']:
target_dir = join(output_dir, d)
if os.path.exists(target_dir):
shutil.rmtree(target_dir)
shutil.copytree(join(fastr_dir, d), target_dir)
lib_fastr_dir = join(fastr_dir, 'lib')
lib_output_dir = join(output_dir, 'lib')
if os.path.exists(lib_output_dir):
shutil.rmtree(lib_output_dir)
os.mkdir(lib_output_dir)
for f in os.listdir(lib_fastr_dir):
source_file = join(lib_fastr_dir, f)
target_file = join(lib_output_dir, f)
if f != '.DS_Store':
if os.path.islink(source_file):
os.symlink(os.readlink(source_file), target_file)
else:
shutil.copy(source_file, target_file)
# copyrights
copyrights_dir = join(fastr_dir, 'mx.fastr', 'copyrights')
with open(join(output_dir, 'COPYRIGHT'), 'w') as outfile:
for copyright_file in os.listdir(copyrights_dir):
if basename(copyright_file).endswith('copyright.star'):
with open(join(copyrights_dir, copyright_file)) as infile:
data = infile.read()
outfile.write(data)
# license/README
shutil.copy(join(fastr_dir, 'LICENSE'), output_dir)
shutil.copy(join(fastr_dir, 'README.md'), output_dir)
# canonicalize R_HOME_DIR in bin/R
bin_dir = join(output_dir, 'bin')
rcmd = join(bin_dir, 'R')
# R is the generic shell script (taken essentially verbatim from GNU R)
with open(rcmd) as f:
lines = f.readlines()
with open(rcmd, 'w') as f:
for line in lines:
if line.startswith('R_HOME_DIR='):
f.write("""
source="${BASH_SOURCE[0]}"
while [ -h "$source" ] ; do
prev_source="$source"
source="$(readlink "$source")";
if [[ "$source" != /* ]]; then
# if the link was relative, it was relative to where it came from
dir="$( cd -P "$( dirname "$prev_source" )" && pwd )"
source="$dir/$source"
fi
done
r_bin="$( cd -P "$( dirname "$source" )" && pwd )"
R_HOME_DIR="$( dirname "$r_bin" )"
""")
elif line.strip() == "#!/bin/sh":
f.write("#!/usr/bin/env bash\n")
else:
f.write(line)
# jar files for the launchers
jars_dir = join(bin_dir, 'fastr_jars')
if not os.path.exists(jars_dir):
os.mkdir(jars_dir)
# Copy all the jar files needed by the launchers
for e in mx.classpath_entries('FASTR'):
source_path = e.classpath_repr()
if mx.is_cache_path(source_path):
target_file_name = e.name.lower().replace('_', '-') + '.jar'
else:
target_file_name = basename(source_path)
assert isfile(source_path)
shutil.copy(source_path, join(jars_dir, target_file_name))
# create the classpath string
classpath = []
for _, _, jars in os.walk(jars_dir):
for jar in jars:
classpath.append(join("$R_HOME/bin/fastr_jars", jar))
classpath_string = ":".join(classpath)
# replace the mx exec scripts with native Java launchers, setting the classpath from above
bin_exec_dir = join(bin_dir, 'exec')
r_launcher = join(self.subject.dir, 'src', 'R_legacy')
template_dict = {'CLASSPATH': classpath_string}
self._template(r_launcher, join(bin_exec_dir, 'R'), template_dict)
shutil.rmtree(join(bin_dir, 'execRextras'))
rscript_launcher = join(self.subject.dir, 'src', 'Rscript_legacy')
self._template(rscript_launcher, join(bin_dir, 'Rscript'),
template_dict)
def _unittest_config_participant(config):
vmArgs, mainClass, mainClassArgs = config
cpIndex, cp = mx.find_classpath_arg(vmArgs)
if cp:
cp = _uniqify(cp.split(os.pathsep))
if isJDK8:
# Remove entries from class path that are in Graal or on the boot class path
redundantClasspathEntries = set()
for dist in [entry.dist() for entry in _jvmci_classpath]:
redundantClasspathEntries.update((d.output_dir() for d in dist.archived_deps() if d.isJavaProject()))
redundantClasspathEntries.add(dist.path)
cp = os.pathsep.join([e for e in cp if e not in redundantClasspathEntries])
vmArgs[cpIndex] = cp
else:
deployedModules = []
redundantClasspathEntries = set()
for dist in [entry.dist() for entry in _jvmci_classpath] + _bootclasspath_appends:
deployedModule = as_java_module(dist, jdk)
deployedModules.append(deployedModule)
redundantClasspathEntries.update(mx.classpath(dist, preferProjects=False, jdk=jdk).split(os.pathsep))
redundantClasspathEntries.update(mx.classpath(dist, preferProjects=True, jdk=jdk).split(os.pathsep))
# Remove entries from the class path that are in the deployed modules
cp = [classpathEntry for classpathEntry in cp if classpathEntry not in redundantClasspathEntries]
vmArgs[cpIndex] = os.pathsep.join(cp)
# Junit libraries are made into automatic modules so that they are visible to tests
# patched into modules. These automatic modules must be declared to be read by
# Graal which means they must also be made root modules (i.e., ``-addmods``)
# since ``-XaddReads`` can only be applied to root modules.
junitCp = [e.classpath_repr() for e in mx.classpath_entries(['JUNIT'])]
junitModules = [_automatic_module_name(e) for e in junitCp]
vmArgs.extend(['-modulepath', os.pathsep.join(junitCp)])
vmArgs.extend(['-addmods', ','.join(junitModules + [m.name for m in deployedModules])])
for deployedModule in deployedModules:
vmArgs.append('-XaddReads:' + deployedModule.name + '=' + ','.join(junitModules))
# Explicitly export concealed JVMCI packages required by Graal. Even though
# normally exported via jdk.vm.ci.services.Services.exportJVMCITo(), the
# Junit harness wants to access JVMCI classes (e.g., when loading classes
# to find test methods) without going through that entry point.
addedExports = {}
for deployedModule in deployedModules:
for concealingModule, packages in deployedModule.concealedRequires.iteritems():
if concealingModule == 'jdk.vm.ci':
for package in packages:
addedExports.setdefault(concealingModule + '/' + package, set()).add(deployedModule.name)
patches = {}
pathToProject = {p.output_dir() : p for p in mx.projects() if p.isJavaProject()}
for classpathEntry in cp:
# Export concealed packages used by the class path entry
_add_exports_for_concealed_packages(classpathEntry, pathToProject, addedExports, 'ALL-UNNAMED', deployedModules)
for deployedModule in deployedModules:
assert deployedModule.dist.path != classpathEntry, deployedModule.dist.path + ' should no longer be on the class path'
# Patch the class path entry into a module if it defines packages already defined by the module.
# Packages definitions cannot be split between modules.
classpathEntryPackages = frozenset(_defined_packages(classpathEntry))
if not classpathEntryPackages.isdisjoint(deployedModule.packages):
patches.setdefault(deployedModule.name, []).append(classpathEntry)
extraPackages = classpathEntryPackages - frozenset(deployedModule.exports.iterkeys())
if extraPackages:
# From http://openjdk.java.net/jeps/261:
# If a package found in a module definition on a patch path is not already exported
# by that module then it will, still, not be exported. It can be exported explicitly
# via either the reflection API or the -XaddExports option.
for package in extraPackages:
addedExports.setdefault(deployedModule.name + '/' + package, set()).update(junitModules + ['ALL-UNNAMED'])
for moduleName, cpEntries in patches.iteritems():
vmArgs.append('-Xpatch:' + moduleName + '=' + os.pathsep.join(cpEntries))
vmArgs.extend(['-XaddExports:' + export + '=' + ','.join(sorted(targets)) for export, targets in addedExports.iteritems()])
if isJDK8:
# Run the VM in a mode where application/test classes can
# access JVMCI loaded classes.
vmArgs.append('-XX:-UseJVMCIClassLoader')
return (vmArgs, mainClass, mainClassArgs)
def _unittest_config_participant(config):
vmArgs, mainClass, mainClassArgs = config
cpIndex, cp = mx.find_classpath_arg(vmArgs)
if cp:
cp = _uniqify(cp.split(os.pathsep))
if isJDK8:
# Remove entries from class path that are in Graal
excluded = set()
for entry in _jvmci_classpath:
dist = entry.dist()
excluded.update((d.output_dir() for d in dist.archived_deps() if d.isJavaProject()))
excluded.add(dist.path)
cp = os.pathsep.join([e for e in cp if e not in excluded])
vmArgs[cpIndex] = cp
else:
# Remove entries from class path that are in the Graal
assert _graal_module_descriptor is not None
module = as_java_module(_graal_module_descriptor.dist(), jdk)
junitCp = [e.classpath_repr() for e in mx.classpath_entries(['JUNIT'])]
excluded = frozenset([classpathEntry.classpath_repr() for classpathEntry in get_module_deps(module.dist)] + junitCp)
cp = [classpathEntry for classpathEntry in cp if classpathEntry not in excluded]
vmArgs[cpIndex] = os.pathsep.join(cp)
# Junit libraries are made into automatic modules so that they are visible to tests
# patched into Graal. These automatic modules must be declared to be read by
# Graal which means they must also be made root modules (i.e., ``-addmods``)
# since ``-XaddReads`` can only be applied to root modules.
junitModules = [_automatic_module_name(e) for e in junitCp]
vmArgs.extend(['-modulepath', os.pathsep.join(junitCp)])
vmArgs.extend(['-addmods', ','.join(junitModules)])
vmArgs.extend(['-XaddReads:' + module.name + '=' + ','.join(junitModules)])
# Explicitly export JVMCI to Graal
addedExports = {}
for concealingModule, packages in module.concealedRequires.iteritems():
if concealingModule == 'jdk.vm.ci':
for package in packages:
addedExports.setdefault(concealingModule + '/' + package, set()).add(module.name)
patches = []
graalConcealedPackages = list(module.conceals)
pathToProject = {p.output_dir() : p for p in mx.projects() if p.isJavaProject()}
for classpathEntry in cp:
# Export concealed JDK packages used by the class path entry
_add_exports_for_concealed_packages(classpathEntry, pathToProject, addedExports, 'ALL-UNNAMED')
# Patch the class path entry into Graal if it defines packages already defined by Graal.
# Packages definitions cannot be split between modules.
packages = frozenset(_defined_packages(classpathEntry))
if not packages.isdisjoint(module.packages):
patches.append(classpathEntry)
extraPackages = packages - module.packages
if extraPackages:
# From http://openjdk.java.net/jeps/261:
# If a package found in a module definition on a patch path is not already exported
# by that module then it will, still, not be exported. It can be exported explicitly
# via either the reflection API or the -XaddExports option.
graalConcealedPackages.extend(extraPackages)
if patches:
vmArgs.append('-Xpatch:' + module.name + '=' + os.pathsep.join(patches))
# Export all Graal packages to make them available to test classes
for package in graalConcealedPackages:
addedExports.setdefault(module.name + '/' + package, set()).update(junitModules + ['ALL-UNNAMED'])
vmArgs.extend(['-XaddExports:' + export + '=' + ','.join(sorted(targets)) for export, targets in addedExports.iteritems()])
if isJDK8:
# Run the VM in a mode where application/test classes can
# access JVMCI loaded classes.
vmArgs.append('-XX:-UseJVMCIClassLoader')
return (vmArgs, mainClass, mainClassArgs)
def _unittest_config_participant(config):
vmArgs, mainClass, mainClassArgs = config
cpIndex, cp = mx.find_classpath_arg(vmArgs)
if cp:
cp = _uniqify(cp.split(os.pathsep))
if isJDK8:
# Remove entries from class path that are in Graal or on the boot class path
redundantClasspathEntries = set()
for dist in [entry.dist() for entry in _jvmci_classpath]:
redundantClasspathEntries.update((d.output_dir() for d in dist.archived_deps() if d.isJavaProject()))
redundantClasspathEntries.add(dist.path)
cp = os.pathsep.join([e for e in cp if e not in redundantClasspathEntries])
vmArgs[cpIndex] = cp
else:
deployedModules = []
redundantClasspathEntries = set()
for dist in [entry.dist() for entry in _jvmci_classpath] + _bootclasspath_appends:
deployedModule = as_java_module(dist, jdk)
deployedModules.append(deployedModule)
redundantClasspathEntries.update(mx.classpath(dist, preferProjects=False, jdk=jdk).split(os.pathsep))
redundantClasspathEntries.update(mx.classpath(dist, preferProjects=True, jdk=jdk).split(os.pathsep))
if hasattr(dist, 'overlaps'):
for o in dist.overlaps:
path = mx.distribution(o).classpath_repr()
if path:
redundantClasspathEntries.add(path)
# Remove entries from the class path that are in the deployed modules
cp = [classpathEntry for classpathEntry in cp if classpathEntry not in redundantClasspathEntries]
vmArgs[cpIndex] = os.pathsep.join(cp)
# Junit libraries are made into automatic modules so that they are visible to tests
# patched into modules. These automatic modules must be declared to be read by
# Graal which means they must also be made root modules (i.e., ``--add-modules``)
# since ``--add-reads`` can only be applied to root modules.
junitCp = [e.classpath_repr() for e in mx.classpath_entries(['JUNIT'])]
junitModules = [_automatic_module_name(e) for e in junitCp]
vmArgs.append('--module-path=' + os.pathsep.join(junitCp))
vmArgs.append('--add-modules=' + ','.join(junitModules + [m.name for m in deployedModules]))
for deployedModule in deployedModules:
vmArgs.append('--add-reads=' + deployedModule.name + '=' + ','.join(junitModules))
# Explicitly export concealed JVMCI packages required by Graal. Even though
# normally exported via jdk.vm.ci.services.Services.exportJVMCITo(), the
# Junit harness wants to access JVMCI classes (e.g., when loading classes
# to find test methods) without going through that entry point.
addedExports = {}
for deployedModule in deployedModules:
for concealingModule, packages in deployedModule.concealedRequires.iteritems():
if concealingModule == 'jdk.vm.ci':
for package in packages:
addedExports.setdefault(concealingModule + '/' + package, set()).add(deployedModule.name)
pathToDep = {p.output_dir() if p.isJavaProject() else p.path: p for p in mx.dependencies() if p.isJavaProject() or p.isJARDistribution()}
for classpathEntry in cp:
# Export concealed packages used by the class path entry
_add_exports_for_concealed_packages(classpathEntry, pathToDep, addedExports, 'ALL-UNNAMED', deployedModules)
for deployedModule in deployedModules:
assert deployedModule.dist.path != classpathEntry, deployedModule.dist.path + ' should no longer be on the class path'
# Ensure the class path entry does not define packages already defined by the module.
# Packages definitions cannot be split between modules.
classpathEntryPackages = frozenset(_defined_packages(classpathEntry))
intersection = classpathEntryPackages.intersection(deployedModule.packages)
if intersection:
mx.abort(classpathEntry + ' cannot extend package(s) defined in the module ' + deployedModule.name + ': ' + ', '.join(intersection))
vmArgs.extend(['--add-exports=' + export + '=' + ','.join(sorted(targets)) for export, targets in addedExports.iteritems()])
if isJDK8:
# Run the VM in a mode where application/test classes can
# access JVMCI loaded classes.
vmArgs.append('-XX:-UseJVMCIClassLoader')
return (vmArgs, mainClass, mainClassArgs)
def build(self):
# copy the release directories
output_dir = self.subject.dir
fastr_dir = _fastr_suite.dir
for d in ['bin', 'include', 'library', 'etc', 'share', 'doc']:
target_dir = join(output_dir, d)
source_dir = join(fastr_dir, d)
if os.path.exists(target_dir):
shutil.rmtree(target_dir)
shutil.copytree(source_dir, target_dir)
lib_fastr_dir = join(fastr_dir, 'lib')
lib_output_dir = join(output_dir, 'lib')
if os.path.exists(lib_output_dir):
shutil.rmtree(lib_output_dir)
os.mkdir(lib_output_dir)
for f in os.listdir(lib_fastr_dir):
source_file = join(lib_fastr_dir, f)
target_file = join(lib_output_dir, f)
if f != '.DS_Store':
if os.path.islink(source_file):
os.symlink(os.readlink(source_file), target_file)
else:
shutil.copy(source_file, target_file)
# copyrights
copyrights_dir = join(fastr_dir, 'mx.fastr', 'copyrights')
with open(join(output_dir, 'COPYRIGHT'), 'w') as outfile:
for copyright_file in os.listdir(copyrights_dir):
if basename(copyright_file).endswith('copyright.star'):
with open(join(copyrights_dir, copyright_file)) as infile:
data = infile.read()
outfile.write(data)
# license/README
shutil.copy(join(fastr_dir, 'LICENSE'), output_dir)
shutil.copy(join(fastr_dir, 'README.md'), output_dir)
# jar files for the launchers
bin_dir = join(output_dir, 'bin')
jars_dir = join(bin_dir, 'fastr_jars')
if not os.path.exists(jars_dir):
os.mkdir(jars_dir)
# Copy all the jar files needed by the launchers
for e in mx.classpath_entries('FASTR'):
source_path = e.classpath_repr()
if mx.is_cache_path(source_path):
target_file_name = e.name.lower().replace('_', '-') + '.jar'
else:
target_file_name = basename(source_path)
assert isfile(source_path)
shutil.copy(source_path, join(jars_dir, target_file_name))
# create the classpath string
classpath = []
for _, _, jars in os.walk(jars_dir):
for jar in jars:
classpath.append(join("$R_HOME/bin/fastr_jars", jar))
classpath_string = ":".join(classpath)
# replace the mx exec scripts with native Java launchers, setting the classpath from above
bin_exec_dir = join(bin_dir, 'exec')
r_launcher = join(self.subject.dir, 'src', 'R_launcher')
template_dict = {'CLASSPATH': classpath_string}
self._template(r_launcher, join(bin_exec_dir, 'R'), template_dict)
shutil.rmtree(join(bin_dir, 'execRextras'))
rscript_launcher = join(self.subject.dir, 'src', 'Rscript_launcher')
self._template(rscript_launcher, join(bin_dir, 'Rscript'),
template_dict)
def make_java_module(dist, jdk):
"""
Creates a Java module from a distribution.
:param JARDistribution dist: the distribution from which to create a module
:param JDKConfig jdk: a JDK with a version >= 9 that can be used to compile the module-info class
:return: the `JavaModuleDescriptor` for the created Java module
"""
info = get_java_module_info(dist)
if info is None:
return None
moduleName, moduleDir, moduleJar = info # pylint: disable=unpacking-non-sequence
mx.log('Building Java module ' + moduleName + ' from ' + dist.name)
exports = {}
requires = {}
concealedRequires = {}
uses = set()
modulepath = list()
usedModules = set()
if dist.suite.getMxCompatibility().moduleDepsEqualDistDeps():
moduledeps = dist.archived_deps()
for dep in mx.classpath_entries(dist, includeSelf=False):
if dep.isJARDistribution():
jmd = as_java_module(
dep, jdk, fatalIfNotCreated=False) or make_java_module(
dep, jdk)
modulepath.append(jmd)
requires[jmd.name] = {jdk.get_transitive_requires_keyword()}
elif (dep.isJdkLibrary()
or dep.isJreLibrary()) and dep.is_provided_by(jdk):
pass
else:
mx.abort(dist.name + ' cannot depend on ' + dep.name +
' as it does not define a module')
else:
moduledeps = get_module_deps(dist)
# Append JDK modules to module path
jdkModules = jdk.get_modules()
if not isinstance(jdkModules, list):
jdkModules = list(jdkModules)
allmodules = modulepath + jdkModules
javaprojects = [d for d in moduledeps if d.isJavaProject()]
# Collect packages in the module first
packages = set()
for dep in javaprojects:
packages.update(dep.defined_java_packages())
for dep in javaprojects:
uses.update(getattr(dep, 'uses', []))
for pkg in getattr(dep, 'runtimeDeps', []):
requires.setdefault(pkg, set(['static']))
for pkg in itertools.chain(
dep.imported_java_packages(projectDepsOnly=False),
getattr(dep, 'imports', [])):
# Only consider packages not defined by the module we're creating. This handles the
# case where we're creating a module that will upgrade an existing upgradeable
# module in the JDK such as jdk.internal.vm.compiler.
if pkg not in packages:
depModule, visibility = lookup_package(allmodules, pkg,
moduleName)
if depModule and depModule.name != moduleName:
requires.setdefault(depModule.name, set())
if visibility == 'exported':
# A distribution based module does not re-export its imported JDK packages
usedModules.add(depModule)
else:
assert visibility == 'concealed'
concealedRequires.setdefault(depModule.name,
set()).add(pkg)
usedModules.add(depModule)
# If an "exports" attribute is not present, all packages are exported
for package in _expand_package_info(
dep, getattr(dep, 'exports', dep.defined_java_packages())):
exports.setdefault(package, [])
provides = {}
if exists(moduleDir):
shutil.rmtree(moduleDir)
for d in [dist] + [md for md in moduledeps if md.isJARDistribution()]:
if d.isJARDistribution():
with zipfile.ZipFile(d.path, 'r') as zf:
# To compile module-info.java, all classes it references must either be given
# as Java source files or already exist as class files in the output directory.
# As such, the jar file for each constituent distribution must be unpacked
# in the output directory.
zf.extractall(path=moduleDir)
names = frozenset(zf.namelist())
# Flatten versioned resources
versionsDir = join(moduleDir, 'META-INF', 'versions')
if exists(versionsDir):
versionedRE = re.compile(
r'META-INF/versions/([1-9][0-9]*)/(.+)')
versions = {}
for arcname in sorted(names):
m = versionedRE.match(arcname)
if m:
version = int(m.group(1))
unversionedName = m.group(2)
versions.setdefault(
version,
{})[unversionedName] = zf.read(arcname)
for version, resources in sorted(versions.iteritems()):
for unversionedName, content in resources.iteritems():
dst = join(moduleDir, unversionedName)
if version <= jdk.javaCompliance.value:
parent = dirname(dst)
if parent and not exists(parent):
os.makedirs(parent)
with open(dst, 'wb') as fp:
fp.write(content)
else:
# Ignore resource whose version is too high
pass
shutil.rmtree(versionsDir)
manifest = join(moduleDir, 'META-INF/MANIFEST.MF')
# Remove Multi-Release attribute from manifest as the jar
# is now flattened. This is also a workaround for
# https://bugs.openjdk.java.net/browse/JDK-8193802
if exists(manifest):
with open(manifest) as fp:
content = fp.readlines()
newContent = [
l for l in content if not 'Multi-Release:' in l
]
if newContent != content:
with open(manifest, 'w') as fp:
fp.write(''.join(newContent))
serviceRE = re.compile(r'META-INF/services/(.+)')
for arcname in names:
m = serviceRE.match(arcname)
if m:
service = m.group(1)
# While a META-INF provider configuration file must use a fully qualified binary
# name[1] of the service, a provides directive in a module descriptor must use
# the fully qualified non-binary name[2] of the service.
#
# [1] https://docs.oracle.com/javase/9/docs/api/java/util/ServiceLoader.html
# [2] https://docs.oracle.com/javase/9/docs/api/java/lang/module/ModuleDescriptor.Provides.html#service--
service = service.replace('$', '.')
assert '/' not in service
provides.setdefault(service, set()).update(
zf.read(arcname).splitlines())
# Service types defined in the module are assumed to be used by the module
serviceClass = service.replace('.', '/') + '.class'
if serviceClass in names:
uses.add(service)
servicesDir = join(moduleDir, 'META-INF', 'services')
if exists(servicesDir):
shutil.rmtree(servicesDir)
jmd = JavaModuleDescriptor(moduleName,
exports,
requires,
uses,
provides,
packages=packages,
concealedRequires=concealedRequires,
jarpath=moduleJar,
dist=dist,
modulepath=modulepath)
# Compile module-info.class
moduleInfo = join(moduleDir, 'module-info.java')
with open(moduleInfo, 'w') as fp:
print >> fp, jmd.as_module_info()
javacCmd = [jdk.javac, '-d', moduleDir]
jdkModuleNames = [m.name for m in jdkModules]
modulepathJars = [
m.jarpath for m in jmd.modulepath
if m.jarpath and m.name not in jdkModuleNames
]
upgrademodulepathJars = [
m.jarpath for m in jmd.modulepath
if m.jarpath and m.name in jdkModuleNames
]
if modulepathJars:
javacCmd.append('--module-path')
javacCmd.append(os.pathsep.join(modulepathJars))
if upgrademodulepathJars:
javacCmd.append('--upgrade-module-path')
javacCmd.append(os.pathsep.join(upgrademodulepathJars))
if concealedRequires:
for module, packages in concealedRequires.iteritems():
for package in packages:
javacCmd.append('--add-exports=' + module + '/' + package +
'=' + moduleName)
javacCmd.append(moduleInfo)
mx.run(javacCmd)
# Create the module jar
shutil.make_archive(moduleJar, 'zip', moduleDir)
os.rename(moduleJar + '.zip', moduleJar)
jmd.save()
return jmd
def _unittest_config_participant(config):
vmArgs, mainClass, mainClassArgs = config
cpIndex, cp = mx.find_classpath_arg(vmArgs)
if cp:
cp = _uniqify(cp.split(os.pathsep))
if isJDK8:
# Remove entries from class path that are in Graal or on the boot class path
redundantClasspathEntries = set()
for dist in [entry.dist() for entry in _jvmci_classpath]:
redundantClasspathEntries.update(
(d.output_dir() for d in dist.archived_deps()
if d.isJavaProject()))
redundantClasspathEntries.add(dist.path)
cp = os.pathsep.join(
[e for e in cp if e not in redundantClasspathEntries])
vmArgs[cpIndex] = cp
else:
deployedModules = []
redundantClasspathEntries = set()
for dist in [entry.dist() for entry in _jvmci_classpath
] + _bootclasspath_appends:
deployedModule = as_java_module(dist, jdk)
deployedModules.append(deployedModule)
redundantClasspathEntries.update(
mx.classpath(dist, preferProjects=False,
jdk=jdk).split(os.pathsep))
redundantClasspathEntries.update(
mx.classpath(dist, preferProjects=True,
jdk=jdk).split(os.pathsep))
if hasattr(dist, 'overlaps'):
for o in dist.overlaps:
path = mx.distribution(o).classpath_repr()
if path:
redundantClasspathEntries.add(path)
# Remove entries from the class path that are in the deployed modules
cp = [
classpathEntry for classpathEntry in cp
if classpathEntry not in redundantClasspathEntries
]
vmArgs[cpIndex] = os.pathsep.join(cp)
# Junit libraries are made into automatic modules so that they are visible to tests
# patched into modules. These automatic modules must be declared to be read by
# Graal which means they must also be made root modules (i.e., ``--add-modules``)
# since ``--add-reads`` can only be applied to root modules.
junitCp = [
e.classpath_repr() for e in mx.classpath_entries(['JUNIT'])
]
junitModules = [_automatic_module_name(e) for e in junitCp]
vmArgs.append('--module-path=' + os.pathsep.join(junitCp))
vmArgs.append('--add-modules=' +
','.join(junitModules +
[m.name for m in deployedModules]))
for deployedModule in deployedModules:
vmArgs.append('--add-reads=' + deployedModule.name + '=' +
','.join(junitModules))
# Explicitly export concealed JVMCI packages required by Graal. Even though
# normally exported via jdk.vm.ci.services.Services.exportJVMCITo(), the
# Junit harness wants to access JVMCI classes (e.g., when loading classes
# to find test methods) without going through that entry point.
addedExports = {}
for deployedModule in deployedModules:
for concealingModule, packages in deployedModule.concealedRequires.iteritems(
):
if concealingModule == 'jdk.vm.ci':
for package in packages:
addedExports.setdefault(
concealingModule + '/' + package,
set()).add(deployedModule.name)
pathToDep = {
p.output_dir() if p.isJavaProject() else p.path: p
for p in mx.dependencies()
if p.isJavaProject() or p.isJARDistribution()
}
for classpathEntry in cp:
# Export concealed packages used by the class path entry
_add_exports_for_concealed_packages(classpathEntry, pathToDep,
addedExports,
'ALL-UNNAMED',
deployedModules)
for deployedModule in deployedModules:
assert deployedModule.dist.path != classpathEntry, deployedModule.dist.path + ' should no longer be on the class path'
# Ensure the class path entry does not define packages already defined by the module.
# Packages definitions cannot be split between modules.
classpathEntryPackages = frozenset(
_defined_packages(classpathEntry))
intersection = classpathEntryPackages.intersection(
deployedModule.packages)
if intersection:
mx.abort(
classpathEntry +
' cannot extend package(s) defined in the module '
+ deployedModule.name + ': ' +
', '.join(intersection))
vmArgs.extend([
'--add-exports=' + export + '=' + ','.join(sorted(targets))
for export, targets in addedExports.iteritems()
])
if isJDK8:
# Run the VM in a mode where application/test classes can
# access JVMCI loaded classes.
vmArgs.append('-XX:-UseJVMCIClassLoader')
return (vmArgs, mainClass, mainClassArgs)
def make_java_module(dist, jdk):
"""
Creates a Java module from a distribution.
This updates the JAR by adding `module-info` classes.
The `META-INF` directory can not be versioned. However, we make an exception here for `META-INF/services`:
if different versions should have different service providers, a `META-INF/_versions/<version>/META-INF/services`
directory can be used (note the `_` before `versions`).
These service provider declarations will be used to build the versioned module-info files and the
`META-INF/_versions/<version>` directories will be removed from the archive.
This is done using a separate versioning directory so that the JAR can be a valid multi-release JAR before this
transformation.
input:
com/foo/MyProvider.class # JDK 8 or earlier specific provider
META-INF/services/com.foo.MyService # Contains: com.foo.MyProvider
META-INF/_versions/9/META-INF/services/com.foo.MyService # Contains: com.foo.MyProvider
META-INF/versions/9/com/foo/MyProvider.class # JDK 9 and 10 specific provider
META-INF/_versions/11/META-INF/services/com.foo.MyService # Contains: provides com.foo.MyService with com.foo.MyProvider
META-INF/versions/11/com/foo/MyProvider.class # JDK 11 and later specific provider
output:
com/foo/MyProvider.class # JDK 8 or earlier specific provider
META-INF/services/com.foo.MyService # Contains: com.foo.MyProvider
META-INF/versions/9/module-info.class # Contains: provides com.foo.MyService with com.foo.MyProvider
META-INF/versions/9/com/foo/MyProvider.class # JDK 9 and 10 specific provider
META-INF/versions/11/module-info.class # Contains: provides com.foo.MyService with com.foo.MyProvider
META-INF/versions/11/com/foo/MyProvider.class # JDK 11 and later specific provider
:param JARDistribution dist: the distribution from which to create a module
:param JDKConfig jdk: a JDK with a version >= 9 that can be used to compile the module-info class
:return: the `JavaModuleDescriptor` for the created Java module
"""
info = get_java_module_info(dist)
if info is None:
return None
moduleName, _, moduleJar = info # pylint: disable=unpacking-non-sequence
mx.log('Building Java module ' + moduleName + ' from ' + dist.name)
exports = {}
requires = {}
concealedRequires = {}
base_uses = set()
modulepath = list()
usedModules = set()
if dist.suite.getMxCompatibility().moduleDepsEqualDistDeps():
moduledeps = dist.archived_deps()
for dep in mx.classpath_entries(dist, includeSelf=False):
if dep.isJARDistribution():
jmd = as_java_module(dep, jdk)
modulepath.append(jmd)
requires[jmd.name] = {jdk.get_transitive_requires_keyword()}
elif (dep.isJdkLibrary()
or dep.isJreLibrary()) and dep.is_provided_by(jdk):
pass
elif dep.isLibrary():
jmd = get_library_as_module(dep, jdk)
modulepath.append(jmd)
requires[jmd.name] = set()
else:
mx.abort(dist.name + ' cannot depend on ' + dep.name +
' as it does not define a module')
else:
moduledeps = get_module_deps(dist)
# Append JDK modules to module path
jdkModules = jdk.get_modules()
if not isinstance(jdkModules, list):
jdkModules = list(jdkModules)
allmodules = modulepath + jdkModules
javaprojects = [d for d in moduledeps if d.isJavaProject()]
# Collect packages in the module first
packages = set()
for dep in javaprojects:
packages.update(dep.defined_java_packages())
for dep in javaprojects:
base_uses.update(getattr(dep, 'uses', []))
for pkg in getattr(dep, 'runtimeDeps', []):
requires.setdefault(pkg, {'static'})
for pkg in itertools.chain(
dep.imported_java_packages(projectDepsOnly=False),
getattr(dep, 'imports', [])):
# Only consider packages not defined by the module we're creating. This handles the
# case where we're creating a module that will upgrade an existing upgradeable
# module in the JDK such as jdk.internal.vm.compiler.
if pkg not in packages:
depModule, visibility = lookup_package(allmodules, pkg,
moduleName)
if depModule and depModule.name != moduleName:
requires.setdefault(depModule.name, set())
if visibility == 'exported':
# A distribution based module does not re-export its imported JDK packages
usedModules.add(depModule)
else:
assert visibility == 'concealed'
concealedRequires.setdefault(depModule.name,
set()).add(pkg)
usedModules.add(depModule)
# If an "exports" attribute is not present, all packages are exported
for package in _expand_package_info(
dep, getattr(dep, 'exports', dep.defined_java_packages())):
if ' to ' in package:
splitpackage = package.split(' to ')
package = splitpackage[0].strip()
if not package:
mx.abort(
'exports attribute cannot have empty package value',
context=dist)
targets = [n.strip() for n in splitpackage[1].split(',')]
if not targets:
mx.abort(
'exports attribute must have at least one target for qualified export',
context=dist)
exports.setdefault(package, targets)
else:
exports.setdefault(package, [])
work_directory = mkdtemp()
try:
files_to_remove = set()
# To compile module-info.java, all classes it references must either be given
# as Java source files or already exist as class files in the output directory.
# As such, the jar file for each constituent distribution must be unpacked
# in the output directory.
versions = {}
for d in [dist] + [md for md in moduledeps if md.isJARDistribution()]:
if d.isJARDistribution():
with zipfile.ZipFile(d.original_path(), 'r') as zf:
for arcname in sorted(zf.namelist()):
m = _versioned_re.match(arcname)
if m:
version = m.group(1)
unversioned_name = m.group(2)
if version <= jdk.javaCompliance:
versions.setdefault(
version, {})[unversioned_name] = arcname
else:
# Ignore resource whose version is too high
pass
if unversioned_name.startswith(
'META-INF/services/'):
files_to_remove.add(arcname)
elif unversioned_name.startswith('META-INF/'):
mx.abort(
"META-INF resources can not be versioned and will make modules fail to load ({})."
.format(arcname))
default_jmd = None
all_versions = set(versions.keys())
if '9' not in all_versions:
# 9 is the first version that supports modules and can be versioned in the JAR:
# if there is no `META-INF/versions/9` then we should add a `module-info.class` to the root of the JAR
# so that the module works on JDK 9.
all_versions = all_versions | {'common'}
default_version = 'common'
else:
default_version = str(max((int(v) for v in all_versions)))
for version in all_versions:
uses = base_uses.copy()
provides = {}
dest_dir = join(work_directory, version)
int_version = int(version) if version != 'common' else -1
for d in [dist
] + [md for md in moduledeps if md.isJARDistribution()]:
if d.isJARDistribution():
with zipfile.ZipFile(d.original_path(), 'r') as zf:
for name in zf.namelist():
m = _versioned_re.match(name)
if m:
file_version = int(m.group(1))
if file_version > int_version:
continue
unversioned_name = m.group(2)
if name.startswith(_special_versioned_prefix):
if not unversioned_name.startswith(
'META-INF/services'):
raise mx.abort(
"The special versioned directory ({}) is only supported for META-INF/services files. Got {}"
.format(_special_versioned_prefix,
name))
if unversioned_name:
dst = join(dest_dir, unversioned_name)
parent = dirname(dst)
if parent and not exists(parent):
os.makedirs(parent)
with open(dst, 'wb') as fp:
fp.write(zf.read(name))
else:
zf.extract(name, dest_dir)
servicesDir = join(dest_dir, 'META-INF', 'services')
if exists(servicesDir):
for servicePathName in os.listdir(servicesDir):
# While a META-INF provider configuration file must use a fully qualified binary
# name[1] of the service, a provides directive in a module descriptor must use
# the fully qualified non-binary name[2] of the service.
#
# [1] https://docs.oracle.com/javase/9/docs/api/java/util/ServiceLoader.html
# [2] https://docs.oracle.com/javase/9/docs/api/java/lang/module/ModuleDescriptor.Provides.html#service--
service = servicePathName.replace('$', '.')
assert '/' not in service
with open(join(servicesDir,
servicePathName)) as fp:
serviceContent = fp.read()
provides.setdefault(service, set()).update(
serviceContent.splitlines())
# Service types defined in the module are assumed to be used by the module
serviceClassfile = service.replace(
'.', '/') + '.class'
if exists(join(dest_dir, serviceClassfile)):
uses.add(service)
jmd = JavaModuleDescriptor(moduleName,
exports,
requires,
uses,
provides,
packages=packages,
concealedRequires=concealedRequires,
jarpath=moduleJar,
dist=dist,
modulepath=modulepath)
# Compile module-info.class
module_info_java = join(dest_dir, 'module-info.java')
with open(module_info_java, 'w') as fp:
print(jmd.as_module_info(), file=fp)
javacCmd = [jdk.javac, '-d', dest_dir]
jdkModuleNames = [m.name for m in jdkModules]
modulepathJars = [
m.jarpath for m in jmd.modulepath
if m.jarpath and m.name not in jdkModuleNames
]
upgrademodulepathJars = [
m.jarpath for m in jmd.modulepath
if m.jarpath and m.name in jdkModuleNames
]
# TODO we should rather use the right JDK
javacCmd += [
'-target', version if version != 'common' else '9', '-source',
version if version != 'common' else '9'
]
if modulepathJars:
javacCmd.append('--module-path')
javacCmd.append(os.pathsep.join(modulepathJars))
if upgrademodulepathJars:
javacCmd.append('--upgrade-module-path')
javacCmd.append(os.pathsep.join(upgrademodulepathJars))
if concealedRequires:
for module, packages_ in concealedRequires.items():
for package in packages_:
javacCmd.append('--add-exports=' + module + '/' +
package + '=' + moduleName)
# https://blogs.oracle.com/darcy/new-javac-warning-for-setting-an-older-source-without-bootclasspath
# Disable the "bootstrap class path not set in conjunction with -source N" warning
# as we're relying on the Java compliance of project to correctly specify a JDK range
# providing the API required by the project. Also disable the warning about unknown
# modules in qualified exports (not sure how to avoid these since we build modules
# separately).
javacCmd.append('-Xlint:-options,-module')
javacCmd.append(module_info_java)
mx.run(javacCmd)
module_info_class = join(dest_dir, 'module-info.class')
# Append the module-info.class
module_info_arc_dir = ''
if version != 'common':
module_info_arc_dir = _versioned_prefix + version + '/'
if version == default_version:
default_jmd = jmd
with ZipFile(moduleJar, 'a') as zf:
zf.write(module_info_class,
module_info_arc_dir + basename(module_info_class))
zf.write(module_info_java,
module_info_arc_dir + basename(module_info_java))
if files_to_remove:
with mx.SafeFileCreation(moduleJar) as sfc:
with ZipFile(moduleJar,
'r') as inzf, ZipFile(sfc.tmpPath, 'w',
inzf.compression) as outzf:
for info in inzf.infolist():
if info.filename not in files_to_remove:
outzf.writestr(info, inzf.read(info))
finally:
shutil.rmtree(work_directory)
default_jmd.save()
return default_jmd
