def symlink_or_copy(target_path, dest_path, debug_gr_8964=False):
# Follow symbolic links in case they go outside my suite directories.
real_target_path = os.path.realpath(target_path)
if debug_gr_8964:
mx.log(' [mx_substratevm.symlink_or_copy:')
mx.log(' suite.dir:' + suite.dir)
mx.log(' target_path: ' + target_path)
mx.log(' real_target_path: ' + real_target_path)
mx.log(' dest_path: ' + dest_path)
if any(
real_target_path.startswith(s.dir)
for s in mx.suites(includeBinary=False)):
# Symbolic link to files in my suites.
sym_target = os.path.relpath(real_target_path, dirname(dest_path))
if debug_gr_8964:
mx.log(' symlink target: ' + sym_target)
try:
os.symlink(sym_target, dest_path)
except AttributeError:
# no `symlink` on Windows
copy2(real_target_path, dest_path)
else:
# Else copy the file to so it can not change out from under me.
if debug_gr_8964:
mx.log(' copy2: ')
copy2(real_target_path, dest_path)
if debug_gr_8964:
mx.log(' ]')
def symlink_or_copy(target_path, dest_path, debug_gr_8964=False):
# Follow symbolic links in case they go outside my suite directories.
real_target_path = os.path.realpath(target_path)
if debug_gr_8964:
mx.log(' [mx_substratevm.symlink_or_copy:')
mx.log(' suite.dir:' + suite.dir)
mx.log(' target_path: ' + target_path)
mx.log(' real_target_path: ' + real_target_path)
mx.log(' dest_path: ' + dest_path)
if any(real_target_path.startswith(s.dir) for s in mx.suites(includeBinary=False)):
# Symbolic link to files in my suites.
sym_target = os.path.relpath(real_target_path, dirname(dest_path))
if debug_gr_8964:
mx.log(' symlink target: ' + sym_target)
try:
os.symlink(sym_target, dest_path)
except AttributeError:
# no `symlink` on Windows
copy2(real_target_path, dest_path)
else:
# Else copy the file to so it can not change out from under me.
if debug_gr_8964:
mx.log(' copy2: ')
copy2(real_target_path, dest_path)
if debug_gr_8964:
mx.log(' ]')
def runLLVM(args=None, out=None, err=None, timeout=None, nonZeroIsFatal=True, get_classpath_options=getClasspathOptions):
"""run lli via the legacy mx java launcher (instead of via the current GraalVM)"""
vmArgs, sulongArgs = truffle_extract_VM_args(args)
dists = []
if "tools" in (s.name for s in mx.suites()):
dists.append('CHROMEINSPECTOR')
return mx.run_java(getCommonOptions(False) + vmArgs + get_classpath_options(dists) + ["com.oracle.truffle.llvm.launcher.LLVMLauncher"] + sulongArgs, timeout=timeout, nonZeroIsFatal=nonZeroIsFatal, out=out, err=err)
def pylint(args):
rcfile = join(dirname(mx.__file__), '.pylintrc')
pythonpath = dirname(mx.__file__)
for suite in mx.suites(True):
pythonpath = os.pathsep.join([pythonpath, suite.mxDir])
def findfiles_by_vc(pyfiles):
for suite in mx.suites(True, includeBinary=False):
if not suite.primary:
continue
files = suite.vc.locate(suite.mxDir, ['*.py'])
for pyfile in files:
pyfile = join(suite.mxDir, pyfile)
if exists(pyfile):
pyfiles.append(pyfile)
pyfiles = []
findfiles_by_vc(pyfiles)
env = os.environ.copy()
env['PYTHONPATH'] = pythonpath
for pyfile in pyfiles:
mx.log('Running pylint on ' + pyfile + '...')
mx.run(['pylint', '--reports=n', '--rcfile=' + rcfile, pyfile], env=env)
return 0
def ideclean(args):
"""remove all IDE project configurations"""
def rm(path):
if exists(path):
os.remove(path)
for s in mx.suites() + [mx._mx_suite]:
rm(join(s.get_mx_output_dir(), 'eclipse-config.zip'))
rm(join(s.get_mx_output_dir(), 'netbeans-config.zip'))
shutil.rmtree(join(s.dir, '.idea'), ignore_errors=True)
for p in mx.projects() + mx._mx_suite.projects:
if not p.isJavaProject():
continue
shutil.rmtree(join(p.dir, '.settings'), ignore_errors=True)
shutil.rmtree(join(p.dir, '.externalToolBuilders'), ignore_errors=True)
shutil.rmtree(join(p.dir, 'nbproject'), ignore_errors=True)
rm(join(p.dir, '.classpath'))
rm(join(p.dir, '.checkstyle'))
rm(join(p.dir, '.project'))
rm(join(p.dir, '.factorypath'))
rm(join(p.dir, p.name + '.iml'))
rm(join(p.dir, 'build.xml'))
rm(join(p.dir, 'eclipse-build.xml'))
try:
rm(join(p.dir, p.name + '.jar'))
except:
mx.log_error("Error removing {0}".format(p.name + '.jar'))
for d in mx._dists.values():
if not d.isJARDistribution():
continue
if d.get_ide_project_dir():
shutil.rmtree(d.get_ide_project_dir(), ignore_errors=True)
def netbeansinit(args,
refreshOnly=False,
buildProcessorJars=True,
doFsckProjects=True):
"""(re)generate NetBeans project configurations"""
jdks = set()
for suite in mx.suites(True) + [mx._mx_suite]:
_netbeansinit_suite(args, jdks, suite, refreshOnly, buildProcessorJars)
if doFsckProjects and not refreshOnly:
mx_ideconfig.fsckprojects([])
mx.log('If using NetBeans:')
# http://stackoverflow.com/questions/24720665/cant-resolve-jdk-internal-package
mx.log(
' 1. Edit etc/netbeans.conf in your NetBeans installation and modify netbeans_default_options variable to include "-J-DCachingArchiveProvider.disableCtSym=true"'
)
mx.log(
' 2. Ensure that the following platform(s) are defined (Tools -> Java Platforms):'
)
for jdk in jdks:
mx.log(' JDK_' + str(jdk.version))
mx.log(
' 3. Open/create a Project Group for the directory containing the projects (File -> Project Group -> New Group... -> Folder of Projects)'
)
def runLLVM(args=None, out=None, get_classpath_options=getClasspathOptions):
"""uses Sulong to execute a LLVM IR file"""
vmArgs, sulongArgs = truffle_extract_VM_args(args)
dists = []
if "tools" in (s.name for s in mx.suites()):
dists.append('CHROMEINSPECTOR')
return mx.run_java(getCommonOptions(False) + vmArgs + get_classpath_options(dists) + ["com.oracle.truffle.llvm.launcher.LLVMLauncher"] + sulongArgs, out=out)
def findfiles_by_vc(pyfiles):
for suite in mx.suites(True, includeBinary=False):
if not suite.primary:
continue
files = suite.vc.locate(suite.mxDir, ['*.py'])
for pyfile in files:
pyfile = join(suite.mxDir, pyfile)
if exists(pyfile):
pyfiles.append(pyfile)
def _get_src_dir(projectname):
for suite in mx.suites():
for p in suite.projects:
if p.name == projectname:
if len(p.source_dirs()) > 0:
return p.source_dirs()[0]
else:
return p.dir
mx.abort("Could not find src dir for project %s" % projectname)
def netbeansinit(args,
refreshOnly=False,
buildProcessorJars=True,
doFsckProjects=True):
"""(re)generate NetBeans project configurations"""
for suite in mx.suites(True) + [mx._mx_suite]:
_netbeansinit_suite(args, suite, refreshOnly, buildProcessorJars)
if doFsckProjects and not refreshOnly:
mx_ideconfig.fsckprojects([])
def native_image_option_properties(option_kind, option_flag, native_image_root):
target_dir = join(native_image_root, option_kind, option_flag)
target_path = remove_existing_symlink(join(target_dir, 'native-image.properties'))
option_properties = None
for suite in mx.suites():
candidate = join(suite.mxDir, option_kind + '-' + option_flag + '.properties')
if exists(candidate):
option_properties = candidate
if option_properties:
mx.logv('Add symlink to ' + str(option_properties))
mkpath(target_dir)
symlink_or_copy(option_properties, target_path)
def symlink_or_copy(target_path, dest_path):
# Follow symbolic links in case they go outside my suite directories.
real_target_path = os.path.realpath(target_path)
if any(real_target_path.startswith(s.dir) for s in mx.suites(includeBinary=False)):
# Symbolic link to files in my suites.
sym_target = os.path.relpath(real_target_path, dirname(dest_path))
try:
os.symlink(sym_target, dest_path)
except AttributeError:
# no `symlink` on Windows
copy2(real_target_path, dest_path)
else:
# Else copy the file to so it can not change out from under me.
copy2(real_target_path, dest_path)
def runLLVMMul(args=None,
out=None,
err=None,
timeout=None,
nonZeroIsFatal=True,
get_classpath_options=getClasspathOptions):
"""run multi-context java launcher"""
vmArgs, sulongArgs = truffle_extract_VM_args(args)
dists = []
if "tools" in (s.name for s in mx.suites()):
dists.append('CHROMEINSPECTOR')
return mx.run_java(
getCommonOptions(False) + vmArgs + get_classpath_options(dists) +
["com.oracle.truffle.llvm.launcher.LLVMMultiContextLauncher"] +
sulongArgs,
timeout=timeout,
nonZeroIsFatal=nonZeroIsFatal,
out=out,
err=err)
def intellijinit(args, refreshOnly=False, doFsckProjects=True, mx_python_modules=True, java_modules=True,
generate_external_projects=True, native_projects=False):
# In a multiple suite context, the .idea directory in each suite
# has to be complete and contain information that is repeated
# in dependent suites.
declared_modules = set()
referenced_modules = set()
sdks = intellij_read_sdks()
for suite in mx.suites(True) + ([mx._mx_suite] if mx_python_modules else []):
_intellij_suite(args, suite, declared_modules, referenced_modules, sdks, refreshOnly, mx_python_modules,
generate_external_projects, java_modules and not suite.isBinarySuite(), suite != mx.primary_suite(),
generate_native_projects=native_projects)
if len(referenced_modules - declared_modules) != 0:
mx.abort('Some referenced modules are missing from modules.xml: {}'.format(referenced_modules - declared_modules))
if mx_python_modules:
# mx module
moduleXml = mx.XMLDoc()
moduleXml.open('module', attributes={'type': 'PYTHON_MODULE', 'version': '4'})
moduleXml.open('component', attributes={'name': 'NewModuleRootManager', 'inherit-compiler-output': 'true'})
moduleXml.element('exclude-output')
moduleXml.open('content', attributes={'url': 'file://$MODULE_DIR$'})
moduleXml.element('sourceFolder', attributes={'url': 'file://$MODULE_DIR$', 'isTestSource': 'false'})
for d in set((p.subDir for p in mx._mx_suite.projects if p.subDir)):
moduleXml.element('excludeFolder', attributes={'url': 'file://$MODULE_DIR$/' + d})
if dirname(mx._mx_suite.get_output_root()) == mx._mx_suite.dir:
moduleXml.element('excludeFolder', attributes={'url': 'file://$MODULE_DIR$/' + basename(mx._mx_suite.get_output_root())})
moduleXml.close('content')
moduleXml.element('orderEntry', attributes={'type': 'jdk', 'jdkType': intellij_python_sdk_type, 'jdkName': intellij_get_python_sdk_name(sdks)})
moduleXml.element('orderEntry', attributes={'type': 'sourceFolder', 'forTests': 'false'})
moduleXml.close('component')
moduleXml.close('module')
mxModuleFile = join(mx._mx_suite.dir, basename(mx._mx_suite.dir) + '.iml')
mx.update_file(mxModuleFile, moduleXml.xml(indent=' ', newl='\n'))
if doFsckProjects and not refreshOnly:
mx_ideconfig.fsckprojects([])
def testdownstream(suite,
repoUrls,
relTargetSuiteDir,
mxCommands,
branch=None):
"""
Tests a downstream repo against the current working directory state of `suite`.
:param mx.Suite suite: the suite to test against the downstream repo
:param list repoUrls: URLs of downstream repos to clone, the first of which is the repo being tested
:param str relTargetSuiteDir: directory of the downstream suite to test relative to the top level
directory of the downstream repo being tested
:param list mxCommands: argument lists for the mx commands run in downstream suite being tested
:param str branch: name(s) of branch to look for in downstream repo(s)
"""
assert len(repoUrls) > 0
repoUrls = [mx_urlrewrites.rewriteurl(url) for url in repoUrls]
workDir = join(suite.get_output_root(), 'testdownstream')
# A mirror of each suites in the same repo as `suite` is created via copying
rel_mirror = os.path.relpath(suite.dir,
mx.SuiteModel.siblings_dir(suite.dir))
in_subdir = os.sep in rel_mirror
suites_in_repo = [suite]
if in_subdir:
base = os.path.dirname(suite.dir)
for e in os.listdir(base):
candidate = join(base, e)
if candidate != suite.dir:
mxDir = mx._is_suite_dir(candidate)
if mxDir:
matches = [s for s in mx.suites() if s.dir == candidate]
if len(matches) == 0:
suites_in_repo.append(
mx.SourceSuite(mxDir, primary=False, load=False))
else:
suites_in_repo.append(matches[0])
if suite.vc:
vc_metadir = mx._safe_path(mx.VC.get_vc(suite.vc_dir).metadir())
blacklist = {suite.vc_dir: [join(suite.vc_dir, vc_metadir)]}
else:
blacklist = {}
for suite_in_repo in suites_in_repo:
output_root = mx._safe_path(suite_in_repo.get_output_root())
blacklist.setdefault(dirname(output_root), []).append(output_root)
def omitted_dirs(d, names):
mx.log('Copying ' + d)
to_omit = []
for blacklisted_dir in blacklist.get(d, []):
mx.log('Omitting ' + blacklisted_dir)
to_omit.append(basename(blacklisted_dir))
return to_omit
if suite.vc_dir and suite.dir != suite.vc_dir:
mirror = join(workDir, basename(suite.vc_dir))
else:
mirror = join(workDir, suite.name)
if exists(mirror):
mx.rmtree(mirror)
mx.copytree(suite.vc_dir, mirror, ignore=omitted_dirs, symlinks=True)
targetDir = None
for repoUrl in repoUrls:
# Deduce a target name from the target URL
url = _urllib_parse.urlparse(repoUrl)
targetName = url.path
if targetName.rfind('/') != -1:
targetName = targetName[targetName.rfind('/') + 1:]
if targetName.endswith('.git'):
targetName = targetName[0:-len('.git')]
repoWorkDir = join(workDir, targetName)
git = mx.GitConfig()
if exists(repoWorkDir):
git.pull(repoWorkDir)
else:
git.clone(repoUrl, repoWorkDir)
if branch is None:
branch = []
elif isinstance(branch, str):
branch = [branch]
else:
assert isinstance(branch, list)
# fall back to the branch of the main repo
active_branch = git.active_branch(suite.dir, abortOnError=False)
if active_branch:
branch.append(active_branch)
updated = False
for branch_name in branch:
if git.update_to_branch(repoWorkDir,
branch_name,
abortOnError=False):
updated = True
break
if not updated:
mx.warn('Could not update {} to any of the following branches: {}'.
format(repoWorkDir, ', '.join(branch)))
if not targetDir:
targetDir = repoWorkDir
assert not isabs(relTargetSuiteDir)
targetSuiteDir = join(targetDir, relTargetSuiteDir)
assert targetSuiteDir.startswith(targetDir)
mxpy = None if suite != mx._mx_suite else join(mirror, 'mx.py')
for command in mxCommands:
mx.logv('[running "mx ' + ' '.join(command) + '" in ' +
targetSuiteDir + ']')
mx.run_mx(command, targetSuiteDir, mxpy=mxpy)
def _intellij_suite(args, s, declared_modules, referenced_modules, sdks, refreshOnly=False, mx_python_modules=False,
generate_external_projects=True, java_modules=True, module_files_only=False, generate_native_projects=False):
libraries = set()
jdk_libraries = set()
project_dir = s.dir
ideaProjectDirectory = join(project_dir, '.idea')
modulesXml = mx.XMLDoc()
if not module_files_only and not s.isBinarySuite():
mx.ensure_dir_exists(ideaProjectDirectory)
nameFile = join(ideaProjectDirectory, '.name')
mx.update_file(nameFile, s.name)
modulesXml.open('project', attributes={'version': '4'})
modulesXml.open('component', attributes={'name': 'ProjectModuleManager'})
modulesXml.open('modules')
def _intellij_exclude_if_exists(xml, p, name, output=False):
root = p.get_output_root() if output else p.dir
path = join(root, name)
if exists(path):
excludeRoot = p.get_output_root() if output else '$MODULE_DIR$'
excludePath = join(excludeRoot, name)
xml.element('excludeFolder', attributes={'url':'file://' + excludePath})
annotationProcessorProfiles = {}
def _complianceToIntellijLanguageLevel(compliance):
# they changed the name format starting with JDK_10
if compliance.value >= 10:
# Lastest Idea 2018.2 only understands JDK_11 so clamp at that value
return 'JDK_' + str(min(compliance.value, 11))
return 'JDK_1_' + str(compliance.value)
def _intellij_external_project(externalProjects, sdks, host):
if externalProjects:
for project_name, project_definition in externalProjects.items():
if not project_definition.get('path', None):
mx.abort("external project {} is missing path attribute".format(project_name))
if not project_definition.get('type', None):
mx.abort("external project {} is missing type attribute".format(project_name))
supported = ['path', 'type', 'source', 'test', 'excluded', 'load_path']
unknown = set(project_definition.keys()) - frozenset(supported)
if unknown:
mx.abort("There are unsupported {} keys in {} external project".format(unknown, project_name))
path = os.path.realpath(join(host.dir, project_definition["path"]))
module_type = project_definition["type"]
moduleXml = mx.XMLDoc()
moduleXml.open('module',
attributes={'type': {'ruby': 'RUBY_MODULE',
'python': 'PYTHON_MODULE',
'web': 'WEB_MODULE'}.get(module_type, 'UKNOWN_MODULE'),
'version': '4'})
moduleXml.open('component',
attributes={'name': 'NewModuleRootManager', 'inherit-compiler-output': 'true'})
moduleXml.element('exclude-output')
moduleXml.open('content', attributes={'url': 'file://$MODULE_DIR$'})
for name in project_definition.get('source', []):
moduleXml.element('sourceFolder',
attributes={'url':'file://$MODULE_DIR$/' + name, 'isTestSource': str(False)})
for name in project_definition.get('test', []):
moduleXml.element('sourceFolder',
attributes={'url':'file://$MODULE_DIR$/' + name, 'isTestSource': str(True)})
for name in project_definition.get('excluded', []):
_intellij_exclude_if_exists(moduleXml, type('', (object,), {"dir": path})(), name)
moduleXml.close('content')
if module_type == "ruby":
moduleXml.element('orderEntry', attributes={'type': 'jdk', 'jdkType': intellij_ruby_sdk_type, 'jdkName': intellij_get_ruby_sdk_name(sdks)})
elif module_type == "python":
moduleXml.element('orderEntry', attributes={'type': 'jdk', 'jdkType': intellij_python_sdk_type, 'jdkName': intellij_get_python_sdk_name(sdks)})
elif module_type == "web":
# nothing to do
pass
else:
mx.abort("External project type {} not supported".format(module_type))
moduleXml.element('orderEntry', attributes={'type': 'sourceFolder', 'forTests': 'false'})
moduleXml.close('component')
load_paths = project_definition.get('load_path', [])
if load_paths:
if not module_type == "ruby":
mx.abort("load_path is supported only for ruby type external project")
moduleXml.open('component', attributes={'name': 'RModuleSettingsStorage'})
load_paths_attributes = {}
load_paths_attributes['number'] = str(len(load_paths))
for i, name in enumerate(load_paths):
load_paths_attributes["string" + str(i)] = "$MODULE_DIR$/" + name
moduleXml.element('LOAD_PATH', load_paths_attributes)
moduleXml.close('component')
moduleXml.close('module')
moduleFile = join(path, project_name + '.iml')
mx.update_file(moduleFile, moduleXml.xml(indent=' ', newl='\n'))
if not module_files_only:
declared_modules.add(project_name)
moduleFilePath = "$PROJECT_DIR$/" + os.path.relpath(moduleFile, s.dir)
modulesXml.element('module', attributes={'fileurl': 'file://' + moduleFilePath, 'filepath': moduleFilePath})
if generate_external_projects:
for p in s.projects_recursive() + mx._mx_suite.projects_recursive():
_intellij_external_project(getattr(p, 'externalProjects', None), sdks, p)
max_checkstyle_version = None
compilerXml = None
if java_modules:
if not module_files_only:
compilerXml = mx.XMLDoc()
compilerXml.open('project', attributes={'version': '4'})
# The IntelliJ parser seems to mishandle empty ADDITIONAL_OPTIONS_OVERRIDE elements
# so only emit the section if there will be something in it.
additionalOptionsOverrides = False
assert not s.isBinarySuite()
# create the modules (1 IntelliJ module = 1 mx project/distribution)
for p in s.projects_recursive() + mx._mx_suite.projects_recursive():
if not p.isJavaProject():
continue
jdk = mx.get_jdk(p.javaCompliance)
assert jdk
# Value of the $MODULE_DIR$ IntelliJ variable and parent directory of the .iml file.
module_dir = mx.ensure_dir_exists(p.dir)
processors = p.annotation_processors()
if processors:
annotationProcessorProfiles.setdefault((p.source_gen_dir_name(),) + tuple(processors), []).append(p)
intellijLanguageLevel = _complianceToIntellijLanguageLevel(p.javaCompliance)
moduleXml = mx.XMLDoc()
moduleXml.open('module', attributes={'type': 'JAVA_MODULE', 'version': '4'})
moduleXml.open('component', attributes={'name': 'NewModuleRootManager', 'LANGUAGE_LEVEL': intellijLanguageLevel, 'inherit-compiler-output': 'false'})
moduleXml.element('output', attributes={'url': 'file://$MODULE_DIR$/' + os.path.relpath(p.output_dir(), module_dir)})
moduleXml.open('content', attributes={'url': 'file://$MODULE_DIR$'})
for src in p.srcDirs:
srcDir = mx.ensure_dir_exists(join(p.dir, src))
moduleXml.element('sourceFolder', attributes={'url':'file://$MODULE_DIR$/' + os.path.relpath(srcDir, module_dir), 'isTestSource': str(p.is_test_project())})
for name in ['.externalToolBuilders', '.settings', 'nbproject']:
_intellij_exclude_if_exists(moduleXml, p, name)
moduleXml.close('content')
if processors:
moduleXml.open('content', attributes={'url': 'file://' + p.get_output_root()})
genDir = p.source_gen_dir()
mx.ensure_dir_exists(genDir)
moduleXml.element('sourceFolder', attributes={'url':'file://' + p.source_gen_dir(), 'isTestSource': str(p.is_test_project()), 'generated': 'true'})
for name in [basename(p.output_dir())]:
_intellij_exclude_if_exists(moduleXml, p, name, output=True)
moduleXml.close('content')
moduleXml.element('orderEntry', attributes={'type': 'sourceFolder', 'forTests': 'false'})
proj = p
dependencies_project_packages = set()
def should_process_dep(dep, edge):
if dep.isTARDistribution() or dep.isNativeProject() or dep.isArchivableProject() or dep.isResourceLibrary():
mx.logv("Ignoring dependency from {} to {}".format(proj.name, dep.name))
return False
return True
def process_dep(dep, edge):
if dep is proj:
return
if dep.isLibrary() or dep.isJARDistribution() or dep.isMavenProject():
libraries.add(dep)
moduleXml.element('orderEntry', attributes={'type': 'library', 'name': dep.name, 'level': 'project'})
elif dep.isJavaProject():
dependencies_project_packages.update(dep.defined_java_packages())
referenced_modules.add(dep.name)
moduleXml.element('orderEntry', attributes={'type': 'module', 'module-name': dep.name})
elif dep.isJdkLibrary():
jdk_libraries.add(dep)
if jdk.javaCompliance < dep.jdkStandardizedSince:
moduleXml.element('orderEntry', attributes={'type': 'library', 'name': dep.name, 'level': 'project'})
else:
mx.logv("{} skipping {} for {}".format(p, dep, jdk)) #pylint: disable=undefined-loop-variable
elif dep.isJreLibrary():
pass
elif dep.isClasspathDependency():
moduleXml.element('orderEntry', attributes={'type': 'library', 'name': dep.name, 'level': 'project'})
else:
mx.abort("Dependency not supported: {0} ({1})".format(dep, dep.__class__.__name__))
p.walk_deps(preVisit=should_process_dep, visit=process_dep, ignoredEdges=[mx.DEP_EXCLUDED])
moduleXml.element('orderEntry', attributes={'type': 'jdk', 'jdkType': intellij_java_sdk_type, 'jdkName': intellij_get_java_sdk_name(sdks, jdk)})
moduleXml.close('component')
if compilerXml and jdk.javaCompliance >= '9':
moduleDeps = p.get_concealed_imported_packages(jdk=jdk)
if moduleDeps:
exports = sorted([(m, pkgs) for m, pkgs in moduleDeps.items() if dependencies_project_packages.isdisjoint(pkgs)])
if exports:
args = []
exported_modules = set()
for m, pkgs in exports:
args += ['--add-exports={}/{}=ALL-UNNAMED'.format(m, pkg) for pkg in pkgs]
exported_modules.add(m)
roots = set(jdk.get_root_modules())
observable_modules = jdk.get_modules()
default_module_graph = mx_javamodules.get_transitive_closure(roots, observable_modules)
module_graph = mx_javamodules.get_transitive_closure(roots | exported_modules, observable_modules)
extra_modules = module_graph - default_module_graph
if extra_modules:
args.append('--add-modules=' + ','.join((m.name for m in extra_modules)))
if not additionalOptionsOverrides:
additionalOptionsOverrides = True
compilerXml.open('component', {'name': 'JavacSettings'})
compilerXml.open('option', {'name': 'ADDITIONAL_OPTIONS_OVERRIDE'})
compilerXml.element('module', {'name': p.name, 'options': ' '.join(args)})
# Checkstyle
csConfig, checkstyleVersion, checkstyleProj = p.get_checkstyle_config()
if csConfig:
max_checkstyle_version = max(max_checkstyle_version, mx.VersionSpec(checkstyleVersion)) if max_checkstyle_version else mx.VersionSpec(checkstyleVersion)
moduleXml.open('component', attributes={'name': 'CheckStyle-IDEA-Module'})
moduleXml.open('option', attributes={'name': 'configuration'})
moduleXml.open('map')
moduleXml.element('entry', attributes={'key': "checkstyle-version", 'value': checkstyleVersion})
moduleXml.element('entry', attributes={'key': "active-configuration", 'value': "PROJECT_RELATIVE:" + join(checkstyleProj.dir, ".checkstyle_checks.xml") + ":" + checkstyleProj.name})
moduleXml.close('map')
moduleXml.close('option')
moduleXml.close('component')
moduleXml.close('module')
moduleFile = join(module_dir, p.name + '.iml')
mx.update_file(moduleFile, moduleXml.xml(indent=' ', newl='\n').rstrip())
if not module_files_only:
declared_modules.add(p.name)
moduleFilePath = "$PROJECT_DIR$/" + os.path.relpath(moduleFile, project_dir)
modulesXml.element('module', attributes={'fileurl': 'file://' + moduleFilePath, 'filepath': moduleFilePath})
if additionalOptionsOverrides:
compilerXml.close('option')
compilerXml.close('component')
if mx_python_modules:
def _python_module(suite):
"""
Gets a tuple describing the IntelliJ module for the python sources of `suite`. The tuple
consists of the module name, module directory and the name of the .iml in the module directory.
"""
name = basename(suite.mxDir)
module_dir = suite.mxDir
return name, mx.ensure_dir_exists(module_dir), name + '.iml'
def _add_declared_module(suite):
if not module_files_only:
name, module_dir, iml_file = _python_module(suite)
declared_modules.add(name)
moduleFilePath = "$PROJECT_DIR$/" + os.path.relpath(join(module_dir, iml_file), project_dir)
modulesXml.element('module', attributes={'fileurl': 'file://' + moduleFilePath, 'filepath': moduleFilePath})
# mx.<suite> python module:
_, module_dir, iml_file = _python_module(s)
moduleXml = mx.XMLDoc()
moduleXml.open('module', attributes={'type': 'PYTHON_MODULE', 'version': '4'})
moduleXml.open('component', attributes={'name': 'NewModuleRootManager', 'inherit-compiler-output': 'true'})
moduleXml.element('exclude-output')
if s.name == 'mx':
# MX itself is special. Python sources are also in the parent folder.
moduleXml.open('content', attributes={'url': 'file://$MODULE_DIR$/..'})
moduleXml.element('sourceFolder', attributes={'url': 'file://$MODULE_DIR$/..', 'isTestSource': 'false'})
else:
moduleXml.open('content', attributes={'url': 'file://$MODULE_DIR$'})
moduleXml.element('sourceFolder', attributes={'url': 'file://$MODULE_DIR$/' + os.path.relpath(s.mxDir, module_dir), 'isTestSource': 'false'})
for d in os.listdir(s.mxDir):
directory = join(s.mxDir, d)
if isdir(directory) and mx.dir_contains_files_recursively(directory, r".*\.java"):
moduleXml.element('excludeFolder', attributes={'url': 'file://$MODULE_DIR$/' + os.path.relpath(directory, module_dir)})
moduleXml.close('content')
moduleXml.element('orderEntry', attributes={'type': 'jdk', 'jdkType': intellij_python_sdk_type, 'jdkName': intellij_get_python_sdk_name(sdks)})
moduleXml.element('orderEntry', attributes={'type': 'sourceFolder', 'forTests': 'false'})
processed_suites = {s.name}
def _mx_projects_suite(visited_suite, suite_import):
if suite_import.name in processed_suites:
return
processed_suites.add(suite_import.name)
dep_suite = mx.suite(suite_import.name)
dep_module_name, _, _ = _python_module(dep_suite)
moduleXml.element('orderEntry', attributes={'type': 'module', 'module-name': dep_module_name})
_add_declared_module(dep_suite)
dep_suite.visit_imports(_mx_projects_suite)
s.visit_imports(_mx_projects_suite)
if s.name != 'mx':
moduleXml.element('orderEntry', attributes={'type': 'module', 'module-name': 'mx.mx'})
moduleXml.close('component')
moduleXml.close('module')
moduleFile = join(module_dir, iml_file)
mx.update_file(moduleFile, moduleXml.xml(indent=' ', newl='\n'))
_add_declared_module(s)
_add_declared_module(mx._mx_suite)
if generate_native_projects:
_intellij_native_projects(s, module_files_only, declared_modules, modulesXml)
if generate_external_projects:
_intellij_external_project(s.suiteDict.get('externalProjects', None), sdks, s)
if not module_files_only:
modulesXml.close('modules')
modulesXml.close('component')
modulesXml.close('project')
moduleXmlFile = join(ideaProjectDirectory, 'modules.xml')
mx.update_file(moduleXmlFile, modulesXml.xml(indent=' ', newl='\n'))
if java_modules and not module_files_only:
unique_library_file_names = set()
librariesDirectory = mx.ensure_dir_exists(join(ideaProjectDirectory, 'libraries'))
mx.ensure_dir_exists(librariesDirectory)
def make_library(name, path, source_path, suite_dir):
libraryXml = mx.XMLDoc()
libraryXml.open('component', attributes={'name': 'libraryTable'})
libraryXml.open('library', attributes={'name': name})
libraryXml.open('CLASSES')
pathX = mx.relpath_or_absolute(path, suite_dir, prefix='$PROJECT_DIR$')
libraryXml.element('root', attributes={'url': 'jar://' + pathX + '!/'})
libraryXml.close('CLASSES')
libraryXml.element('JAVADOC')
if sourcePath:
libraryXml.open('SOURCES')
if os.path.isdir(sourcePath):
sourcePathX = mx.relpath_or_absolute(sourcePath, suite_dir, prefix='$PROJECT_DIR$')
libraryXml.element('root', attributes={'url': 'file://' + sourcePathX})
else:
source_pathX = mx.relpath_or_absolute(source_path, suite_dir, prefix='$PROJECT_DIR$')
libraryXml.element('root', attributes={'url': 'jar://' + source_pathX + '!/'})
libraryXml.close('SOURCES')
else:
libraryXml.element('SOURCES')
libraryXml.close('library')
libraryXml.close('component')
libraryFile = join(librariesDirectory, _intellij_library_file_name(name, unique_library_file_names))
return mx.update_file(libraryFile, libraryXml.xml(indent=' ', newl='\n'))
# Setup the libraries that were used above
for library in libraries:
sourcePath = None
if library.isLibrary():
path = library.get_path(True)
if library.sourcePath:
sourcePath = library.get_source_path(True)
elif library.isMavenProject():
path = library.get_path(True)
sourcePath = library.get_source_path(True)
elif library.isJARDistribution():
path = library.path
if library.sourcesPath:
sourcePath = library.sourcesPath
elif library.isClasspathDependency():
path = library.classpath_repr()
else:
mx.abort('Dependency not supported: {} ({})'.format(library.name, library.__class__.__name__))
make_library(library.name, path, sourcePath, s.dir)
jdk = mx.get_jdk()
updated = False
for library in jdk_libraries:
if library.classpath_repr(jdk) is not None:
if make_library(library.name, library.classpath_repr(jdk), library.get_source_path(jdk), s.dir):
updated = True
if jdk_libraries and updated:
mx.log("Setting up JDK libraries using {0}".format(jdk))
# Set annotation processor profiles up, and link them to modules in compiler.xml
compilerXml.open('component', attributes={'name': 'CompilerConfiguration'})
compilerXml.element('option', attributes={'name': "DEFAULT_COMPILER", 'value': 'Javac'})
# using the --release option with javac interferes with using --add-modules which is required for some projects
compilerXml.element('option', attributes={'name': "USE_RELEASE_OPTION", 'value': 'false'})
compilerXml.element('resourceExtensions')
compilerXml.open('wildcardResourcePatterns')
compilerXml.element('entry', attributes={'name': '!?*.java'})
compilerXml.close('wildcardResourcePatterns')
if annotationProcessorProfiles:
compilerXml.open('annotationProcessing')
for t, modules in sorted(annotationProcessorProfiles.items()):
source_gen_dir = t[0]
processors = t[1:]
compilerXml.open('profile', attributes={'default': 'false', 'name': '-'.join([ap.name for ap in processors]) + "-" + source_gen_dir, 'enabled': 'true'})
compilerXml.element('sourceOutputDir', attributes={'name': join(os.pardir, source_gen_dir)})
compilerXml.element('sourceTestOutputDir', attributes={'name': join(os.pardir, source_gen_dir)})
compilerXml.open('processorPath', attributes={'useClasspath': 'false'})
# IntelliJ supports both directories and jars on the annotation processor path whereas
# Eclipse only supports jars.
for apDep in processors:
def processApDep(dep, edge):
if dep.isLibrary() or dep.isJARDistribution():
compilerXml.element('entry', attributes={'name': mx.relpath_or_absolute(dep.path, s.dir, prefix='$PROJECT_DIR$')})
elif dep.isProject():
compilerXml.element('entry', attributes={'name': mx.relpath_or_absolute(dep.output_dir(), s.dir, prefix='$PROJECT_DIR$')})
apDep.walk_deps(visit=processApDep)
compilerXml.close('processorPath')
for module in modules:
compilerXml.element('module', attributes={'name': module.name})
compilerXml.close('profile')
compilerXml.close('annotationProcessing')
compilerXml.close('component')
if compilerXml:
compilerXml.close('project')
compilerFile = join(ideaProjectDirectory, 'compiler.xml')
mx.update_file(compilerFile, compilerXml.xml(indent=' ', newl='\n'))
if not module_files_only:
# Write misc.xml for global JDK config
miscXml = mx.XMLDoc()
miscXml.open('project', attributes={'version' : '4'})
if java_modules:
mainJdk = mx.get_jdk()
miscXml.open('component', attributes={'name' : 'ProjectRootManager', 'version': '2', 'languageLevel': _complianceToIntellijLanguageLevel(mainJdk.javaCompliance), 'project-jdk-name': intellij_get_java_sdk_name(sdks, mainJdk), 'project-jdk-type': intellij_java_sdk_type})
miscXml.element('output', attributes={'url' : 'file://$PROJECT_DIR$/' + os.path.relpath(s.get_output_root(), s.dir)})
miscXml.close('component')
else:
miscXml.element('component', attributes={'name' : 'ProjectRootManager', 'version': '2', 'project-jdk-name': intellij_get_python_sdk_name(sdks), 'project-jdk-type': intellij_python_sdk_type})
miscXml.close('project')
miscFile = join(ideaProjectDirectory, 'misc.xml')
mx.update_file(miscFile, miscXml.xml(indent=' ', newl='\n'))
# Generate a default configuration for debugging Graal
runConfig = mx.XMLDoc()
runConfig.open('component', attributes={'name' : 'ProjectRunConfigurationManager'})
runConfig.open('configuration', attributes={'default' :'false', 'name' : 'GraalDebug', 'type' : 'Remote', 'factoryName': 'Remote'})
runConfig.element('option', attributes={'name' : 'USE_SOCKET_TRANSPORT', 'value' : 'true'})
runConfig.element('option', attributes={'name' : 'SERVER_MODE', 'value' : 'false'})
runConfig.element('option', attributes={'name' : 'SHMEM_ADDRESS', 'value' : 'javadebug'})
runConfig.element('option', attributes={'name' : 'HOST', 'value' : 'localhost'})
runConfig.element('option', attributes={'name' : 'PORT', 'value' : '8000'})
runConfig.open('RunnerSettings', attributes={'RunnerId' : 'Debug'})
runConfig.element('option', attributes={'name' : 'DEBUG_PORT', 'value' : '8000'})
runConfig.element('option', attributes={'name' : 'LOCAL', 'value' : 'false'})
runConfig.close('RunnerSettings')
runConfig.element('method')
runConfig.close('configuration')
runConfig.close('component')
runConfigFile = join(ideaProjectDirectory, 'runConfigurations', 'GraalDebug.xml')
mx.ensure_dir_exists(join(ideaProjectDirectory, 'runConfigurations'))
mx.update_file(runConfigFile, runConfig.xml(indent=' ', newl='\n'))
if java_modules:
# Eclipse formatter config
corePrefsSources = s.eclipse_settings_sources().get('org.eclipse.jdt.core.prefs')
uiPrefsSources = s.eclipse_settings_sources().get('org.eclipse.jdt.ui.prefs')
if corePrefsSources:
miscXml = mx.XMLDoc()
miscXml.open('project', attributes={'version' : '4'})
out = StringIO()
print('# GENERATED -- DO NOT EDIT', file=out)
for source in corePrefsSources:
print('# Source:', source, file=out)
with open(source) as fileName:
for line in fileName:
if line.startswith('org.eclipse.jdt.core.formatter.'):
print(line.strip(), file=out)
formatterConfigFile = join(ideaProjectDirectory, 'EclipseCodeFormatter.prefs')
mx.update_file(formatterConfigFile, out.getvalue())
importConfigFile = None
if uiPrefsSources:
out = StringIO()
print('# GENERATED -- DO NOT EDIT', file=out)
for source in uiPrefsSources:
print('# Source:', source, file=out)
with open(source) as fileName:
for line in fileName:
if line.startswith('org.eclipse.jdt.ui.importorder') \
or line.startswith('org.eclipse.jdt.ui.ondemandthreshold') \
or line.startswith('org.eclipse.jdt.ui.staticondemandthreshold'):
print(line.strip(), file=out)
importConfigFile = join(ideaProjectDirectory, 'EclipseImports.prefs')
mx.update_file(importConfigFile, out.getvalue())
miscXml.open('component', attributes={'name' : 'EclipseCodeFormatterProjectSettings'})
miscXml.open('option', attributes={'name' : 'projectSpecificProfile'})
miscXml.open('ProjectSpecificProfile')
miscXml.element('option', attributes={'name' : 'formatter', 'value' : 'ECLIPSE'})
custom_eclipse_exe = mx.get_env('ECLIPSE_EXE')
if custom_eclipse_exe:
custom_eclipse = dirname(custom_eclipse_exe)
if mx.is_darwin():
custom_eclipse = join(dirname(custom_eclipse), 'Eclipse')
if not exists(custom_eclipse_exe):
mx.abort('Custom eclipse "{}" does not exist'.format(custom_eclipse_exe))
miscXml.element('option', attributes={'name' : 'eclipseVersion', 'value' : 'CUSTOM'})
miscXml.element('option', attributes={'name' : 'pathToEclipse', 'value' : custom_eclipse})
miscXml.element('option', attributes={'name' : 'pathToConfigFileJava', 'value' : '$PROJECT_DIR$/.idea/' + basename(formatterConfigFile)})
if importConfigFile:
miscXml.element('option', attributes={'name' : 'importOrderConfigFilePath', 'value' : '$PROJECT_DIR$/.idea/' + basename(importConfigFile)})
miscXml.element('option', attributes={'name' : 'importOrderFromFile', 'value' : 'true'})
miscXml.close('ProjectSpecificProfile')
miscXml.close('option')
miscXml.close('component')
miscXml.close('project')
miscFile = join(ideaProjectDirectory, 'eclipseCodeFormatter.xml')
mx.update_file(miscFile, miscXml.xml(indent=' ', newl='\n'))
if java_modules:
# Write codestyle settings
mx.ensure_dir_exists(join(ideaProjectDirectory, 'codeStyles'))
codeStyleConfigXml = mx.XMLDoc()
codeStyleConfigXml.open('component', attributes={'name': 'ProjectCodeStyleConfiguration'})
codeStyleConfigXml.open('state')
codeStyleConfigXml.element('option', attributes={'name': 'USE_PER_PROJECT_SETTINGS', 'value': 'true'})
codeStyleConfigXml.close('state')
codeStyleConfigXml.close('component')
codeStyleConfigFile = join(ideaProjectDirectory, 'codeStyles', 'codeStyleConfig.xml')
mx.update_file(codeStyleConfigFile, codeStyleConfigXml.xml(indent=' ', newl='\n'))
codeStyleProjectXml = mx.XMLDoc()
codeStyleProjectXml.open('component', attributes={'name': 'ProjectCodeStyleConfiguration'})
codeStyleProjectXml.open('code_scheme', attributes={'name': 'Project', 'version': '173'})
codeStyleProjectXml.open('JavaCodeStyleSettings')
# We cannot entirely disable wildcards import, but we can set the threshold to an insane number.
codeStyleProjectXml.element('option', attributes={'name': 'CLASS_COUNT_TO_USE_IMPORT_ON_DEMAND', 'value': '65536'})
codeStyleProjectXml.element('option', attributes={'name': 'NAMES_COUNT_TO_USE_IMPORT_ON_DEMAND', 'value': '65536'})
codeStyleProjectXml.close('JavaCodeStyleSettings')
codeStyleProjectXml.close('code_scheme')
codeStyleProjectXml.close('component')
codeStyleProjectFile = join(ideaProjectDirectory, 'codeStyles', 'Project.xml')
mx.update_file(codeStyleProjectFile, codeStyleProjectXml.xml(indent=' ', newl='\n'))
# Write checkstyle-idea.xml for the CheckStyle-IDEA
checkstyleXml = mx.XMLDoc()
checkstyleXml.open('project', attributes={'version': '4'})
checkstyleXml.open('component', attributes={'name': 'CheckStyle-IDEA'})
checkstyleXml.open('option', attributes={'name' : "configuration"})
checkstyleXml.open('map')
if max_checkstyle_version:
checkstyleXml.element('entry', attributes={'key': "checkstyle-version", 'value': str(max_checkstyle_version)})
# Initialize an entry for each style that is used
checkstyleConfigs = set([])
for p in s.projects_recursive():
if not p.isJavaProject():
continue
csConfig, checkstyleVersion, checkstyleProj = p.get_checkstyle_config()
if not csConfig or csConfig in checkstyleConfigs:
continue
checkstyleConfigs.add(csConfig)
checkstyleXml.element('entry', attributes={'key' : "location-" + str(len(checkstyleConfigs)), 'value': "PROJECT_RELATIVE:" + join(checkstyleProj.dir, ".checkstyle_checks.xml") + ":" + checkstyleProj.name})
checkstyleXml.close('map')
checkstyleXml.close('option')
checkstyleXml.close('component')
checkstyleXml.close('project')
checkstyleFile = join(ideaProjectDirectory, 'checkstyle-idea.xml')
mx.update_file(checkstyleFile, checkstyleXml.xml(indent=' ', newl='\n'))
# mx integration
def antTargetName(dist):
return 'archive_' + dist.name
def artifactFileName(dist):
return dist.name.replace('.', '_').replace('-', '_') + '.xml'
validDistributions = [dist for dist in mx.sorted_dists() if not dist.suite.isBinarySuite() and not dist.isTARDistribution()]
# 1) Make an ant file for archiving the distributions.
antXml = mx.XMLDoc()
antXml.open('project', attributes={'name': s.name, 'default': 'archive'})
for dist in validDistributions:
antXml.open('target', attributes={'name': antTargetName(dist)})
antXml.open('exec', attributes={'executable': sys.executable})
antXml.element('arg', attributes={'value': join(mx._mx_home, 'mx.py')})
antXml.element('arg', attributes={'value': 'archive'})
antXml.element('arg', attributes={'value': '@' + dist.name})
antXml.close('exec')
antXml.close('target')
antXml.close('project')
antFile = join(ideaProjectDirectory, 'ant-mx-archive.xml')
mx.update_file(antFile, antXml.xml(indent=' ', newl='\n'))
# 2) Tell IDEA that there is an ant-build.
ant_mx_archive_xml = 'file://$PROJECT_DIR$/.idea/ant-mx-archive.xml'
metaAntXml = mx.XMLDoc()
metaAntXml.open('project', attributes={'version': '4'})
metaAntXml.open('component', attributes={'name': 'AntConfiguration'})
metaAntXml.open('buildFile', attributes={'url': ant_mx_archive_xml})
metaAntXml.close('buildFile')
metaAntXml.close('component')
metaAntXml.close('project')
metaAntFile = join(ideaProjectDirectory, 'ant.xml')
mx.update_file(metaAntFile, metaAntXml.xml(indent=' ', newl='\n'))
# 3) Make an artifact for every distribution
validArtifactNames = {artifactFileName(dist) for dist in validDistributions}
artifactsDir = join(ideaProjectDirectory, 'artifacts')
mx.ensure_dir_exists(artifactsDir)
for fileName in os.listdir(artifactsDir):
filePath = join(artifactsDir, fileName)
if os.path.isfile(filePath) and fileName not in validArtifactNames:
os.remove(filePath)
for dist in validDistributions:
artifactXML = mx.XMLDoc()
artifactXML.open('component', attributes={'name': 'ArtifactManager'})
artifactXML.open('artifact', attributes={'build-on-make': 'true', 'name': dist.name})
artifactXML.open('output-path', data='$PROJECT_DIR$/mxbuild/artifacts/' + dist.name)
artifactXML.close('output-path')
artifactXML.open('properties', attributes={'id': 'ant-postprocessing'})
artifactXML.open('options', attributes={'enabled': 'true'})
artifactXML.open('file', data=ant_mx_archive_xml)
artifactXML.close('file')
artifactXML.open('target', data=antTargetName(dist))
artifactXML.close('target')
artifactXML.close('options')
artifactXML.close('properties')
artifactXML.open('root', attributes={'id': 'root'})
for javaProject in [dep for dep in dist.archived_deps() if dep.isJavaProject()]:
artifactXML.element('element', attributes={'id': 'module-output', 'name': javaProject.name})
for javaProject in [dep for dep in dist.deps if dep.isLibrary() or dep.isDistribution()]:
artifactXML.element('element', attributes={'id': 'artifact', 'artifact-name': javaProject.name})
artifactXML.close('root')
artifactXML.close('artifact')
artifactXML.close('component')
artifactFile = join(artifactsDir, artifactFileName(dist))
mx.update_file(artifactFile, artifactXML.xml(indent=' ', newl='\n'))
def intellij_scm_name(vc_kind):
if vc_kind == 'git':
return 'Git'
elif vc_kind == 'hg':
return 'hg4idea'
vcsXml = mx.XMLDoc()
vcsXml.open('project', attributes={'version': '4'})
vcsXml.open('component', attributes={'name': 'VcsDirectoryMappings'})
suites_for_vcs = mx.suites() + ([mx._mx_suite] if mx_python_modules else [])
sourceSuitesWithVCS = [vc_suite for vc_suite in suites_for_vcs if vc_suite.isSourceSuite() and vc_suite.vc is not None]
uniqueSuitesVCS = {(vc_suite.vc_dir, vc_suite.vc.kind) for vc_suite in sourceSuitesWithVCS}
for vcs_dir, kind in uniqueSuitesVCS:
vcsXml.element('mapping', attributes={'directory': vcs_dir, 'vcs': intellij_scm_name(kind)})
vcsXml.close('component')
vcsXml.close('project')
vcsFile = join(ideaProjectDirectory, 'vcs.xml')
mx.update_file(vcsFile, vcsXml.xml(indent=' ', newl='\n'))
def testdownstream(suite,
repoUrls,
relTargetSuiteDir,
mxCommands,
branch=None):
"""
Tests a downstream repo against the current working directory state of `suite`.
:param mx.Suite suite: the suite to test against the downstream repo
:param list repoUrls: URLs of downstream repos to clone, the first of which is the repo being tested
:param str relTargetSuiteDir: directory of the downstream suite to test relative to the top level
directory of the downstream repo being tested
:param list mxCommands: argument lists for the mx commands run in downstream suite being tested
:param str branch: name of branch to look for in downstream repo(s)
"""
assert len(repoUrls) > 0
repoUrls = [mx_urlrewrites.rewriteurl(url) for url in repoUrls]
workDir = join(suite.get_output_root(), 'testdownstream')
# A mirror of each suites in the same repo as `suite` is created via copying
rel_mirror = os.path.relpath(suite.dir,
mx.SuiteModel.siblings_dir(suite.dir))
in_subdir = os.sep in rel_mirror
suites_in_repo = [suite]
if in_subdir:
base = os.path.dirname(suite.dir)
for e in os.listdir(base):
candidate = join(base, e)
if candidate != suite.dir:
mxDir = mx._is_suite_dir(candidate)
if mxDir:
matches = [s for s in mx.suites() if s.dir == candidate]
if len(matches) == 0:
suites_in_repo.append(
mx.SourceSuite(mxDir, primary=False, load=False))
else:
suites_in_repo.append(matches[0])
for suite_in_repo in suites_in_repo:
if suite_in_repo.vc_dir and suite_in_repo.dir != suite_in_repo.vc_dir:
mirror = join(workDir, basename(suite_in_repo.vc_dir),
suite_in_repo.name)
else:
mirror = join(workDir, suite_in_repo.name)
if exists(mirror):
mx.rmtree(mirror)
output_root = mx._safe_path(suite_in_repo.get_output_root())
def ignore_output_root(d, names):
mx.log('Copying ' + d)
if d == os.path.dirname(output_root):
mx.log('Omitting ' + output_root)
return [os.path.basename(output_root)]
return []
mx.copytree(suite_in_repo.dir,
mirror,
ignore=ignore_output_root,
symlinks=True)
targetDir = None
for repoUrl in repoUrls:
# Deduce a target name from the target URL
url = _urllib_parse.urlparse(repoUrl)
targetName = url.path
if targetName.rfind('/') != -1:
targetName = targetName[targetName.rfind('/') + 1:]
if targetName.endswith('.git'):
targetName = targetName[0:-len('.git')]
repoWorkDir = join(workDir, targetName)
git = mx.GitConfig()
if exists(repoWorkDir):
git.pull(repoWorkDir)
else:
git.clone(repoUrl, repoWorkDir)
# See if there's a matching (non-master) branch and use it if there is
if not branch:
branch = git.git_command(
suite.dir, ['rev-parse', '--abbrev-ref', 'HEAD']).strip()
if branch != 'master':
git.git_command(repoWorkDir, ['checkout', branch],
abortOnError=False)
if not targetDir:
targetDir = repoWorkDir
assert not isabs(relTargetSuiteDir)
targetSuiteDir = join(targetDir, relTargetSuiteDir)
assert targetSuiteDir.startswith(targetDir)
mxpy = None if suite != mx._mx_suite else join(mirror, 'mx.py')
for command in mxCommands:
mx.logv('[running "mx ' + ' '.join(command) + '" in ' +
targetSuiteDir + ']')
mx.run_mx(command, targetSuiteDir, mxpy=mxpy)
def testdownstream(suite,
repoUrls,
relTargetSuiteDir,
mxCommands,
branch=None):
"""
Tests a downstream repo against the current working directory state of `suite`.
:param mx.Suite suite: the suite to test against the downstream repo
:param list repoUrls: URLs of downstream repos to clone, the first of which is the repo being tested
:param str relTargetSuiteDir: directory of the downstream suite to test relative to the top level
directory of the downstream repo being tested
:param list mxCommands: argument lists for the mx commands run in downstream suite being tested
:param str branch: name of branch to look for in downstream repo(s)
"""
assert len(repoUrls) > 0
workDir = join(suite.get_output_root(), 'testdownstream')
# A mirror of each suites in the same repo as `suite` is created with symlinks
rel_mirror = os.path.relpath(suite.dir,
mx.SuiteModel.siblings_dir(suite.dir))
in_subdir = os.sep in rel_mirror
suites_in_repo = [suite]
if in_subdir:
base = os.path.dirname(suite.dir)
for e in os.listdir(base):
candidate = join(base, e)
if candidate != suite.dir:
mxDir = mx._is_suite_dir(candidate)
if mxDir:
matches = [s for s in mx.suites() if s.dir == candidate]
if len(matches) == 0:
suites_in_repo.append(
mx.SourceSuite(mxDir, primary=False, load=False))
else:
suites_in_repo.append(matches[0])
for suite_in_repo in suites_in_repo:
rel_mirror = os.path.relpath(
suite_in_repo.dir, mx.SuiteModel.siblings_dir(suite_in_repo.dir))
mirror = join(workDir, rel_mirror)
if exists(mirror):
shutil.rmtree(mirror)
mx.ensure_dir_exists(mirror)
for f in os.listdir(suite_in_repo.dir):
subDir = join(suite_in_repo.dir, f)
if subDir == suite_in_repo.get_output_root():
continue
src = join(suite_in_repo.dir, f)
dst = join(mirror, f)
mx.logv('[Creating symlink from {} to {}]'.format(dst, src))
relsrc = os.path.relpath(src, os.path.dirname(dst))
os.symlink(relsrc, dst)
targetDir = None
for repoUrl in repoUrls:
# Deduce a target name from the target URL
url = urlparse(repoUrl)
targetName = url.path
if targetName.rfind('/') != -1:
targetName = targetName[targetName.rfind('/') + 1:]
if targetName.endswith('.git'):
targetName = targetName[0:-len('.git')]
repoWorkDir = join(workDir, targetName)
git = mx.GitConfig()
if exists(repoWorkDir):
git.pull(repoWorkDir)
else:
git.clone(repoUrl, repoWorkDir)
# See if there's a matching (non-master) branch and use it if there is
if not branch:
branch = git.git_command(
suite.dir, ['rev-parse', '--abbrev-ref', 'HEAD']).strip()
if branch != 'master':
git.git_command(repoWorkDir, ['checkout', branch],
abortOnError=False)
if not targetDir:
targetDir = repoWorkDir
assert not isabs(relTargetSuiteDir)
targetSuiteDir = join(targetDir, relTargetSuiteDir)
assert targetSuiteDir.startswith(targetDir)
mxpy = None if suite != mx._mx_suite else join(mirror, 'mx.py')
for command in mxCommands:
mx.logv('[running "mx ' + ' '.join(command) + '" in ' +
targetSuiteDir + ']')
mx.run_mx(command, targetSuiteDir, mxpy=mxpy)
def gc_dists(args):
""" Garbage collect mx distributions."""
parser = argparse.ArgumentParser(prog='mx gc-dists', description='''Garbage collect layout distributions.
By default, it collects all found layout distributions that are *not* part of the current configuration (see `--keep-current`).
This command respects mx level suite filtering (e.g., `mx --suite my-suite gc-dists`).
''', epilog='''If the environment variable `MX_GC_AFTER_BUILD` is set, %(prog)s will be executed after `mx build`
using the content of the environment variable as parameters.''')
# mutually exclusive groups do not support title and description - wrapping in another group as a workaround
action_group_desc = parser.add_argument_group('actions', 'What to do with the result. One of the following arguments is required.')
action_group = action_group_desc.add_mutually_exclusive_group(required=True)
action_group.add_argument('-f', '--force', action='store_true', help='remove layout distributions without further questions')
action_group.add_argument('-n', '--dry-run', action='store_true', help='show what would be removed without actually doing anything')
action_group.add_argument('-i', '--interactive', action='store_true', help='ask for every layout distributions whether it should be removed')
keep_current_group_desc = parser.add_argument_group('current configuration handling', description='How to deal with the current configuration, i.e., what `mx build` would rebuild.')
keep_current_group = keep_current_group_desc.add_mutually_exclusive_group()
keep_current_group.add_argument('--keep-current', action='store_true', default=True, help='keep layout distributions of the current configuration (default)')
keep_current_group.add_argument('--no-keep-current', action='store_false', dest='keep_current', help='remove layout distributions of the current configuration')
filter_group = parser.add_argument_group('result filters', description='Filter can be combined.')
filter_group.add_argument('--reverse', action='store_true', help='reverse the result')
filter_group.add_argument('--older-than', action=TimeAction, help='only show results older than the specified point in time (format: {})'.format(TimeAction.fmt.replace('%', '%%')))
try:
parsed_args = parser.parse_args(args)
except ValueError as ve:
parser.error(str(ve))
suites = mx.suites(opt_limit_to_suite=True, includeBinary=False, include_mx=False)
c = []
for s in suites:
c += _gc_layout_dists(s, parsed_args)
if not c:
mx.log("Nothing to do!")
return
if parsed_args.older_than:
c = [x for x in c if x[1] < parsed_args.older_than]
# sort by mod date
c = sorted(c, key=lambda x: x[1], reverse=parsed_args.reverse)
# calculate max sizes
max_path = 0
max_mod_time = 0
max_size = 0
for path, mod_time, size in c:
max_path = max(len(path), max_path)
max_mod_time = max(len(_format_datetime(mod_time)), max_mod_time)
max_size = max(len(_format_bytes(size)), max_size)
msg_fmt = '{0:<' + str(max_path) + '} modified {1:<' + str(max_mod_time + len(' ago')) +'} {2:<' + str(max_size) + '}'
size_sum = 0
for path, mod_time, size in c:
if parsed_args.dry_run:
mx.log(msg_fmt.format(path, _format_datetime(mod_time) + ' ago', _format_bytes(size)))
size_sum += size
else:
msg = '{0} (modified {1} ago, size {2})'.format(path, _format_datetime(mod_time), _format_bytes(size))
if parsed_args.force or parsed_args.interactive and mx.ask_yes_no('Delete ' + msg):
mx.log('rm ' + path)
mx.rmtree(path)
size_sum += size
if parsed_args.dry_run:
mx.log('Would free ' + _format_bytes(size_sum))
else:
mx.log('Freed ' + _format_bytes(size_sum))
def fsckprojects(args):
"""find directories corresponding to deleted Java projects and delete them"""
for suite in mx.suites(True, includeBinary=False):
projectDirs = [p.dir for p in suite.projects]
distIdeDirs = [
d.get_ide_project_dir() for d in suite.dists
if d.isJARDistribution() and d.get_ide_project_dir() is not None
]
for dirpath, dirnames, files in os.walk(suite.dir):
if dirpath == suite.dir:
# no point in traversing vc metadata dir, lib, .workspace
# if there are nested source suites must not scan those now, as they are not in projectDirs (but contain .project files)
omitted = [suite.mxDir, 'lib', '.workspace', 'mx.imports']
if suite.vc:
omitted.append(suite.vc.metadir())
dirnames[:] = [d for d in dirnames if d not in omitted]
elif dirpath == suite.get_output_root():
# don't want to traverse output dir
dirnames[:] = []
elif dirpath == suite.mxDir:
# don't want to traverse mx.name as it contains a .project
dirnames[:] = []
elif dirpath in projectDirs:
# don't traverse subdirs of an existing project in this suite
dirnames[:] = []
elif dirpath in distIdeDirs:
# don't traverse subdirs of an existing distribution in this suite
dirnames[:] = []
else:
maybe_project = basename(dirpath)
if not mx._removedDeps.get(maybe_project):
projectConfigFiles = frozenset([
'.classpath', '.project', 'nbproject',
maybe_project + '.iml'
])
indicators = projectConfigFiles.intersection(files)
if len(indicators) != 0:
indicators = [
os.path.relpath(join(dirpath, i), suite.vc_dir)
for i in indicators
]
indicatorsInVC = suite.vc.locate(
suite.vc_dir, indicators)
# Only proceed if there are indicator files that are not under VC
if len(indicators) > len(indicatorsInVC):
if mx.ask_yes_no(
dirpath +
' looks like a removed project -- delete it',
'n'):
shutil.rmtree(dirpath)
mx.log('Deleted ' + dirpath)
ideaProjectDirectory = join(suite.dir, '.idea')
librariesDirectory = join(ideaProjectDirectory, 'libraries')
if librariesDirectory and exists(librariesDirectory):
neededLibraries = set()
unique_library_file_names = set()
for p in suite.projects_recursive(
) + mx._mx_suite.projects_recursive():
if not p.isJavaProject():
continue
def processDep(dep, edge):
if dep is p:
return
if dep.isLibrary() or dep.isJARDistribution(
) or dep.isJdkLibrary() or dep.isMavenProject(
) or dep.isClasspathDependency():
neededLibraries.add(dep)
p.walk_deps(visit=processDep, ignoredEdges=[mx.DEP_EXCLUDED])
neededLibraryFiles = frozenset([
mx_ide_intellij._intellij_library_file_name(
l.name, unique_library_file_names) for l in neededLibraries
])
existingLibraryFiles = frozenset(os.listdir(librariesDirectory))
for library_file in existingLibraryFiles - neededLibraryFiles:
file_path = join(librariesDirectory, library_file)
relative_file_path = os.path.relpath(file_path, os.curdir)
if mx.ask_yes_no(
relative_file_path +
' looks like a removed library -- delete it', 'n'):
os.remove(file_path)
mx.log('Deleted ' + relative_file_path)
def testdownstream(suite, repoUrls, relTargetSuiteDir, mxCommands, branch=None):
"""
Tests a downstream repo against the current working directory state of `suite`.
:param mx.Suite suite: the suite to test against the downstream repo
:param list repoUrls: URLs of downstream repos to clone, the first of which is the repo being tested
:param str relTargetSuiteDir: directory of the downstream suite to test relative to the top level
directory of the downstream repo being tested
:param list mxCommands: argument lists for the mx commands run in downstream suite being tested
:param str branch: name of branch to look for in downstream repo(s)
"""
assert len(repoUrls) > 0
workDir = join(suite.get_output_root(), 'testdownstream')
# A mirror of each suites in the same repo as `suite` is created via copying
rel_mirror = os.path.relpath(suite.dir, mx.SuiteModel.siblings_dir(suite.dir))
in_subdir = os.sep in rel_mirror
suites_in_repo = [suite]
if in_subdir:
base = os.path.dirname(suite.dir)
for e in os.listdir(base):
candidate = join(base, e)
if candidate != suite.dir:
mxDir = mx._is_suite_dir(candidate)
if mxDir:
matches = [s for s in mx.suites() if s.dir == candidate]
if len(matches) == 0:
suites_in_repo.append(mx.SourceSuite(mxDir, primary=False, load=False))
else:
suites_in_repo.append(matches[0])
for suite_in_repo in suites_in_repo:
if suite_in_repo.vc_dir and suite_in_repo.dir != suite_in_repo.vc_dir:
mirror = join(workDir, basename(suite_in_repo.vc_dir), suite_in_repo.name)
else:
mirror = join(workDir, suite_in_repo.name)
if exists(mirror):
shutil.rmtree(mirror)
output_root = suite_in_repo.get_output_root()
def ignore_output_root(d, names):
mx.log('Copying ' + d)
if d == os.path.dirname(output_root):
mx.log('Omitting ' + output_root)
return [os.path.basename(output_root)]
return []
shutil.copytree(suite_in_repo.dir, mirror, ignore=ignore_output_root)
targetDir = None
for repoUrl in repoUrls:
# Deduce a target name from the target URL
url = urlparse(repoUrl)
targetName = url.path
if targetName.rfind('/') != -1:
targetName = targetName[targetName.rfind('/') + 1:]
if targetName.endswith('.git'):
targetName = targetName[0:-len('.git')]
repoWorkDir = join(workDir, targetName)
git = mx.GitConfig()
if exists(repoWorkDir):
git.pull(repoWorkDir)
else:
git.clone(repoUrl, repoWorkDir)
# See if there's a matching (non-master) branch and use it if there is
if not branch:
branch = git.git_command(suite.dir, ['rev-parse', '--abbrev-ref', 'HEAD']).strip()
if branch != 'master':
git.git_command(repoWorkDir, ['checkout', branch], abortOnError=False)
if not targetDir:
targetDir = repoWorkDir
assert not isabs(relTargetSuiteDir)
targetSuiteDir = join(targetDir, relTargetSuiteDir)
assert targetSuiteDir.startswith(targetDir)
mxpy = None if suite != mx._mx_suite else join(mirror, 'mx.py')
for command in mxCommands:
mx.logv('[running "mx ' + ' '.join(command) + '" in ' + targetSuiteDir + ']')
mx.run_mx(command, targetSuiteDir, mxpy=mxpy)
