def symlink_or_copy(target_path, dest_path, debug_gr_8964=False): # Follow symbolic links in case they go outside my suite directories. real_target_path = os.path.realpath(target_path) if debug_gr_8964: mx.log(' [mx_substratevm.symlink_or_copy:') mx.log(' suite.dir:' + suite.dir) mx.log(' target_path: ' + target_path) mx.log(' real_target_path: ' + real_target_path) mx.log(' dest_path: ' + dest_path) if any( real_target_path.startswith(s.dir) for s in mx.suites(includeBinary=False)): # Symbolic link to files in my suites. sym_target = os.path.relpath(real_target_path, dirname(dest_path)) if debug_gr_8964: mx.log(' symlink target: ' + sym_target) try: os.symlink(sym_target, dest_path) except AttributeError: # no `symlink` on Windows copy2(real_target_path, dest_path) else: # Else copy the file to so it can not change out from under me. if debug_gr_8964: mx.log(' copy2: ') copy2(real_target_path, dest_path) if debug_gr_8964: mx.log(' ]')
def symlink_or_copy(target_path, dest_path, debug_gr_8964=False): # Follow symbolic links in case they go outside my suite directories. real_target_path = os.path.realpath(target_path) if debug_gr_8964: mx.log(' [mx_substratevm.symlink_or_copy:') mx.log(' suite.dir:' + suite.dir) mx.log(' target_path: ' + target_path) mx.log(' real_target_path: ' + real_target_path) mx.log(' dest_path: ' + dest_path) if any(real_target_path.startswith(s.dir) for s in mx.suites(includeBinary=False)): # Symbolic link to files in my suites. sym_target = os.path.relpath(real_target_path, dirname(dest_path)) if debug_gr_8964: mx.log(' symlink target: ' + sym_target) try: os.symlink(sym_target, dest_path) except AttributeError: # no `symlink` on Windows copy2(real_target_path, dest_path) else: # Else copy the file to so it can not change out from under me. if debug_gr_8964: mx.log(' copy2: ') copy2(real_target_path, dest_path) if debug_gr_8964: mx.log(' ]')
def runLLVM(args=None, out=None, err=None, timeout=None, nonZeroIsFatal=True, get_classpath_options=getClasspathOptions): """run lli via the legacy mx java launcher (instead of via the current GraalVM)""" vmArgs, sulongArgs = truffle_extract_VM_args(args) dists = [] if "tools" in (s.name for s in mx.suites()): dists.append('CHROMEINSPECTOR') return mx.run_java(getCommonOptions(False) + vmArgs + get_classpath_options(dists) + ["com.oracle.truffle.llvm.launcher.LLVMLauncher"] + sulongArgs, timeout=timeout, nonZeroIsFatal=nonZeroIsFatal, out=out, err=err)
def pylint(args): rcfile = join(dirname(mx.__file__), '.pylintrc') pythonpath = dirname(mx.__file__) for suite in mx.suites(True): pythonpath = os.pathsep.join([pythonpath, suite.mxDir]) def findfiles_by_vc(pyfiles): for suite in mx.suites(True, includeBinary=False): if not suite.primary: continue files = suite.vc.locate(suite.mxDir, ['*.py']) for pyfile in files: pyfile = join(suite.mxDir, pyfile) if exists(pyfile): pyfiles.append(pyfile) pyfiles = [] findfiles_by_vc(pyfiles) env = os.environ.copy() env['PYTHONPATH'] = pythonpath for pyfile in pyfiles: mx.log('Running pylint on ' + pyfile + '...') mx.run(['pylint', '--reports=n', '--rcfile=' + rcfile, pyfile], env=env) return 0
def ideclean(args): """remove all IDE project configurations""" def rm(path): if exists(path): os.remove(path) for s in mx.suites() + [mx._mx_suite]: rm(join(s.get_mx_output_dir(), 'eclipse-config.zip')) rm(join(s.get_mx_output_dir(), 'netbeans-config.zip')) shutil.rmtree(join(s.dir, '.idea'), ignore_errors=True) for p in mx.projects() + mx._mx_suite.projects: if not p.isJavaProject(): continue shutil.rmtree(join(p.dir, '.settings'), ignore_errors=True) shutil.rmtree(join(p.dir, '.externalToolBuilders'), ignore_errors=True) shutil.rmtree(join(p.dir, 'nbproject'), ignore_errors=True) rm(join(p.dir, '.classpath')) rm(join(p.dir, '.checkstyle')) rm(join(p.dir, '.project')) rm(join(p.dir, '.factorypath')) rm(join(p.dir, p.name + '.iml')) rm(join(p.dir, 'build.xml')) rm(join(p.dir, 'eclipse-build.xml')) try: rm(join(p.dir, p.name + '.jar')) except: mx.log_error("Error removing {0}".format(p.name + '.jar')) for d in mx._dists.values(): if not d.isJARDistribution(): continue if d.get_ide_project_dir(): shutil.rmtree(d.get_ide_project_dir(), ignore_errors=True)
def netbeansinit(args, refreshOnly=False, buildProcessorJars=True, doFsckProjects=True): """(re)generate NetBeans project configurations""" jdks = set() for suite in mx.suites(True) + [mx._mx_suite]: _netbeansinit_suite(args, jdks, suite, refreshOnly, buildProcessorJars) if doFsckProjects and not refreshOnly: mx_ideconfig.fsckprojects([]) mx.log('If using NetBeans:') # http://stackoverflow.com/questions/24720665/cant-resolve-jdk-internal-package mx.log( ' 1. Edit etc/netbeans.conf in your NetBeans installation and modify netbeans_default_options variable to include "-J-DCachingArchiveProvider.disableCtSym=true"' ) mx.log( ' 2. Ensure that the following platform(s) are defined (Tools -> Java Platforms):' ) for jdk in jdks: mx.log(' JDK_' + str(jdk.version)) mx.log( ' 3. Open/create a Project Group for the directory containing the projects (File -> Project Group -> New Group... -> Folder of Projects)' )
def runLLVM(args=None, out=None, get_classpath_options=getClasspathOptions): """uses Sulong to execute a LLVM IR file""" vmArgs, sulongArgs = truffle_extract_VM_args(args) dists = [] if "tools" in (s.name for s in mx.suites()): dists.append('CHROMEINSPECTOR') return mx.run_java(getCommonOptions(False) + vmArgs + get_classpath_options(dists) + ["com.oracle.truffle.llvm.launcher.LLVMLauncher"] + sulongArgs, out=out)
def findfiles_by_vc(pyfiles): for suite in mx.suites(True, includeBinary=False): if not suite.primary: continue files = suite.vc.locate(suite.mxDir, ['*.py']) for pyfile in files: pyfile = join(suite.mxDir, pyfile) if exists(pyfile): pyfiles.append(pyfile)
def _get_src_dir(projectname): for suite in mx.suites(): for p in suite.projects: if p.name == projectname: if len(p.source_dirs()) > 0: return p.source_dirs()[0] else: return p.dir mx.abort("Could not find src dir for project %s" % projectname)
def netbeansinit(args, refreshOnly=False, buildProcessorJars=True, doFsckProjects=True): """(re)generate NetBeans project configurations""" for suite in mx.suites(True) + [mx._mx_suite]: _netbeansinit_suite(args, suite, refreshOnly, buildProcessorJars) if doFsckProjects and not refreshOnly: mx_ideconfig.fsckprojects([])
def native_image_option_properties(option_kind, option_flag, native_image_root): target_dir = join(native_image_root, option_kind, option_flag) target_path = remove_existing_symlink(join(target_dir, 'native-image.properties')) option_properties = None for suite in mx.suites(): candidate = join(suite.mxDir, option_kind + '-' + option_flag + '.properties') if exists(candidate): option_properties = candidate if option_properties: mx.logv('Add symlink to ' + str(option_properties)) mkpath(target_dir) symlink_or_copy(option_properties, target_path)
def symlink_or_copy(target_path, dest_path): # Follow symbolic links in case they go outside my suite directories. real_target_path = os.path.realpath(target_path) if any(real_target_path.startswith(s.dir) for s in mx.suites(includeBinary=False)): # Symbolic link to files in my suites. sym_target = os.path.relpath(real_target_path, dirname(dest_path)) try: os.symlink(sym_target, dest_path) except AttributeError: # no `symlink` on Windows copy2(real_target_path, dest_path) else: # Else copy the file to so it can not change out from under me. copy2(real_target_path, dest_path)
def runLLVMMul(args=None, out=None, err=None, timeout=None, nonZeroIsFatal=True, get_classpath_options=getClasspathOptions): """run multi-context java launcher""" vmArgs, sulongArgs = truffle_extract_VM_args(args) dists = [] if "tools" in (s.name for s in mx.suites()): dists.append('CHROMEINSPECTOR') return mx.run_java( getCommonOptions(False) + vmArgs + get_classpath_options(dists) + ["com.oracle.truffle.llvm.launcher.LLVMMultiContextLauncher"] + sulongArgs, timeout=timeout, nonZeroIsFatal=nonZeroIsFatal, out=out, err=err)
def intellijinit(args, refreshOnly=False, doFsckProjects=True, mx_python_modules=True, java_modules=True, generate_external_projects=True, native_projects=False): # In a multiple suite context, the .idea directory in each suite # has to be complete and contain information that is repeated # in dependent suites. declared_modules = set() referenced_modules = set() sdks = intellij_read_sdks() for suite in mx.suites(True) + ([mx._mx_suite] if mx_python_modules else []): _intellij_suite(args, suite, declared_modules, referenced_modules, sdks, refreshOnly, mx_python_modules, generate_external_projects, java_modules and not suite.isBinarySuite(), suite != mx.primary_suite(), generate_native_projects=native_projects) if len(referenced_modules - declared_modules) != 0: mx.abort('Some referenced modules are missing from modules.xml: {}'.format(referenced_modules - declared_modules)) if mx_python_modules: # mx module moduleXml = mx.XMLDoc() moduleXml.open('module', attributes={'type': 'PYTHON_MODULE', 'version': '4'}) moduleXml.open('component', attributes={'name': 'NewModuleRootManager', 'inherit-compiler-output': 'true'}) moduleXml.element('exclude-output') moduleXml.open('content', attributes={'url': 'file://$MODULE_DIR$'}) moduleXml.element('sourceFolder', attributes={'url': 'file://$MODULE_DIR$', 'isTestSource': 'false'}) for d in set((p.subDir for p in mx._mx_suite.projects if p.subDir)): moduleXml.element('excludeFolder', attributes={'url': 'file://$MODULE_DIR$/' + d}) if dirname(mx._mx_suite.get_output_root()) == mx._mx_suite.dir: moduleXml.element('excludeFolder', attributes={'url': 'file://$MODULE_DIR$/' + basename(mx._mx_suite.get_output_root())}) moduleXml.close('content') moduleXml.element('orderEntry', attributes={'type': 'jdk', 'jdkType': intellij_python_sdk_type, 'jdkName': intellij_get_python_sdk_name(sdks)}) moduleXml.element('orderEntry', attributes={'type': 'sourceFolder', 'forTests': 'false'}) moduleXml.close('component') moduleXml.close('module') mxModuleFile = join(mx._mx_suite.dir, basename(mx._mx_suite.dir) + '.iml') mx.update_file(mxModuleFile, moduleXml.xml(indent=' ', newl='\n')) if doFsckProjects and not refreshOnly: mx_ideconfig.fsckprojects([])
def testdownstream(suite, repoUrls, relTargetSuiteDir, mxCommands, branch=None): """ Tests a downstream repo against the current working directory state of `suite`. :param mx.Suite suite: the suite to test against the downstream repo :param list repoUrls: URLs of downstream repos to clone, the first of which is the repo being tested :param str relTargetSuiteDir: directory of the downstream suite to test relative to the top level directory of the downstream repo being tested :param list mxCommands: argument lists for the mx commands run in downstream suite being tested :param str branch: name(s) of branch to look for in downstream repo(s) """ assert len(repoUrls) > 0 repoUrls = [mx_urlrewrites.rewriteurl(url) for url in repoUrls] workDir = join(suite.get_output_root(), 'testdownstream') # A mirror of each suites in the same repo as `suite` is created via copying rel_mirror = os.path.relpath(suite.dir, mx.SuiteModel.siblings_dir(suite.dir)) in_subdir = os.sep in rel_mirror suites_in_repo = [suite] if in_subdir: base = os.path.dirname(suite.dir) for e in os.listdir(base): candidate = join(base, e) if candidate != suite.dir: mxDir = mx._is_suite_dir(candidate) if mxDir: matches = [s for s in mx.suites() if s.dir == candidate] if len(matches) == 0: suites_in_repo.append( mx.SourceSuite(mxDir, primary=False, load=False)) else: suites_in_repo.append(matches[0]) if suite.vc: vc_metadir = mx._safe_path(mx.VC.get_vc(suite.vc_dir).metadir()) blacklist = {suite.vc_dir: [join(suite.vc_dir, vc_metadir)]} else: blacklist = {} for suite_in_repo in suites_in_repo: output_root = mx._safe_path(suite_in_repo.get_output_root()) blacklist.setdefault(dirname(output_root), []).append(output_root) def omitted_dirs(d, names): mx.log('Copying ' + d) to_omit = [] for blacklisted_dir in blacklist.get(d, []): mx.log('Omitting ' + blacklisted_dir) to_omit.append(basename(blacklisted_dir)) return to_omit if suite.vc_dir and suite.dir != suite.vc_dir: mirror = join(workDir, basename(suite.vc_dir)) else: mirror = join(workDir, suite.name) if exists(mirror): mx.rmtree(mirror) mx.copytree(suite.vc_dir, mirror, ignore=omitted_dirs, symlinks=True) targetDir = None for repoUrl in repoUrls: # Deduce a target name from the target URL url = _urllib_parse.urlparse(repoUrl) targetName = url.path if targetName.rfind('/') != -1: targetName = targetName[targetName.rfind('/') + 1:] if targetName.endswith('.git'): targetName = targetName[0:-len('.git')] repoWorkDir = join(workDir, targetName) git = mx.GitConfig() if exists(repoWorkDir): git.pull(repoWorkDir) else: git.clone(repoUrl, repoWorkDir) if branch is None: branch = [] elif isinstance(branch, str): branch = [branch] else: assert isinstance(branch, list) # fall back to the branch of the main repo active_branch = git.active_branch(suite.dir, abortOnError=False) if active_branch: branch.append(active_branch) updated = False for branch_name in branch: if git.update_to_branch(repoWorkDir, branch_name, abortOnError=False): updated = True break if not updated: mx.warn('Could not update {} to any of the following branches: {}'. format(repoWorkDir, ', '.join(branch))) if not targetDir: targetDir = repoWorkDir assert not isabs(relTargetSuiteDir) targetSuiteDir = join(targetDir, relTargetSuiteDir) assert targetSuiteDir.startswith(targetDir) mxpy = None if suite != mx._mx_suite else join(mirror, 'mx.py') for command in mxCommands: mx.logv('[running "mx ' + ' '.join(command) + '" in ' + targetSuiteDir + ']') mx.run_mx(command, targetSuiteDir, mxpy=mxpy)
def _intellij_suite(args, s, declared_modules, referenced_modules, sdks, refreshOnly=False, mx_python_modules=False, generate_external_projects=True, java_modules=True, module_files_only=False, generate_native_projects=False): libraries = set() jdk_libraries = set() project_dir = s.dir ideaProjectDirectory = join(project_dir, '.idea') modulesXml = mx.XMLDoc() if not module_files_only and not s.isBinarySuite(): mx.ensure_dir_exists(ideaProjectDirectory) nameFile = join(ideaProjectDirectory, '.name') mx.update_file(nameFile, s.name) modulesXml.open('project', attributes={'version': '4'}) modulesXml.open('component', attributes={'name': 'ProjectModuleManager'}) modulesXml.open('modules') def _intellij_exclude_if_exists(xml, p, name, output=False): root = p.get_output_root() if output else p.dir path = join(root, name) if exists(path): excludeRoot = p.get_output_root() if output else '$MODULE_DIR$' excludePath = join(excludeRoot, name) xml.element('excludeFolder', attributes={'url':'file://' + excludePath}) annotationProcessorProfiles = {} def _complianceToIntellijLanguageLevel(compliance): # they changed the name format starting with JDK_10 if compliance.value >= 10: # Lastest Idea 2018.2 only understands JDK_11 so clamp at that value return 'JDK_' + str(min(compliance.value, 11)) return 'JDK_1_' + str(compliance.value) def _intellij_external_project(externalProjects, sdks, host): if externalProjects: for project_name, project_definition in externalProjects.items(): if not project_definition.get('path', None): mx.abort("external project {} is missing path attribute".format(project_name)) if not project_definition.get('type', None): mx.abort("external project {} is missing type attribute".format(project_name)) supported = ['path', 'type', 'source', 'test', 'excluded', 'load_path'] unknown = set(project_definition.keys()) - frozenset(supported) if unknown: mx.abort("There are unsupported {} keys in {} external project".format(unknown, project_name)) path = os.path.realpath(join(host.dir, project_definition["path"])) module_type = project_definition["type"] moduleXml = mx.XMLDoc() moduleXml.open('module', attributes={'type': {'ruby': 'RUBY_MODULE', 'python': 'PYTHON_MODULE', 'web': 'WEB_MODULE'}.get(module_type, 'UKNOWN_MODULE'), 'version': '4'}) moduleXml.open('component', attributes={'name': 'NewModuleRootManager', 'inherit-compiler-output': 'true'}) moduleXml.element('exclude-output') moduleXml.open('content', attributes={'url': 'file://$MODULE_DIR$'}) for name in project_definition.get('source', []): moduleXml.element('sourceFolder', attributes={'url':'file://$MODULE_DIR$/' + name, 'isTestSource': str(False)}) for name in project_definition.get('test', []): moduleXml.element('sourceFolder', attributes={'url':'file://$MODULE_DIR$/' + name, 'isTestSource': str(True)}) for name in project_definition.get('excluded', []): _intellij_exclude_if_exists(moduleXml, type('', (object,), {"dir": path})(), name) moduleXml.close('content') if module_type == "ruby": moduleXml.element('orderEntry', attributes={'type': 'jdk', 'jdkType': intellij_ruby_sdk_type, 'jdkName': intellij_get_ruby_sdk_name(sdks)}) elif module_type == "python": moduleXml.element('orderEntry', attributes={'type': 'jdk', 'jdkType': intellij_python_sdk_type, 'jdkName': intellij_get_python_sdk_name(sdks)}) elif module_type == "web": # nothing to do pass else: mx.abort("External project type {} not supported".format(module_type)) moduleXml.element('orderEntry', attributes={'type': 'sourceFolder', 'forTests': 'false'}) moduleXml.close('component') load_paths = project_definition.get('load_path', []) if load_paths: if not module_type == "ruby": mx.abort("load_path is supported only for ruby type external project") moduleXml.open('component', attributes={'name': 'RModuleSettingsStorage'}) load_paths_attributes = {} load_paths_attributes['number'] = str(len(load_paths)) for i, name in enumerate(load_paths): load_paths_attributes["string" + str(i)] = "$MODULE_DIR$/" + name moduleXml.element('LOAD_PATH', load_paths_attributes) moduleXml.close('component') moduleXml.close('module') moduleFile = join(path, project_name + '.iml') mx.update_file(moduleFile, moduleXml.xml(indent=' ', newl='\n')) if not module_files_only: declared_modules.add(project_name) moduleFilePath = "$PROJECT_DIR$/" + os.path.relpath(moduleFile, s.dir) modulesXml.element('module', attributes={'fileurl': 'file://' + moduleFilePath, 'filepath': moduleFilePath}) if generate_external_projects: for p in s.projects_recursive() + mx._mx_suite.projects_recursive(): _intellij_external_project(getattr(p, 'externalProjects', None), sdks, p) max_checkstyle_version = None compilerXml = None if java_modules: if not module_files_only: compilerXml = mx.XMLDoc() compilerXml.open('project', attributes={'version': '4'}) # The IntelliJ parser seems to mishandle empty ADDITIONAL_OPTIONS_OVERRIDE elements # so only emit the section if there will be something in it. additionalOptionsOverrides = False assert not s.isBinarySuite() # create the modules (1 IntelliJ module = 1 mx project/distribution) for p in s.projects_recursive() + mx._mx_suite.projects_recursive(): if not p.isJavaProject(): continue jdk = mx.get_jdk(p.javaCompliance) assert jdk # Value of the $MODULE_DIR$ IntelliJ variable and parent directory of the .iml file. module_dir = mx.ensure_dir_exists(p.dir) processors = p.annotation_processors() if processors: annotationProcessorProfiles.setdefault((p.source_gen_dir_name(),) + tuple(processors), []).append(p) intellijLanguageLevel = _complianceToIntellijLanguageLevel(p.javaCompliance) moduleXml = mx.XMLDoc() moduleXml.open('module', attributes={'type': 'JAVA_MODULE', 'version': '4'}) moduleXml.open('component', attributes={'name': 'NewModuleRootManager', 'LANGUAGE_LEVEL': intellijLanguageLevel, 'inherit-compiler-output': 'false'}) moduleXml.element('output', attributes={'url': 'file://$MODULE_DIR$/' + os.path.relpath(p.output_dir(), module_dir)}) moduleXml.open('content', attributes={'url': 'file://$MODULE_DIR$'}) for src in p.srcDirs: srcDir = mx.ensure_dir_exists(join(p.dir, src)) moduleXml.element('sourceFolder', attributes={'url':'file://$MODULE_DIR$/' + os.path.relpath(srcDir, module_dir), 'isTestSource': str(p.is_test_project())}) for name in ['.externalToolBuilders', '.settings', 'nbproject']: _intellij_exclude_if_exists(moduleXml, p, name) moduleXml.close('content') if processors: moduleXml.open('content', attributes={'url': 'file://' + p.get_output_root()}) genDir = p.source_gen_dir() mx.ensure_dir_exists(genDir) moduleXml.element('sourceFolder', attributes={'url':'file://' + p.source_gen_dir(), 'isTestSource': str(p.is_test_project()), 'generated': 'true'}) for name in [basename(p.output_dir())]: _intellij_exclude_if_exists(moduleXml, p, name, output=True) moduleXml.close('content') moduleXml.element('orderEntry', attributes={'type': 'sourceFolder', 'forTests': 'false'}) proj = p dependencies_project_packages = set() def should_process_dep(dep, edge): if dep.isTARDistribution() or dep.isNativeProject() or dep.isArchivableProject() or dep.isResourceLibrary(): mx.logv("Ignoring dependency from {} to {}".format(proj.name, dep.name)) return False return True def process_dep(dep, edge): if dep is proj: return if dep.isLibrary() or dep.isJARDistribution() or dep.isMavenProject(): libraries.add(dep) moduleXml.element('orderEntry', attributes={'type': 'library', 'name': dep.name, 'level': 'project'}) elif dep.isJavaProject(): dependencies_project_packages.update(dep.defined_java_packages()) referenced_modules.add(dep.name) moduleXml.element('orderEntry', attributes={'type': 'module', 'module-name': dep.name}) elif dep.isJdkLibrary(): jdk_libraries.add(dep) if jdk.javaCompliance < dep.jdkStandardizedSince: moduleXml.element('orderEntry', attributes={'type': 'library', 'name': dep.name, 'level': 'project'}) else: mx.logv("{} skipping {} for {}".format(p, dep, jdk)) #pylint: disable=undefined-loop-variable elif dep.isJreLibrary(): pass elif dep.isClasspathDependency(): moduleXml.element('orderEntry', attributes={'type': 'library', 'name': dep.name, 'level': 'project'}) else: mx.abort("Dependency not supported: {0} ({1})".format(dep, dep.__class__.__name__)) p.walk_deps(preVisit=should_process_dep, visit=process_dep, ignoredEdges=[mx.DEP_EXCLUDED]) moduleXml.element('orderEntry', attributes={'type': 'jdk', 'jdkType': intellij_java_sdk_type, 'jdkName': intellij_get_java_sdk_name(sdks, jdk)}) moduleXml.close('component') if compilerXml and jdk.javaCompliance >= '9': moduleDeps = p.get_concealed_imported_packages(jdk=jdk) if moduleDeps: exports = sorted([(m, pkgs) for m, pkgs in moduleDeps.items() if dependencies_project_packages.isdisjoint(pkgs)]) if exports: args = [] exported_modules = set() for m, pkgs in exports: args += ['--add-exports={}/{}=ALL-UNNAMED'.format(m, pkg) for pkg in pkgs] exported_modules.add(m) roots = set(jdk.get_root_modules()) observable_modules = jdk.get_modules() default_module_graph = mx_javamodules.get_transitive_closure(roots, observable_modules) module_graph = mx_javamodules.get_transitive_closure(roots | exported_modules, observable_modules) extra_modules = module_graph - default_module_graph if extra_modules: args.append('--add-modules=' + ','.join((m.name for m in extra_modules))) if not additionalOptionsOverrides: additionalOptionsOverrides = True compilerXml.open('component', {'name': 'JavacSettings'}) compilerXml.open('option', {'name': 'ADDITIONAL_OPTIONS_OVERRIDE'}) compilerXml.element('module', {'name': p.name, 'options': ' '.join(args)}) # Checkstyle csConfig, checkstyleVersion, checkstyleProj = p.get_checkstyle_config() if csConfig: max_checkstyle_version = max(max_checkstyle_version, mx.VersionSpec(checkstyleVersion)) if max_checkstyle_version else mx.VersionSpec(checkstyleVersion) moduleXml.open('component', attributes={'name': 'CheckStyle-IDEA-Module'}) moduleXml.open('option', attributes={'name': 'configuration'}) moduleXml.open('map') moduleXml.element('entry', attributes={'key': "checkstyle-version", 'value': checkstyleVersion}) moduleXml.element('entry', attributes={'key': "active-configuration", 'value': "PROJECT_RELATIVE:" + join(checkstyleProj.dir, ".checkstyle_checks.xml") + ":" + checkstyleProj.name}) moduleXml.close('map') moduleXml.close('option') moduleXml.close('component') moduleXml.close('module') moduleFile = join(module_dir, p.name + '.iml') mx.update_file(moduleFile, moduleXml.xml(indent=' ', newl='\n').rstrip()) if not module_files_only: declared_modules.add(p.name) moduleFilePath = "$PROJECT_DIR$/" + os.path.relpath(moduleFile, project_dir) modulesXml.element('module', attributes={'fileurl': 'file://' + moduleFilePath, 'filepath': moduleFilePath}) if additionalOptionsOverrides: compilerXml.close('option') compilerXml.close('component') if mx_python_modules: def _python_module(suite): """ Gets a tuple describing the IntelliJ module for the python sources of `suite`. The tuple consists of the module name, module directory and the name of the .iml in the module directory. """ name = basename(suite.mxDir) module_dir = suite.mxDir return name, mx.ensure_dir_exists(module_dir), name + '.iml' def _add_declared_module(suite): if not module_files_only: name, module_dir, iml_file = _python_module(suite) declared_modules.add(name) moduleFilePath = "$PROJECT_DIR$/" + os.path.relpath(join(module_dir, iml_file), project_dir) modulesXml.element('module', attributes={'fileurl': 'file://' + moduleFilePath, 'filepath': moduleFilePath}) # mx.<suite> python module: _, module_dir, iml_file = _python_module(s) moduleXml = mx.XMLDoc() moduleXml.open('module', attributes={'type': 'PYTHON_MODULE', 'version': '4'}) moduleXml.open('component', attributes={'name': 'NewModuleRootManager', 'inherit-compiler-output': 'true'}) moduleXml.element('exclude-output') if s.name == 'mx': # MX itself is special. Python sources are also in the parent folder. moduleXml.open('content', attributes={'url': 'file://$MODULE_DIR$/..'}) moduleXml.element('sourceFolder', attributes={'url': 'file://$MODULE_DIR$/..', 'isTestSource': 'false'}) else: moduleXml.open('content', attributes={'url': 'file://$MODULE_DIR$'}) moduleXml.element('sourceFolder', attributes={'url': 'file://$MODULE_DIR$/' + os.path.relpath(s.mxDir, module_dir), 'isTestSource': 'false'}) for d in os.listdir(s.mxDir): directory = join(s.mxDir, d) if isdir(directory) and mx.dir_contains_files_recursively(directory, r".*\.java"): moduleXml.element('excludeFolder', attributes={'url': 'file://$MODULE_DIR$/' + os.path.relpath(directory, module_dir)}) moduleXml.close('content') moduleXml.element('orderEntry', attributes={'type': 'jdk', 'jdkType': intellij_python_sdk_type, 'jdkName': intellij_get_python_sdk_name(sdks)}) moduleXml.element('orderEntry', attributes={'type': 'sourceFolder', 'forTests': 'false'}) processed_suites = {s.name} def _mx_projects_suite(visited_suite, suite_import): if suite_import.name in processed_suites: return processed_suites.add(suite_import.name) dep_suite = mx.suite(suite_import.name) dep_module_name, _, _ = _python_module(dep_suite) moduleXml.element('orderEntry', attributes={'type': 'module', 'module-name': dep_module_name}) _add_declared_module(dep_suite) dep_suite.visit_imports(_mx_projects_suite) s.visit_imports(_mx_projects_suite) if s.name != 'mx': moduleXml.element('orderEntry', attributes={'type': 'module', 'module-name': 'mx.mx'}) moduleXml.close('component') moduleXml.close('module') moduleFile = join(module_dir, iml_file) mx.update_file(moduleFile, moduleXml.xml(indent=' ', newl='\n')) _add_declared_module(s) _add_declared_module(mx._mx_suite) if generate_native_projects: _intellij_native_projects(s, module_files_only, declared_modules, modulesXml) if generate_external_projects: _intellij_external_project(s.suiteDict.get('externalProjects', None), sdks, s) if not module_files_only: modulesXml.close('modules') modulesXml.close('component') modulesXml.close('project') moduleXmlFile = join(ideaProjectDirectory, 'modules.xml') mx.update_file(moduleXmlFile, modulesXml.xml(indent=' ', newl='\n')) if java_modules and not module_files_only: unique_library_file_names = set() librariesDirectory = mx.ensure_dir_exists(join(ideaProjectDirectory, 'libraries')) mx.ensure_dir_exists(librariesDirectory) def make_library(name, path, source_path, suite_dir): libraryXml = mx.XMLDoc() libraryXml.open('component', attributes={'name': 'libraryTable'}) libraryXml.open('library', attributes={'name': name}) libraryXml.open('CLASSES') pathX = mx.relpath_or_absolute(path, suite_dir, prefix='$PROJECT_DIR$') libraryXml.element('root', attributes={'url': 'jar://' + pathX + '!/'}) libraryXml.close('CLASSES') libraryXml.element('JAVADOC') if sourcePath: libraryXml.open('SOURCES') if os.path.isdir(sourcePath): sourcePathX = mx.relpath_or_absolute(sourcePath, suite_dir, prefix='$PROJECT_DIR$') libraryXml.element('root', attributes={'url': 'file://' + sourcePathX}) else: source_pathX = mx.relpath_or_absolute(source_path, suite_dir, prefix='$PROJECT_DIR$') libraryXml.element('root', attributes={'url': 'jar://' + source_pathX + '!/'}) libraryXml.close('SOURCES') else: libraryXml.element('SOURCES') libraryXml.close('library') libraryXml.close('component') libraryFile = join(librariesDirectory, _intellij_library_file_name(name, unique_library_file_names)) return mx.update_file(libraryFile, libraryXml.xml(indent=' ', newl='\n')) # Setup the libraries that were used above for library in libraries: sourcePath = None if library.isLibrary(): path = library.get_path(True) if library.sourcePath: sourcePath = library.get_source_path(True) elif library.isMavenProject(): path = library.get_path(True) sourcePath = library.get_source_path(True) elif library.isJARDistribution(): path = library.path if library.sourcesPath: sourcePath = library.sourcesPath elif library.isClasspathDependency(): path = library.classpath_repr() else: mx.abort('Dependency not supported: {} ({})'.format(library.name, library.__class__.__name__)) make_library(library.name, path, sourcePath, s.dir) jdk = mx.get_jdk() updated = False for library in jdk_libraries: if library.classpath_repr(jdk) is not None: if make_library(library.name, library.classpath_repr(jdk), library.get_source_path(jdk), s.dir): updated = True if jdk_libraries and updated: mx.log("Setting up JDK libraries using {0}".format(jdk)) # Set annotation processor profiles up, and link them to modules in compiler.xml compilerXml.open('component', attributes={'name': 'CompilerConfiguration'}) compilerXml.element('option', attributes={'name': "DEFAULT_COMPILER", 'value': 'Javac'}) # using the --release option with javac interferes with using --add-modules which is required for some projects compilerXml.element('option', attributes={'name': "USE_RELEASE_OPTION", 'value': 'false'}) compilerXml.element('resourceExtensions') compilerXml.open('wildcardResourcePatterns') compilerXml.element('entry', attributes={'name': '!?*.java'}) compilerXml.close('wildcardResourcePatterns') if annotationProcessorProfiles: compilerXml.open('annotationProcessing') for t, modules in sorted(annotationProcessorProfiles.items()): source_gen_dir = t[0] processors = t[1:] compilerXml.open('profile', attributes={'default': 'false', 'name': '-'.join([ap.name for ap in processors]) + "-" + source_gen_dir, 'enabled': 'true'}) compilerXml.element('sourceOutputDir', attributes={'name': join(os.pardir, source_gen_dir)}) compilerXml.element('sourceTestOutputDir', attributes={'name': join(os.pardir, source_gen_dir)}) compilerXml.open('processorPath', attributes={'useClasspath': 'false'}) # IntelliJ supports both directories and jars on the annotation processor path whereas # Eclipse only supports jars. for apDep in processors: def processApDep(dep, edge): if dep.isLibrary() or dep.isJARDistribution(): compilerXml.element('entry', attributes={'name': mx.relpath_or_absolute(dep.path, s.dir, prefix='$PROJECT_DIR$')}) elif dep.isProject(): compilerXml.element('entry', attributes={'name': mx.relpath_or_absolute(dep.output_dir(), s.dir, prefix='$PROJECT_DIR$')}) apDep.walk_deps(visit=processApDep) compilerXml.close('processorPath') for module in modules: compilerXml.element('module', attributes={'name': module.name}) compilerXml.close('profile') compilerXml.close('annotationProcessing') compilerXml.close('component') if compilerXml: compilerXml.close('project') compilerFile = join(ideaProjectDirectory, 'compiler.xml') mx.update_file(compilerFile, compilerXml.xml(indent=' ', newl='\n')) if not module_files_only: # Write misc.xml for global JDK config miscXml = mx.XMLDoc() miscXml.open('project', attributes={'version' : '4'}) if java_modules: mainJdk = mx.get_jdk() miscXml.open('component', attributes={'name' : 'ProjectRootManager', 'version': '2', 'languageLevel': _complianceToIntellijLanguageLevel(mainJdk.javaCompliance), 'project-jdk-name': intellij_get_java_sdk_name(sdks, mainJdk), 'project-jdk-type': intellij_java_sdk_type}) miscXml.element('output', attributes={'url' : 'file://$PROJECT_DIR$/' + os.path.relpath(s.get_output_root(), s.dir)}) miscXml.close('component') else: miscXml.element('component', attributes={'name' : 'ProjectRootManager', 'version': '2', 'project-jdk-name': intellij_get_python_sdk_name(sdks), 'project-jdk-type': intellij_python_sdk_type}) miscXml.close('project') miscFile = join(ideaProjectDirectory, 'misc.xml') mx.update_file(miscFile, miscXml.xml(indent=' ', newl='\n')) # Generate a default configuration for debugging Graal runConfig = mx.XMLDoc() runConfig.open('component', attributes={'name' : 'ProjectRunConfigurationManager'}) runConfig.open('configuration', attributes={'default' :'false', 'name' : 'GraalDebug', 'type' : 'Remote', 'factoryName': 'Remote'}) runConfig.element('option', attributes={'name' : 'USE_SOCKET_TRANSPORT', 'value' : 'true'}) runConfig.element('option', attributes={'name' : 'SERVER_MODE', 'value' : 'false'}) runConfig.element('option', attributes={'name' : 'SHMEM_ADDRESS', 'value' : 'javadebug'}) runConfig.element('option', attributes={'name' : 'HOST', 'value' : 'localhost'}) runConfig.element('option', attributes={'name' : 'PORT', 'value' : '8000'}) runConfig.open('RunnerSettings', attributes={'RunnerId' : 'Debug'}) runConfig.element('option', attributes={'name' : 'DEBUG_PORT', 'value' : '8000'}) runConfig.element('option', attributes={'name' : 'LOCAL', 'value' : 'false'}) runConfig.close('RunnerSettings') runConfig.element('method') runConfig.close('configuration') runConfig.close('component') runConfigFile = join(ideaProjectDirectory, 'runConfigurations', 'GraalDebug.xml') mx.ensure_dir_exists(join(ideaProjectDirectory, 'runConfigurations')) mx.update_file(runConfigFile, runConfig.xml(indent=' ', newl='\n')) if java_modules: # Eclipse formatter config corePrefsSources = s.eclipse_settings_sources().get('org.eclipse.jdt.core.prefs') uiPrefsSources = s.eclipse_settings_sources().get('org.eclipse.jdt.ui.prefs') if corePrefsSources: miscXml = mx.XMLDoc() miscXml.open('project', attributes={'version' : '4'}) out = StringIO() print('# GENERATED -- DO NOT EDIT', file=out) for source in corePrefsSources: print('# Source:', source, file=out) with open(source) as fileName: for line in fileName: if line.startswith('org.eclipse.jdt.core.formatter.'): print(line.strip(), file=out) formatterConfigFile = join(ideaProjectDirectory, 'EclipseCodeFormatter.prefs') mx.update_file(formatterConfigFile, out.getvalue()) importConfigFile = None if uiPrefsSources: out = StringIO() print('# GENERATED -- DO NOT EDIT', file=out) for source in uiPrefsSources: print('# Source:', source, file=out) with open(source) as fileName: for line in fileName: if line.startswith('org.eclipse.jdt.ui.importorder') \ or line.startswith('org.eclipse.jdt.ui.ondemandthreshold') \ or line.startswith('org.eclipse.jdt.ui.staticondemandthreshold'): print(line.strip(), file=out) importConfigFile = join(ideaProjectDirectory, 'EclipseImports.prefs') mx.update_file(importConfigFile, out.getvalue()) miscXml.open('component', attributes={'name' : 'EclipseCodeFormatterProjectSettings'}) miscXml.open('option', attributes={'name' : 'projectSpecificProfile'}) miscXml.open('ProjectSpecificProfile') miscXml.element('option', attributes={'name' : 'formatter', 'value' : 'ECLIPSE'}) custom_eclipse_exe = mx.get_env('ECLIPSE_EXE') if custom_eclipse_exe: custom_eclipse = dirname(custom_eclipse_exe) if mx.is_darwin(): custom_eclipse = join(dirname(custom_eclipse), 'Eclipse') if not exists(custom_eclipse_exe): mx.abort('Custom eclipse "{}" does not exist'.format(custom_eclipse_exe)) miscXml.element('option', attributes={'name' : 'eclipseVersion', 'value' : 'CUSTOM'}) miscXml.element('option', attributes={'name' : 'pathToEclipse', 'value' : custom_eclipse}) miscXml.element('option', attributes={'name' : 'pathToConfigFileJava', 'value' : '$PROJECT_DIR$/.idea/' + basename(formatterConfigFile)}) if importConfigFile: miscXml.element('option', attributes={'name' : 'importOrderConfigFilePath', 'value' : '$PROJECT_DIR$/.idea/' + basename(importConfigFile)}) miscXml.element('option', attributes={'name' : 'importOrderFromFile', 'value' : 'true'}) miscXml.close('ProjectSpecificProfile') miscXml.close('option') miscXml.close('component') miscXml.close('project') miscFile = join(ideaProjectDirectory, 'eclipseCodeFormatter.xml') mx.update_file(miscFile, miscXml.xml(indent=' ', newl='\n')) if java_modules: # Write codestyle settings mx.ensure_dir_exists(join(ideaProjectDirectory, 'codeStyles')) codeStyleConfigXml = mx.XMLDoc() codeStyleConfigXml.open('component', attributes={'name': 'ProjectCodeStyleConfiguration'}) codeStyleConfigXml.open('state') codeStyleConfigXml.element('option', attributes={'name': 'USE_PER_PROJECT_SETTINGS', 'value': 'true'}) codeStyleConfigXml.close('state') codeStyleConfigXml.close('component') codeStyleConfigFile = join(ideaProjectDirectory, 'codeStyles', 'codeStyleConfig.xml') mx.update_file(codeStyleConfigFile, codeStyleConfigXml.xml(indent=' ', newl='\n')) codeStyleProjectXml = mx.XMLDoc() codeStyleProjectXml.open('component', attributes={'name': 'ProjectCodeStyleConfiguration'}) codeStyleProjectXml.open('code_scheme', attributes={'name': 'Project', 'version': '173'}) codeStyleProjectXml.open('JavaCodeStyleSettings') # We cannot entirely disable wildcards import, but we can set the threshold to an insane number. codeStyleProjectXml.element('option', attributes={'name': 'CLASS_COUNT_TO_USE_IMPORT_ON_DEMAND', 'value': '65536'}) codeStyleProjectXml.element('option', attributes={'name': 'NAMES_COUNT_TO_USE_IMPORT_ON_DEMAND', 'value': '65536'}) codeStyleProjectXml.close('JavaCodeStyleSettings') codeStyleProjectXml.close('code_scheme') codeStyleProjectXml.close('component') codeStyleProjectFile = join(ideaProjectDirectory, 'codeStyles', 'Project.xml') mx.update_file(codeStyleProjectFile, codeStyleProjectXml.xml(indent=' ', newl='\n')) # Write checkstyle-idea.xml for the CheckStyle-IDEA checkstyleXml = mx.XMLDoc() checkstyleXml.open('project', attributes={'version': '4'}) checkstyleXml.open('component', attributes={'name': 'CheckStyle-IDEA'}) checkstyleXml.open('option', attributes={'name' : "configuration"}) checkstyleXml.open('map') if max_checkstyle_version: checkstyleXml.element('entry', attributes={'key': "checkstyle-version", 'value': str(max_checkstyle_version)}) # Initialize an entry for each style that is used checkstyleConfigs = set([]) for p in s.projects_recursive(): if not p.isJavaProject(): continue csConfig, checkstyleVersion, checkstyleProj = p.get_checkstyle_config() if not csConfig or csConfig in checkstyleConfigs: continue checkstyleConfigs.add(csConfig) checkstyleXml.element('entry', attributes={'key' : "location-" + str(len(checkstyleConfigs)), 'value': "PROJECT_RELATIVE:" + join(checkstyleProj.dir, ".checkstyle_checks.xml") + ":" + checkstyleProj.name}) checkstyleXml.close('map') checkstyleXml.close('option') checkstyleXml.close('component') checkstyleXml.close('project') checkstyleFile = join(ideaProjectDirectory, 'checkstyle-idea.xml') mx.update_file(checkstyleFile, checkstyleXml.xml(indent=' ', newl='\n')) # mx integration def antTargetName(dist): return 'archive_' + dist.name def artifactFileName(dist): return dist.name.replace('.', '_').replace('-', '_') + '.xml' validDistributions = [dist for dist in mx.sorted_dists() if not dist.suite.isBinarySuite() and not dist.isTARDistribution()] # 1) Make an ant file for archiving the distributions. antXml = mx.XMLDoc() antXml.open('project', attributes={'name': s.name, 'default': 'archive'}) for dist in validDistributions: antXml.open('target', attributes={'name': antTargetName(dist)}) antXml.open('exec', attributes={'executable': sys.executable}) antXml.element('arg', attributes={'value': join(mx._mx_home, 'mx.py')}) antXml.element('arg', attributes={'value': 'archive'}) antXml.element('arg', attributes={'value': '@' + dist.name}) antXml.close('exec') antXml.close('target') antXml.close('project') antFile = join(ideaProjectDirectory, 'ant-mx-archive.xml') mx.update_file(antFile, antXml.xml(indent=' ', newl='\n')) # 2) Tell IDEA that there is an ant-build. ant_mx_archive_xml = 'file://$PROJECT_DIR$/.idea/ant-mx-archive.xml' metaAntXml = mx.XMLDoc() metaAntXml.open('project', attributes={'version': '4'}) metaAntXml.open('component', attributes={'name': 'AntConfiguration'}) metaAntXml.open('buildFile', attributes={'url': ant_mx_archive_xml}) metaAntXml.close('buildFile') metaAntXml.close('component') metaAntXml.close('project') metaAntFile = join(ideaProjectDirectory, 'ant.xml') mx.update_file(metaAntFile, metaAntXml.xml(indent=' ', newl='\n')) # 3) Make an artifact for every distribution validArtifactNames = {artifactFileName(dist) for dist in validDistributions} artifactsDir = join(ideaProjectDirectory, 'artifacts') mx.ensure_dir_exists(artifactsDir) for fileName in os.listdir(artifactsDir): filePath = join(artifactsDir, fileName) if os.path.isfile(filePath) and fileName not in validArtifactNames: os.remove(filePath) for dist in validDistributions: artifactXML = mx.XMLDoc() artifactXML.open('component', attributes={'name': 'ArtifactManager'}) artifactXML.open('artifact', attributes={'build-on-make': 'true', 'name': dist.name}) artifactXML.open('output-path', data='$PROJECT_DIR$/mxbuild/artifacts/' + dist.name) artifactXML.close('output-path') artifactXML.open('properties', attributes={'id': 'ant-postprocessing'}) artifactXML.open('options', attributes={'enabled': 'true'}) artifactXML.open('file', data=ant_mx_archive_xml) artifactXML.close('file') artifactXML.open('target', data=antTargetName(dist)) artifactXML.close('target') artifactXML.close('options') artifactXML.close('properties') artifactXML.open('root', attributes={'id': 'root'}) for javaProject in [dep for dep in dist.archived_deps() if dep.isJavaProject()]: artifactXML.element('element', attributes={'id': 'module-output', 'name': javaProject.name}) for javaProject in [dep for dep in dist.deps if dep.isLibrary() or dep.isDistribution()]: artifactXML.element('element', attributes={'id': 'artifact', 'artifact-name': javaProject.name}) artifactXML.close('root') artifactXML.close('artifact') artifactXML.close('component') artifactFile = join(artifactsDir, artifactFileName(dist)) mx.update_file(artifactFile, artifactXML.xml(indent=' ', newl='\n')) def intellij_scm_name(vc_kind): if vc_kind == 'git': return 'Git' elif vc_kind == 'hg': return 'hg4idea' vcsXml = mx.XMLDoc() vcsXml.open('project', attributes={'version': '4'}) vcsXml.open('component', attributes={'name': 'VcsDirectoryMappings'}) suites_for_vcs = mx.suites() + ([mx._mx_suite] if mx_python_modules else []) sourceSuitesWithVCS = [vc_suite for vc_suite in suites_for_vcs if vc_suite.isSourceSuite() and vc_suite.vc is not None] uniqueSuitesVCS = {(vc_suite.vc_dir, vc_suite.vc.kind) for vc_suite in sourceSuitesWithVCS} for vcs_dir, kind in uniqueSuitesVCS: vcsXml.element('mapping', attributes={'directory': vcs_dir, 'vcs': intellij_scm_name(kind)}) vcsXml.close('component') vcsXml.close('project') vcsFile = join(ideaProjectDirectory, 'vcs.xml') mx.update_file(vcsFile, vcsXml.xml(indent=' ', newl='\n'))
def testdownstream(suite, repoUrls, relTargetSuiteDir, mxCommands, branch=None): """ Tests a downstream repo against the current working directory state of `suite`. :param mx.Suite suite: the suite to test against the downstream repo :param list repoUrls: URLs of downstream repos to clone, the first of which is the repo being tested :param str relTargetSuiteDir: directory of the downstream suite to test relative to the top level directory of the downstream repo being tested :param list mxCommands: argument lists for the mx commands run in downstream suite being tested :param str branch: name of branch to look for in downstream repo(s) """ assert len(repoUrls) > 0 repoUrls = [mx_urlrewrites.rewriteurl(url) for url in repoUrls] workDir = join(suite.get_output_root(), 'testdownstream') # A mirror of each suites in the same repo as `suite` is created via copying rel_mirror = os.path.relpath(suite.dir, mx.SuiteModel.siblings_dir(suite.dir)) in_subdir = os.sep in rel_mirror suites_in_repo = [suite] if in_subdir: base = os.path.dirname(suite.dir) for e in os.listdir(base): candidate = join(base, e) if candidate != suite.dir: mxDir = mx._is_suite_dir(candidate) if mxDir: matches = [s for s in mx.suites() if s.dir == candidate] if len(matches) == 0: suites_in_repo.append( mx.SourceSuite(mxDir, primary=False, load=False)) else: suites_in_repo.append(matches[0]) for suite_in_repo in suites_in_repo: if suite_in_repo.vc_dir and suite_in_repo.dir != suite_in_repo.vc_dir: mirror = join(workDir, basename(suite_in_repo.vc_dir), suite_in_repo.name) else: mirror = join(workDir, suite_in_repo.name) if exists(mirror): mx.rmtree(mirror) output_root = mx._safe_path(suite_in_repo.get_output_root()) def ignore_output_root(d, names): mx.log('Copying ' + d) if d == os.path.dirname(output_root): mx.log('Omitting ' + output_root) return [os.path.basename(output_root)] return [] mx.copytree(suite_in_repo.dir, mirror, ignore=ignore_output_root, symlinks=True) targetDir = None for repoUrl in repoUrls: # Deduce a target name from the target URL url = _urllib_parse.urlparse(repoUrl) targetName = url.path if targetName.rfind('/') != -1: targetName = targetName[targetName.rfind('/') + 1:] if targetName.endswith('.git'): targetName = targetName[0:-len('.git')] repoWorkDir = join(workDir, targetName) git = mx.GitConfig() if exists(repoWorkDir): git.pull(repoWorkDir) else: git.clone(repoUrl, repoWorkDir) # See if there's a matching (non-master) branch and use it if there is if not branch: branch = git.git_command( suite.dir, ['rev-parse', '--abbrev-ref', 'HEAD']).strip() if branch != 'master': git.git_command(repoWorkDir, ['checkout', branch], abortOnError=False) if not targetDir: targetDir = repoWorkDir assert not isabs(relTargetSuiteDir) targetSuiteDir = join(targetDir, relTargetSuiteDir) assert targetSuiteDir.startswith(targetDir) mxpy = None if suite != mx._mx_suite else join(mirror, 'mx.py') for command in mxCommands: mx.logv('[running "mx ' + ' '.join(command) + '" in ' + targetSuiteDir + ']') mx.run_mx(command, targetSuiteDir, mxpy=mxpy)
def testdownstream(suite, repoUrls, relTargetSuiteDir, mxCommands, branch=None): """ Tests a downstream repo against the current working directory state of `suite`. :param mx.Suite suite: the suite to test against the downstream repo :param list repoUrls: URLs of downstream repos to clone, the first of which is the repo being tested :param str relTargetSuiteDir: directory of the downstream suite to test relative to the top level directory of the downstream repo being tested :param list mxCommands: argument lists for the mx commands run in downstream suite being tested :param str branch: name of branch to look for in downstream repo(s) """ assert len(repoUrls) > 0 workDir = join(suite.get_output_root(), 'testdownstream') # A mirror of each suites in the same repo as `suite` is created with symlinks rel_mirror = os.path.relpath(suite.dir, mx.SuiteModel.siblings_dir(suite.dir)) in_subdir = os.sep in rel_mirror suites_in_repo = [suite] if in_subdir: base = os.path.dirname(suite.dir) for e in os.listdir(base): candidate = join(base, e) if candidate != suite.dir: mxDir = mx._is_suite_dir(candidate) if mxDir: matches = [s for s in mx.suites() if s.dir == candidate] if len(matches) == 0: suites_in_repo.append( mx.SourceSuite(mxDir, primary=False, load=False)) else: suites_in_repo.append(matches[0]) for suite_in_repo in suites_in_repo: rel_mirror = os.path.relpath( suite_in_repo.dir, mx.SuiteModel.siblings_dir(suite_in_repo.dir)) mirror = join(workDir, rel_mirror) if exists(mirror): shutil.rmtree(mirror) mx.ensure_dir_exists(mirror) for f in os.listdir(suite_in_repo.dir): subDir = join(suite_in_repo.dir, f) if subDir == suite_in_repo.get_output_root(): continue src = join(suite_in_repo.dir, f) dst = join(mirror, f) mx.logv('[Creating symlink from {} to {}]'.format(dst, src)) relsrc = os.path.relpath(src, os.path.dirname(dst)) os.symlink(relsrc, dst) targetDir = None for repoUrl in repoUrls: # Deduce a target name from the target URL url = urlparse(repoUrl) targetName = url.path if targetName.rfind('/') != -1: targetName = targetName[targetName.rfind('/') + 1:] if targetName.endswith('.git'): targetName = targetName[0:-len('.git')] repoWorkDir = join(workDir, targetName) git = mx.GitConfig() if exists(repoWorkDir): git.pull(repoWorkDir) else: git.clone(repoUrl, repoWorkDir) # See if there's a matching (non-master) branch and use it if there is if not branch: branch = git.git_command( suite.dir, ['rev-parse', '--abbrev-ref', 'HEAD']).strip() if branch != 'master': git.git_command(repoWorkDir, ['checkout', branch], abortOnError=False) if not targetDir: targetDir = repoWorkDir assert not isabs(relTargetSuiteDir) targetSuiteDir = join(targetDir, relTargetSuiteDir) assert targetSuiteDir.startswith(targetDir) mxpy = None if suite != mx._mx_suite else join(mirror, 'mx.py') for command in mxCommands: mx.logv('[running "mx ' + ' '.join(command) + '" in ' + targetSuiteDir + ']') mx.run_mx(command, targetSuiteDir, mxpy=mxpy)
def gc_dists(args): """ Garbage collect mx distributions.""" parser = argparse.ArgumentParser(prog='mx gc-dists', description='''Garbage collect layout distributions. By default, it collects all found layout distributions that are *not* part of the current configuration (see `--keep-current`). This command respects mx level suite filtering (e.g., `mx --suite my-suite gc-dists`). ''', epilog='''If the environment variable `MX_GC_AFTER_BUILD` is set, %(prog)s will be executed after `mx build` using the content of the environment variable as parameters.''') # mutually exclusive groups do not support title and description - wrapping in another group as a workaround action_group_desc = parser.add_argument_group('actions', 'What to do with the result. One of the following arguments is required.') action_group = action_group_desc.add_mutually_exclusive_group(required=True) action_group.add_argument('-f', '--force', action='store_true', help='remove layout distributions without further questions') action_group.add_argument('-n', '--dry-run', action='store_true', help='show what would be removed without actually doing anything') action_group.add_argument('-i', '--interactive', action='store_true', help='ask for every layout distributions whether it should be removed') keep_current_group_desc = parser.add_argument_group('current configuration handling', description='How to deal with the current configuration, i.e., what `mx build` would rebuild.') keep_current_group = keep_current_group_desc.add_mutually_exclusive_group() keep_current_group.add_argument('--keep-current', action='store_true', default=True, help='keep layout distributions of the current configuration (default)') keep_current_group.add_argument('--no-keep-current', action='store_false', dest='keep_current', help='remove layout distributions of the current configuration') filter_group = parser.add_argument_group('result filters', description='Filter can be combined.') filter_group.add_argument('--reverse', action='store_true', help='reverse the result') filter_group.add_argument('--older-than', action=TimeAction, help='only show results older than the specified point in time (format: {})'.format(TimeAction.fmt.replace('%', '%%'))) try: parsed_args = parser.parse_args(args) except ValueError as ve: parser.error(str(ve)) suites = mx.suites(opt_limit_to_suite=True, includeBinary=False, include_mx=False) c = [] for s in suites: c += _gc_layout_dists(s, parsed_args) if not c: mx.log("Nothing to do!") return if parsed_args.older_than: c = [x for x in c if x[1] < parsed_args.older_than] # sort by mod date c = sorted(c, key=lambda x: x[1], reverse=parsed_args.reverse) # calculate max sizes max_path = 0 max_mod_time = 0 max_size = 0 for path, mod_time, size in c: max_path = max(len(path), max_path) max_mod_time = max(len(_format_datetime(mod_time)), max_mod_time) max_size = max(len(_format_bytes(size)), max_size) msg_fmt = '{0:<' + str(max_path) + '} modified {1:<' + str(max_mod_time + len(' ago')) +'} {2:<' + str(max_size) + '}' size_sum = 0 for path, mod_time, size in c: if parsed_args.dry_run: mx.log(msg_fmt.format(path, _format_datetime(mod_time) + ' ago', _format_bytes(size))) size_sum += size else: msg = '{0} (modified {1} ago, size {2})'.format(path, _format_datetime(mod_time), _format_bytes(size)) if parsed_args.force or parsed_args.interactive and mx.ask_yes_no('Delete ' + msg): mx.log('rm ' + path) mx.rmtree(path) size_sum += size if parsed_args.dry_run: mx.log('Would free ' + _format_bytes(size_sum)) else: mx.log('Freed ' + _format_bytes(size_sum))
def fsckprojects(args): """find directories corresponding to deleted Java projects and delete them""" for suite in mx.suites(True, includeBinary=False): projectDirs = [p.dir for p in suite.projects] distIdeDirs = [ d.get_ide_project_dir() for d in suite.dists if d.isJARDistribution() and d.get_ide_project_dir() is not None ] for dirpath, dirnames, files in os.walk(suite.dir): if dirpath == suite.dir: # no point in traversing vc metadata dir, lib, .workspace # if there are nested source suites must not scan those now, as they are not in projectDirs (but contain .project files) omitted = [suite.mxDir, 'lib', '.workspace', 'mx.imports'] if suite.vc: omitted.append(suite.vc.metadir()) dirnames[:] = [d for d in dirnames if d not in omitted] elif dirpath == suite.get_output_root(): # don't want to traverse output dir dirnames[:] = [] elif dirpath == suite.mxDir: # don't want to traverse mx.name as it contains a .project dirnames[:] = [] elif dirpath in projectDirs: # don't traverse subdirs of an existing project in this suite dirnames[:] = [] elif dirpath in distIdeDirs: # don't traverse subdirs of an existing distribution in this suite dirnames[:] = [] else: maybe_project = basename(dirpath) if not mx._removedDeps.get(maybe_project): projectConfigFiles = frozenset([ '.classpath', '.project', 'nbproject', maybe_project + '.iml' ]) indicators = projectConfigFiles.intersection(files) if len(indicators) != 0: indicators = [ os.path.relpath(join(dirpath, i), suite.vc_dir) for i in indicators ] indicatorsInVC = suite.vc.locate( suite.vc_dir, indicators) # Only proceed if there are indicator files that are not under VC if len(indicators) > len(indicatorsInVC): if mx.ask_yes_no( dirpath + ' looks like a removed project -- delete it', 'n'): shutil.rmtree(dirpath) mx.log('Deleted ' + dirpath) ideaProjectDirectory = join(suite.dir, '.idea') librariesDirectory = join(ideaProjectDirectory, 'libraries') if librariesDirectory and exists(librariesDirectory): neededLibraries = set() unique_library_file_names = set() for p in suite.projects_recursive( ) + mx._mx_suite.projects_recursive(): if not p.isJavaProject(): continue def processDep(dep, edge): if dep is p: return if dep.isLibrary() or dep.isJARDistribution( ) or dep.isJdkLibrary() or dep.isMavenProject( ) or dep.isClasspathDependency(): neededLibraries.add(dep) p.walk_deps(visit=processDep, ignoredEdges=[mx.DEP_EXCLUDED]) neededLibraryFiles = frozenset([ mx_ide_intellij._intellij_library_file_name( l.name, unique_library_file_names) for l in neededLibraries ]) existingLibraryFiles = frozenset(os.listdir(librariesDirectory)) for library_file in existingLibraryFiles - neededLibraryFiles: file_path = join(librariesDirectory, library_file) relative_file_path = os.path.relpath(file_path, os.curdir) if mx.ask_yes_no( relative_file_path + ' looks like a removed library -- delete it', 'n'): os.remove(file_path) mx.log('Deleted ' + relative_file_path)
def testdownstream(suite, repoUrls, relTargetSuiteDir, mxCommands, branch=None): """ Tests a downstream repo against the current working directory state of `suite`. :param mx.Suite suite: the suite to test against the downstream repo :param list repoUrls: URLs of downstream repos to clone, the first of which is the repo being tested :param str relTargetSuiteDir: directory of the downstream suite to test relative to the top level directory of the downstream repo being tested :param list mxCommands: argument lists for the mx commands run in downstream suite being tested :param str branch: name of branch to look for in downstream repo(s) """ assert len(repoUrls) > 0 workDir = join(suite.get_output_root(), 'testdownstream') # A mirror of each suites in the same repo as `suite` is created via copying rel_mirror = os.path.relpath(suite.dir, mx.SuiteModel.siblings_dir(suite.dir)) in_subdir = os.sep in rel_mirror suites_in_repo = [suite] if in_subdir: base = os.path.dirname(suite.dir) for e in os.listdir(base): candidate = join(base, e) if candidate != suite.dir: mxDir = mx._is_suite_dir(candidate) if mxDir: matches = [s for s in mx.suites() if s.dir == candidate] if len(matches) == 0: suites_in_repo.append(mx.SourceSuite(mxDir, primary=False, load=False)) else: suites_in_repo.append(matches[0]) for suite_in_repo in suites_in_repo: if suite_in_repo.vc_dir and suite_in_repo.dir != suite_in_repo.vc_dir: mirror = join(workDir, basename(suite_in_repo.vc_dir), suite_in_repo.name) else: mirror = join(workDir, suite_in_repo.name) if exists(mirror): shutil.rmtree(mirror) output_root = suite_in_repo.get_output_root() def ignore_output_root(d, names): mx.log('Copying ' + d) if d == os.path.dirname(output_root): mx.log('Omitting ' + output_root) return [os.path.basename(output_root)] return [] shutil.copytree(suite_in_repo.dir, mirror, ignore=ignore_output_root) targetDir = None for repoUrl in repoUrls: # Deduce a target name from the target URL url = urlparse(repoUrl) targetName = url.path if targetName.rfind('/') != -1: targetName = targetName[targetName.rfind('/') + 1:] if targetName.endswith('.git'): targetName = targetName[0:-len('.git')] repoWorkDir = join(workDir, targetName) git = mx.GitConfig() if exists(repoWorkDir): git.pull(repoWorkDir) else: git.clone(repoUrl, repoWorkDir) # See if there's a matching (non-master) branch and use it if there is if not branch: branch = git.git_command(suite.dir, ['rev-parse', '--abbrev-ref', 'HEAD']).strip() if branch != 'master': git.git_command(repoWorkDir, ['checkout', branch], abortOnError=False) if not targetDir: targetDir = repoWorkDir assert not isabs(relTargetSuiteDir) targetSuiteDir = join(targetDir, relTargetSuiteDir) assert targetSuiteDir.startswith(targetDir) mxpy = None if suite != mx._mx_suite else join(mirror, 'mx.py') for command in mxCommands: mx.logv('[running "mx ' + ' '.join(command) + '" in ' + targetSuiteDir + ']') mx.run_mx(command, targetSuiteDir, mxpy=mxpy)