def _checkout_upstream_revision(upstream_commit, candidate_upstream_branches, upstream_suite, downstream_suite): """ :type upstream_commit: str :type candidate_upstream_branches: str :type upstream_suite: str :type downstream_suite: str :rtype: bool """ for upstream_branch in candidate_upstream_branches: mx.log("Analyzing branch '{}':".format(upstream_branch)) rev_parse_output = _run_git_cmd( downstream_suite.vc_dir, ['rev-parse', '--verify', 'origin/{}'.format(upstream_branch)], regex=r'[a-f0-9]{40}$', abortOnError=False) if rev_parse_output is None: mx.log( " - the downstream repository does not contain a branch named '{}'" .format(upstream_branch)) continue mx.log( " - the downstream repository contains a branch named '{}'".format( upstream_branch)) downstream_branch = upstream_branch mx.log( " - searching 'origin/{}' of the downstream repo in '{}' for a commit that imports revision '{}' of '{}'" .format(downstream_branch, downstream_suite.vc_dir, upstream_commit, upstream_suite.name)) # Print the oldest (`--reverse`) revision (`--pretty=%H`) of a commit in the matching branch of the repository of the downstream suite that contains `PullRequest: ` in the commit message (`--grep=...` and `-m`) and mentions the upstream commit (`-S`) downstream_commit_cmd = [ 'log', 'origin/{}'.format(downstream_branch), '--pretty=%H', '--grep=PullRequest: ', '--reverse', '-m', '-S', upstream_commit, '--', downstream_suite.suite_py() ] downstream_commit = _run_git_cmd( downstream_suite.vc_dir, downstream_commit_cmd, regex=r'[a-f0-9]{40}(\n[a-f0-9]{40})?$', abortOnError=False) if downstream_commit is None: mx.log( " - cannot find a revision in branch 'origin/{}' of '{}' that imports revision '{}' of '{}'" .format(downstream_branch, downstream_suite.vc_dir, upstream_commit, upstream_suite.name)) continue downstream_commit = downstream_commit.split('\n')[0] mx.log( "Checking out revision '{}' of downstream suite '{}', which imports revision '{}' of '{}'" .format(downstream_commit, downstream_suite.name, upstream_commit, upstream_suite.name)) mx.GitConfig().update(downstream_suite.vc_dir, downstream_commit, mayPull=False, clean=False, abortOnError=True) return True return False
def testdownstream(args): """test downstream users of the Truffle API""" jruby_dir = 'jruby' jruby_repo = 'https://github.com/jruby/jruby.git' jruby_branch = 'truffle-head' git = mx.GitConfig() if os.path.exists('jruby'): git.run(['git', 'reset', 'HEAD', '--hard'], nonZeroIsFatal=True, cwd=jruby_dir) git.pull('jruby') else: git.clone(jruby_repo, jruby_dir) git.run(['git', 'checkout', jruby_branch], nonZeroIsFatal=True, cwd=jruby_dir) dev_version = _suite.release_version(snapshotSuffix='SNAPSHOT') subprocess.check_call(['tool/truffle/set_truffle_version.sh', dev_version], cwd=jruby_dir) mx.build([]) mx.maven_install([]) subprocess.check_call(['./mvnw', '-X', 'clean'], cwd=jruby_dir) subprocess.check_call([ './mvnw', '-X', ], cwd=jruby_dir) subprocess.check_call(['bin/jruby', 'tool/jt.rb', 'test', 'fast'], cwd=jruby_dir)
def _run_downstream_svm(commands, binary=False): new_rewrites = None if binary: localmvn = "/tmp/graalpythonsnapshots" localmvnrepl = "file://%s" % localmvn publicmvn = mx.repository("python-public-snapshots").url publicmvnpattern = re.compile(publicmvn) git = mx.GitConfig() new_rewrites = [{ publicmvnpattern.pattern: { "replacement": localmvnrepl } }] for rewrite in _urlrewrites: if rewrite.pattern.match(publicmvn): # we replace rewrites of our public repo pass elif publicmvnpattern.match(rewrite.replacement): # we rewrite to what we want new_rewrites.append( {rewrite.pattern.pattern: { "replacement": localmvnrepl }}) else: new_rewrites.append({ rewrite.pattern.pattern: { "replacement": rewrite.replacement } }) os.environ["TRUFFLE_PYTHON_VERSION"] = git.tip(_suite.dir).strip() os.environ["TRUFFLE_SULONG_VERSION"] = git.tip(_sulong.dir).strip() prev_urlrewrites = os.environ.get("MX_URLREWRITES") os.environ["MX_URLREWRITES"] = json.dumps(new_rewrites) mx.command_function("deploy-binary")( ["--all-suites", "python-local-snapshots", localmvnrepl]) try: mx.log(str(dict(os.environ))) testdownstream(_suite, [ mx.suite("truffle").vc._remote_url( mx.suite("truffle").dir, "origin") ], "substratevm", commands) finally: if binary: os.environ.pop("TRUFFLE_PYTHON_VERSION") os.environ.pop("TRUFFLE_SULONG_VERSION") if prev_urlrewrites: os.environ["MX_URLREWRITES"] = prev_urlrewrites else: os.environ.pop("MX_URLREWRITES") shutil.rmtree(localmvn, ignore_errors=True)
def _run_git_cmd(vc_dir, cmd, regex=None, abortOnError=True): """ :type vc_dir: str :type cmd: list[str] :type regex: str | None :type abortOnError: bool :rtype: str """ git = mx.GitConfig() output = (git.git_command(vc_dir, cmd, abortOnError=abortOnError) or '').strip() if regex is not None and re.match(regex, output, re.MULTILINE) is None: if abortOnError: raise mx.abort("Unexpected output running command '{cmd}'. Expected a match for '{regex}', got:\n{output}".format(cmd=' '.join(map(pipes.quote, ['git', '-C', vc_dir, '--no-pager'] + cmd)), regex=regex, output=output)) return None return output
def _fetch_test262(): """clones/updates test262 test-suite""" _location = join(_suite.dir, 'lib', 'test262') _clone = False if not mx.isdir(_location): _clone = True else: if not mx.isdir(join(_location, '.git')): # Not a git repository, an old version of the test-suite extracted from an archive most likely. shutil.rmtree(_location) _clone = True if _clone: NoCRLFGitConfig().clone(url=mx_urlrewrites.rewriteurl(TEST262_REPO), dest=_location, rev=TEST262_REV, abortOnError=True) else: mx.GitConfig().update(_location, rev=TEST262_REV, mayPull=True, abortOnError=True)
def testgraal(args): cloneFrom = mx.get_env("GRAAL_URL") if not cloneFrom: cloneFrom = "http://github.com/graalvm/graal-core" graalSuiteSubDir = mx.get_env("GRAAL_SUITE_SUBDIR") suite = mx.suite('truffle') suiteDir = suite.dir workDir = join(suite.get_output_root(), 'sanitycheck') mx.ensure_dir_exists(join(workDir, suite.name)) for f in os.listdir(suiteDir): subDir = os.path.join(suiteDir, f) if subDir == suite.get_output_root(): continue src = join(suiteDir, f) tgt = join(workDir, suite.name, f) if isdir(src): if exists(tgt): shutil.rmtree(tgt) shutil.copytree(src, tgt) else: shutil.copy(src, tgt) sanityDir = join(workDir, 'sanity') git = mx.GitConfig() if exists(sanityDir): git.pull(sanityDir) else: git.clone(cloneFrom, sanityDir) sanitySuiteDir = sanityDir if graalSuiteSubDir is None else join( sanityDir, graalSuiteSubDir) return mx.run_mx([ '-v', '--java-home=' + mx.get_jdk().home, 'gate', '-B--force-deprecation-as-warning', '--tags', 'build,test' ], sanitySuiteDir)
def testdownstream(suite, repoUrls, relTargetSuiteDir, mxCommands, branch=None): """ Tests a downstream repo against the current working directory state of `suite`. :param mx.Suite suite: the suite to test against the downstream repo :param list repoUrls: URLs of downstream repos to clone, the first of which is the repo being tested :param str relTargetSuiteDir: directory of the downstream suite to test relative to the top level directory of the downstream repo being tested :param list mxCommands: argument lists for the mx commands run in downstream suite being tested :param str branch: name of branch to look for in downstream repo(s) """ assert len(repoUrls) > 0 workDir = join(suite.get_output_root(), 'testdownstream') # A mirror of each suites in the same repo as `suite` is created with symlinks rel_mirror = os.path.relpath(suite.dir, mx.SuiteModel.siblings_dir(suite.dir)) in_subdir = os.sep in rel_mirror suites_in_repo = [suite] if in_subdir: base = os.path.dirname(suite.dir) for e in os.listdir(base): candidate = join(base, e) if candidate != suite.dir: mxDir = mx._is_suite_dir(candidate) if mxDir: matches = [s for s in mx.suites() if s.dir == candidate] if len(matches) == 0: suites_in_repo.append( mx.SourceSuite(mxDir, primary=False, load=False)) else: suites_in_repo.append(matches[0]) for suite_in_repo in suites_in_repo: rel_mirror = os.path.relpath( suite_in_repo.dir, mx.SuiteModel.siblings_dir(suite_in_repo.dir)) mirror = join(workDir, rel_mirror) if exists(mirror): shutil.rmtree(mirror) mx.ensure_dir_exists(mirror) for f in os.listdir(suite_in_repo.dir): subDir = join(suite_in_repo.dir, f) if subDir == suite_in_repo.get_output_root(): continue src = join(suite_in_repo.dir, f) dst = join(mirror, f) mx.logv('[Creating symlink from {} to {}]'.format(dst, src)) relsrc = os.path.relpath(src, os.path.dirname(dst)) os.symlink(relsrc, dst) targetDir = None for repoUrl in repoUrls: # Deduce a target name from the target URL url = urlparse(repoUrl) targetName = url.path if targetName.rfind('/') != -1: targetName = targetName[targetName.rfind('/') + 1:] if targetName.endswith('.git'): targetName = targetName[0:-len('.git')] repoWorkDir = join(workDir, targetName) git = mx.GitConfig() if exists(repoWorkDir): git.pull(repoWorkDir) else: git.clone(repoUrl, repoWorkDir) # See if there's a matching (non-master) branch and use it if there is if not branch: branch = git.git_command( suite.dir, ['rev-parse', '--abbrev-ref', 'HEAD']).strip() if branch != 'master': git.git_command(repoWorkDir, ['checkout', branch], abortOnError=False) if not targetDir: targetDir = repoWorkDir assert not isabs(relTargetSuiteDir) targetSuiteDir = join(targetDir, relTargetSuiteDir) assert targetSuiteDir.startswith(targetDir) mxpy = None if suite != mx._mx_suite else join(mirror, 'mx.py') for command in mxCommands: mx.logv('[running "mx ' + ' '.join(command) + '" in ' + targetSuiteDir + ']') mx.run_mx(command, targetSuiteDir, mxpy=mxpy)
def checkout_downstream(args): """checkout a revision of the downstream suite that imports the commit checked-out in the upstream suite, or the closest parent commit""" parser = ArgumentParser( prog='mx checkout-downstream', description= 'Checkout a revision of the downstream suite that imports the currently checked-out version of the upstream suite' ) parser.add_argument('upstream', action='store', help='the name of the upstream suite (e.g., compiler)') parser.add_argument( 'downstream', action='store', help='the name of the downstream suite (e.g., graal-enterprise)') parser.add_argument( '--no-fetch', action='store_true', help= 'do not fetch remote content for the upstream and downstream repositories' ) args = parser.parse_args(args) def get_suite(name): suite = mx.suite(name, fatalIfMissing=False) if suite is None: raise mx.abort( "Cannot load the '{}' suite. Did you forget a dynamic import or pass a repository name rather than a suite name (e.g., 'graal' rather than 'compiler')?" .format(name)) return suite upstream_suite = get_suite(args.upstream) downstream_suite = get_suite(args.downstream) if upstream_suite.vc_dir == downstream_suite.vc_dir: raise mx.abort( "Suites '{}' and '{}' are part of the same repository, cloned in '{}'" .format(upstream_suite.name, downstream_suite.name, upstream_suite.vc_dir)) if len(downstream_suite.suite_imports) == 0: raise mx.abort( "Downstream suite '{}' does not have dependencies".format( downstream_suite.name)) if upstream_suite.name not in ( suite_import.name for suite_import in downstream_suite.suite_imports): raise mx.abort( "'{}' is not a dependency of '{}'. Valid dependencies are:\n - {}". format( upstream_suite.name, downstream_suite.name, '\n - '.join([s.name for s in downstream_suite.suite_imports]))) git = mx.GitConfig() for suite in upstream_suite, downstream_suite: if not git.is_this_vc(suite.vc_dir): raise mx.abort("Suite '{}' is not part of a Git repo.".format( suite.name)) if not args.no_fetch: mx.log("Fetching remote content from '{}'".format( git.default_pull(downstream_suite.vc_dir))) git.pull(downstream_suite.vc_dir, rev=None, update=False, abortOnError=True) # Print the revision (`--pretty=%H`) of the first (`--max-count=1`) merge commit (`--merges`) in the upstream repository that contains `PullRequest: ` in the commit message (`--grep=...`) upstream_commit_cmd = [ 'log', '--pretty=%H', '--grep=PullRequest: ', '--merges', '--max-count=1' ] upstream_commit = _run_git_cmd(upstream_suite.vc_dir, upstream_commit_cmd, regex=r'[a-f0-9]{40}$') # We now need to find a revision in `downstream_suite` that imports `upstream_commit` of `upstream_suite`. # For doing this, we grep the log of a set of branches in `downstream_suite`, checking out the revision of the first branch that matches. # As a consequence, the order in which we check branches in `downstream_suite` is fundamental: # 1. we check if the `${DOWNSTREAM_BRANCH}` env var is set. If so, we look for branches named `${DOWNSTREAM_BRANCH}` and `${DOWNSTREAM_BRANCH}_gate` # 2. we (optionally) fetch `upstream_suite` and ask git which branches of `upstream_suite` contain `upstream_commit` ci_upstream_branch_candidates = [] ci_downstream_branch = mx.get_env('DOWNSTREAM_BRANCH', None) if ci_downstream_branch is not None: ci_upstream_branch_candidates.extend( [ci_downstream_branch, ci_downstream_branch + '_gate']) mx.log( "The '$DOWNSTREAM_BRANCH' env var is set. Adding '{}' to the list of upstream branch candidates." .format(ci_upstream_branch_candidates)) if not _checkout_upstream_revision( upstream_commit, ci_upstream_branch_candidates, upstream_suite, downstream_suite): if not args.no_fetch: mx.log("Fetching remote content from '{}'".format( git.default_pull(upstream_suite.vc_dir))) git.pull(upstream_suite.vc_dir, rev=None, update=False, abortOnError=True) # Print the list of branches that contain the upstream commit upstream_branches_out = _run_git_cmd( upstream_suite.vc_dir, ['branch', '-a', '--contains', upstream_commit ]) # old git versions do not support `--format` upstream_branch_candidates = [] for ub in upstream_branches_out.split('\n'): ub = re.sub(r'^\*', '', ub).lstrip() ub = re.sub('^remotes/origin/', '', ub) if not re.match(r'\(HEAD detached at [a-z0-9]+\)$', ub): upstream_branch_candidates.append(ub) mx.log( "The most recent merge perfomed by the CI on the active branch of the upstream repository is at revision '{}', which is part of the following branches:\n- {}" .format(upstream_commit, '\n- '.join(upstream_branch_candidates))) if not _checkout_upstream_revision( upstream_commit, upstream_branch_candidates, upstream_suite, downstream_suite): raise mx.abort( "Cannot find a revision of '{}' that imports revision '{}' of '{}" .format(downstream_suite.vc_dir, upstream_commit, upstream_suite.name))
def testdownstream(suite, repoUrls, relTargetSuiteDir, mxCommands, branch=None): """ Tests a downstream repo against the current working directory state of `suite`. :param mx.Suite suite: the suite to test against the downstream repo :param list repoUrls: URLs of downstream repos to clone, the first of which is the repo being tested :param str relTargetSuiteDir: directory of the downstream suite to test relative to the top level directory of the downstream repo being tested :param list mxCommands: argument lists for the mx commands run in downstream suite being tested :param str branch: name(s) of branch to look for in downstream repo(s) """ assert len(repoUrls) > 0 repoUrls = [mx_urlrewrites.rewriteurl(url) for url in repoUrls] workDir = join(suite.get_output_root(), 'testdownstream') # A mirror of each suites in the same repo as `suite` is created via copying rel_mirror = os.path.relpath(suite.dir, mx.SuiteModel.siblings_dir(suite.dir)) in_subdir = os.sep in rel_mirror suites_in_repo = [suite] if in_subdir: base = os.path.dirname(suite.dir) for e in os.listdir(base): candidate = join(base, e) if candidate != suite.dir: mxDir = mx._is_suite_dir(candidate) if mxDir: matches = [s for s in mx.suites() if s.dir == candidate] if len(matches) == 0: suites_in_repo.append( mx.SourceSuite(mxDir, primary=False, load=False)) else: suites_in_repo.append(matches[0]) if suite.vc: vc_metadir = mx._safe_path(mx.VC.get_vc(suite.vc_dir).metadir()) blacklist = {suite.vc_dir: [join(suite.vc_dir, vc_metadir)]} else: blacklist = {} for suite_in_repo in suites_in_repo: output_root = mx._safe_path(suite_in_repo.get_output_root()) blacklist.setdefault(dirname(output_root), []).append(output_root) def omitted_dirs(d, names): mx.log('Copying ' + d) to_omit = [] for blacklisted_dir in blacklist.get(d, []): mx.log('Omitting ' + blacklisted_dir) to_omit.append(basename(blacklisted_dir)) return to_omit if suite.vc_dir and suite.dir != suite.vc_dir: mirror = join(workDir, basename(suite.vc_dir)) else: mirror = join(workDir, suite.name) if exists(mirror): mx.rmtree(mirror) mx.copytree(suite.vc_dir, mirror, ignore=omitted_dirs, symlinks=True) targetDir = None for repoUrl in repoUrls: # Deduce a target name from the target URL url = _urllib_parse.urlparse(repoUrl) targetName = url.path if targetName.rfind('/') != -1: targetName = targetName[targetName.rfind('/') + 1:] if targetName.endswith('.git'): targetName = targetName[0:-len('.git')] repoWorkDir = join(workDir, targetName) git = mx.GitConfig() if exists(repoWorkDir): git.pull(repoWorkDir) else: git.clone(repoUrl, repoWorkDir) if branch is None: branch = [] elif isinstance(branch, str): branch = [branch] else: assert isinstance(branch, list) # fall back to the branch of the main repo active_branch = git.active_branch(suite.dir, abortOnError=False) if active_branch: branch.append(active_branch) updated = False for branch_name in branch: if git.update_to_branch(repoWorkDir, branch_name, abortOnError=False): updated = True break if not updated: mx.warn('Could not update {} to any of the following branches: {}'. format(repoWorkDir, ', '.join(branch))) if not targetDir: targetDir = repoWorkDir assert not isabs(relTargetSuiteDir) targetSuiteDir = join(targetDir, relTargetSuiteDir) assert targetSuiteDir.startswith(targetDir) mxpy = None if suite != mx._mx_suite else join(mirror, 'mx.py') for command in mxCommands: mx.logv('[running "mx ' + ' '.join(command) + '" in ' + targetSuiteDir + ']') mx.run_mx(command, targetSuiteDir, mxpy=mxpy)
def checkout_downstream(args): """checkout a revision of the downstream suite that imports the commit checked-out in the upstream suite, or the closest parent commit""" parser = ArgumentParser( prog='mx checkout-downstream', description= 'Checkout a revision of the downstream suite that imports the currently checked-out version of the upstream suite' ) parser.add_argument('upstream', action='store', help='the name of the upstream suite (e.g., compiler)') parser.add_argument( 'downstream', action='store', help='the name of the downstream suite (e.g., graal-enterprise)') args = parser.parse_args(args) def get_suite(name): suite = mx.suite(name, fatalIfMissing=False) if suite is None: raise mx.abort( "Cannot load the '{}' suite. Did you forget a dynamic import or pass a repository name rather than a suite name (e.g., 'graal' rather than 'compiler')?" .format(name)) return suite upstream_suite = get_suite(args.upstream) downstream_suite = get_suite(args.downstream) if upstream_suite.vc_dir == downstream_suite.vc_dir: raise mx.abort( "Suites '{}' and '{}' are part of the same repository, cloned in '{}'" .format(upstream_suite.name, downstream_suite.name, upstream_suite.vc_dir)) if len(downstream_suite.suite_imports) == 0: raise mx.abort( "Downstream suite '{}' does not have dependencies".format( downstream_suite.name)) if upstream_suite.name not in ( suite_import.name for suite_import in downstream_suite.suite_imports): raise mx.abort( "'{}' is not a dependency of '{}'. Valid dependencies are:\n - {}". format( upstream_suite.name, downstream_suite.name, '\n - '.join([s.name for s in downstream_suite.suite_imports]))) git = mx.GitConfig() for suite in upstream_suite, downstream_suite: if not git.is_this_vc(suite.vc_dir): raise mx.abort("Suite '{}' is not part of a Git repo.".format( suite.name)) def run_git_cmd(vc_dir, cmd, regex=None, abortOnError=True): """ :type vc_dir: str :type cmd: list[str] :type regex: str | None :rtype: str """ output = (git.git_command(vc_dir, cmd, abortOnError=abortOnError) or '').strip() if regex is not None and re.match(regex, output, re.MULTILINE) is None: if abortOnError: raise mx.abort( "Unexpected output running command '{cmd}'. Expected a match for '{regex}', got:\n{output}" .format(cmd=' '.join( map(pipes.quote, ['git', '-C', vc_dir, '--no-pager'] + cmd)), regex=regex, output=output)) else: return None return output mx.log("Fetching remote content from '{}'".format( git.default_pull(downstream_suite.vc_dir))) git.pull(downstream_suite.vc_dir, rev=None, update=False, abortOnError=True) # Print the revision (`--pretty=%H`) of the first (`--max-count=1`) merge commit (`--merges`) in the upstream repository that contains `PullRequest: ` in the commit message (`--grep=...`) upstream_commit_cmd = [ 'log', '--pretty=%H', '--grep=PullRequest: ', '--merges', '--max-count=1' ] upstream_commit = run_git_cmd(upstream_suite.vc_dir, upstream_commit_cmd, regex=r'[a-f0-9]{40}$') upstream_branch = run_git_cmd(upstream_suite.vc_dir, ['rev-parse', '--abbrev-ref', 'HEAD']) if upstream_branch == 'HEAD': upstream_branch = 'master' mx.log("The active branch of the upstream repository is '{}'".format( upstream_branch)) rev_parse_output = run_git_cmd( downstream_suite.vc_dir, ['rev-parse', '--verify', 'origin/{}'.format(upstream_branch)], regex=r'[a-f0-9]{40}$', abortOnError=False) if rev_parse_output is None: mx.log( "The downstream repository does not contain a branch named '{}'. Defaulting to 'master'" .format(upstream_branch)) downstream_branch = 'master' else: mx.log("The downstream repository contains a branch named '{}'".format( upstream_branch)) downstream_branch = upstream_branch mx.log( "Searching 'origin/{}' of the downstream repo in '{}' for a commit that imports revision '{}' of '{}'" .format(downstream_branch, downstream_suite.vc_dir, upstream_commit, upstream_suite.name)) # Print the oldest (`--reverse`) revision (`--pretty=%H`) of a commit in the matching branch of the repository of the downstream suite that contains `PullRequest: ` in the commit message (`--grep=...` and `-m`) and mentions the upstream commit (`-S`) downstream_commit_cmd = [ 'log', 'origin/{}'.format(downstream_branch), '--pretty=%H', '--grep=PullRequest: ', '--reverse', '-m', '-S', upstream_commit, '--', downstream_suite.suite_py() ] downstream_commit = run_git_cmd(downstream_suite.vc_dir, downstream_commit_cmd, regex=r'[a-f0-9]{40}(\n[a-f0-9]{40})?$', abortOnError=False) if downstream_commit is None: raise mx.abort( "Cannot find a revision in branch 'origin/{}' of '{}' that imports revision '{}' of '{}'" .format(downstream_branch, downstream_suite.vc_dir, upstream_commit, upstream_suite.name)) downstream_commit = downstream_commit.split('\n')[0] mx.log( "Checking out revision '{}' of downstream suite '{}', which imports revision '{}' of '{}'" .format(downstream_commit, downstream_suite.name, upstream_commit, upstream_suite.name)) git.update(downstream_suite.vc_dir, downstream_commit, mayPull=False, clean=False, abortOnError=True)
def testdownstream(suite, repoUrls, relTargetSuiteDir, mxCommands, branch=None): """ Tests a downstream repo against the current working directory state of `suite`. :param mx.Suite suite: the suite to test against the downstream repo :param list repoUrls: URLs of downstream repos to clone, the first of which is the repo being tested :param str relTargetSuiteDir: directory of the downstream suite to test relative to the top level directory of the downstream repo being tested :param list mxCommands: argument lists for the mx commands run in downstream suite being tested :param str branch: name of branch to look for in downstream repo(s) """ assert len(repoUrls) > 0 repoUrls = [mx_urlrewrites.rewriteurl(url) for url in repoUrls] workDir = join(suite.get_output_root(), 'testdownstream') # A mirror of each suites in the same repo as `suite` is created via copying rel_mirror = os.path.relpath(suite.dir, mx.SuiteModel.siblings_dir(suite.dir)) in_subdir = os.sep in rel_mirror suites_in_repo = [suite] if in_subdir: base = os.path.dirname(suite.dir) for e in os.listdir(base): candidate = join(base, e) if candidate != suite.dir: mxDir = mx._is_suite_dir(candidate) if mxDir: matches = [s for s in mx.suites() if s.dir == candidate] if len(matches) == 0: suites_in_repo.append( mx.SourceSuite(mxDir, primary=False, load=False)) else: suites_in_repo.append(matches[0]) for suite_in_repo in suites_in_repo: if suite_in_repo.vc_dir and suite_in_repo.dir != suite_in_repo.vc_dir: mirror = join(workDir, basename(suite_in_repo.vc_dir), suite_in_repo.name) else: mirror = join(workDir, suite_in_repo.name) if exists(mirror): mx.rmtree(mirror) output_root = mx._safe_path(suite_in_repo.get_output_root()) def ignore_output_root(d, names): mx.log('Copying ' + d) if d == os.path.dirname(output_root): mx.log('Omitting ' + output_root) return [os.path.basename(output_root)] return [] mx.copytree(suite_in_repo.dir, mirror, ignore=ignore_output_root, symlinks=True) targetDir = None for repoUrl in repoUrls: # Deduce a target name from the target URL url = _urllib_parse.urlparse(repoUrl) targetName = url.path if targetName.rfind('/') != -1: targetName = targetName[targetName.rfind('/') + 1:] if targetName.endswith('.git'): targetName = targetName[0:-len('.git')] repoWorkDir = join(workDir, targetName) git = mx.GitConfig() if exists(repoWorkDir): git.pull(repoWorkDir) else: git.clone(repoUrl, repoWorkDir) # See if there's a matching (non-master) branch and use it if there is if not branch: branch = git.git_command( suite.dir, ['rev-parse', '--abbrev-ref', 'HEAD']).strip() if branch != 'master': git.git_command(repoWorkDir, ['checkout', branch], abortOnError=False) if not targetDir: targetDir = repoWorkDir assert not isabs(relTargetSuiteDir) targetSuiteDir = join(targetDir, relTargetSuiteDir) assert targetSuiteDir.startswith(targetDir) mxpy = None if suite != mx._mx_suite else join(mirror, 'mx.py') for command in mxCommands: mx.logv('[running "mx ' + ' '.join(command) + '" in ' + targetSuiteDir + ']') mx.run_mx(command, targetSuiteDir, mxpy=mxpy)