def is_repo(path): """Returns a boolean if the path is a Git repo.""" try: git("-C", path, "rev-parse", "--is-inside-work-tree") except sh.ErrorReturnCode: return False return True
def fix_head_for_github( base_commit_ref: Optional[str] = None, head_ref: Optional[str] = None, ) -> Iterator[Optional[str]]: """ GHA can checkout the incorrect commit for a PR (it will create a fake merge commit), so we need to reset the head to the actual PR branch head before continuing. Note that this code is written in a generic manner, so that it becomes a no-op when the CI system has not artifically altered the HEAD ref. :return: The baseline ref as a commit hash """ stashed_rev: Optional[str] = None base_ref: Optional[str] = base_commit_ref if get_git_repo() is None: yield base_ref return if base_ref: # Preserve location of head^ after we possibly change location below base_ref = git(["rev-parse", base_ref]).stdout.decode("utf-8").rstrip() if head_ref: stashed_rev = git(["branch", "--show-current"]).stdout.decode("utf-8").rstrip() if not stashed_rev: stashed_rev = git(["rev-parse", "HEAD"]).stdout.decode("utf-8").rstrip() click.echo(f"| not on head ref {head_ref}; checking that out now...", err=True) git.checkout([head_ref]) try: if base_ref is not None: merge_base = git("merge-base", base_ref, "HEAD").rstrip() # fmt:off click.echo("| reporting findings introduced by these commits:", err=True) print_git_log(f"{merge_base}..HEAD") if merge_base != git("rev-parse", base_ref).rstrip(): click.echo( "| also reporting findings fixed by these commits from the baseline branch:", err=True) print_git_log(f"{merge_base}..{base_ref}") click.echo("| to exclude these latter commits, run with", err=True) click.echo( f"| --baseline-ref $(git merge-base {base_commit_ref} HEAD)", err=True) # fmt: on yield base_ref finally: if stashed_rev is not None: click.echo(f"| returning to original head revision {stashed_rev}", err=True) git.checkout([stashed_rev])
def _cut_stable_release(layer_list, charm_list, ancillary_list, filter_by_tag, dry_run): """This will merge each layers master onto the stable branches. PLEASE NOTE: This step should come after each stable branch has been tagged and references a current stable bundle revision. layer_list: YAML spec containing git repos and their upstream/downstream properties charm_list: YAML spec containing git repos and their upstream/downstream properties """ layer_list = yaml.safe_load(Path(layer_list).read_text(encoding="utf8")) charm_list = yaml.safe_load(Path(charm_list).read_text(encoding="utf8")) ancillary_list = yaml.safe_load( Path(ancillary_list).read_text(encoding="utf8")) new_env = os.environ.copy() for layer_map in layer_list + charm_list + ancillary_list: for layer_name, repos in layer_map.items(): downstream = repos["downstream"] if not repos.get("needs_stable", True): continue tags = repos.get("tags", None) if tags: if not any(match in filter_by_tag for match in tags): continue log.info( f"Releasing :: {layer_name:^35} :: from: master to: stable") if not dry_run: downstream = f"https://{new_env['CDKBOT_GH_USR']}:{new_env['CDKBOT_GH_PSW']}@github.com/{downstream}" identifier = str(uuid.uuid4()) os.makedirs(identifier) for line in git.clone(downstream, identifier, _iter=True): log.info(line) git_rev_master = git("rev-parse", "origin/master", _cwd=identifier).stdout.decode() git_rev_stable = git("rev-parse", "origin/stable", _cwd=identifier).stdout.decode() if git_rev_master == git_rev_stable: log.info( f"Skipping :: {layer_name:^35} :: master == stable") continue git.config("user.email", "*****@*****.**", _cwd=identifier) git.config("user.name", "cdkbot", _cwd=identifier) git.config("--global", "push.default", "simple") git.checkout("-f", "stable", _cwd=identifier) git.merge("master", "--no-ff", _cwd=identifier) for line in git.push("origin", "stable", _cwd=identifier, _iter=True): log.info(line)
def base_commit_ref(self) -> Optional[str]: target_branch = os.getenv("CI_MERGE_REQUEST_TARGET_BRANCH_NAME") if not target_branch: return None head_sha = git("rev-parse", "HEAD").stdout.strip() git.fetch(self._get_remote_url(), target_branch) base_sha = (git("merge-base", "--all", head_sha, "FETCH_HEAD").stdout.decode().strip()) return base_sha
def git_set_email_and_name(email, name): text_helper.print_info('[GIT] Seteando email y usuario') try: git('config', '--global', 'user.email', email) git('config', '--global', 'user.name', name) except: text_helper.print_error( '[GIT] Hubo un error seteando el mail y usuario') return False text_helper.print_success( '[GIT] Mail y usario seteados satisfactoriamente')
def commit(self, short=False): """Commit hash of downstream repo.""" if not Path(self.src_path).exists(): raise BuildException(f"Could not locate {self.src_path}") if short: git_commit = git("describe", dirty=True, always=True, _cwd=self.src_path) else: git_commit = git("rev-parse", "HEAD", _cwd=self.src_path) return git_commit.stdout.decode().strip()
def git_set_credentials(password, username): text_helper.print_info('[GIT] Seteando credenciales') try: git('config', '--global', 'credential.https://github.com.username', username) git('config', '--global', 'credential.https://github.com.password', password) except: text_helper.print_error( '[GIT] Hubo un error seteando las credenciales') return False text_helper.print_success('[GIT] Credenciales seteadas satisfactoriamente')
def git_clone(repo): text_helper.print_info('[GIT] Clonando: ' + utils.take_repo_name(repo)) try: git('clone', repo, './Workspace/' + utils.take_repo_name(repo), _cwd=pathlib.Path.home()) except: text_helper.print_error( '[GIT] Hubo un error al intentar clonar el repositorio') return False text_helper.print_success('[GIT] Se clono satisfactoriamente: ' + utils.take_repo_name(repo))
def base_commit_ref(self) -> Optional[str]: if self.event_name == "pull_request": pr_base = self.glom_event(T["pull_request"]["base"]["sha"]) # The pull request "base" that GitHub sends us is not necessarily the merge base, # so we need to get the merge-base from Git return git("merge-base", pr_base, "HEAD").stdout.decode().strip() return None
def _find_branchoff_point(self, attempt_count: int = 0) -> str: fetch_depth = 4 ** attempt_count # fetch 4, 16, 64, 256, 1024, ... if attempt_count >= self.MAX_FETCH_ATTEMPT_COUNT: # get all commits on last try fetch_depth = 2 ** 31 - 1 # git expects a signed 32-bit integer if attempt_count: # skip fetching on first try debug_echo( f"fetching {fetch_depth} commits to find branch-off point of pull request" ) git.fetch("origin", "--depth", fetch_depth, self.base_branch_tip) git.fetch("origin", "--depth", fetch_depth, self.head_ref) try: # check if both branches connect to the yet-unknown branch-off point now process = git("merge-base", self.base_branch_tip, self.head_ref) except sh.ErrorReturnCode as error: output = error.stderr.decode().strip() if ( output # output is empty when unable to find branch-off point and "Not a valid " not in output # the error when a ref is missing ): exit_with_sh_error(error) if attempt_count >= self.MAX_FETCH_ATTEMPT_COUNT: raise ActionFailure( "Could not find branch-off point between " f"the baseline tip {self.base_branch_tip} and current head '{self.head_ref}' " ) return self._find_branchoff_point(attempt_count + 1) else: return process.stdout.decode().strip()
def get_version_info(): version = None sha = git("rev-parse", "--short", "HEAD").stdout.decode("utf-8") try: version = git("describe", "--tags", "--exact-match").stdout.decode("utf-8").strip() except sh.ErrorReturnCode_128: # No exact match pass if "GITHUB_SHA" in os.environ: sha = os.environ["GITHUB_SHA"] if not version: version = "{}-{}".format(date.today().strftime("%Y%m%d"), sha[:7]) return sha, version
def commit(self): """Commit hash of downstream repo""" if not Path(self.src_path).exists(): raise BuildException(f"Could not locate {self.src_path}") git_commit = git("rev-parse", "HEAD", _cwd=self.src_path) return git_commit.stdout.decode().strip()
def get_version_info(): version = None sha = git("rev-parse", "--short", "HEAD").stdout.decode("utf-8") try: version = git("describe", "--tags", "--exact-match").stdout.decode("utf-8").strip() except sh.ErrorReturnCode_128: # No exact match pass if "TRAVIS" in os.environ and os.environ["TRAVIS"] == "true": sha = os.environ["TRAVIS_COMMIT"] if os.environ["TRAVIS_PULL_REQUEST"] != "false": sha = os.environ["TRAVIS_PULL_REQUEST_SHA"] if not version: version="{}-{}".format(date.today().strftime("%Y%m%d"), sha[:7]) return sha, version
def is_commit_pullable(ref): try: # git diff-tree --no-commit-id --name-only -r <> files = git('diff-tree', '--no-commit-id', '--name-only', '-r', ref.strip()) files = str(files).split() return "noexec" not in files except GitError as e: log_exception(e) return False
def _baseline_context(self) -> Iterator[None]: """ Runs a block of code on files from the current branch HEAD. :raises ActionFailure: If git cannot detect a HEAD commit :raises ActionFailure: If unmerged files are detected """ repo = get_git_repo() if not repo: yield return self._abort_on_pending_changes() self._abort_on_conflicting_untracked_paths() current_tree = git("write-tree").stdout.decode().strip() try: for a in self._status.added: a.unlink() git.checkout(self._base_commit, "--", ".") yield finally: # git checkout will fail if the checked-out index deletes all files in the repo # In this case, we still want to continue without error. # Note that we have no good way of detecting this issue without inspecting the checkout output # message, which means we are fragile with respect to git version here. try: git.checkout(current_tree.strip(), "--", ".") except sh.ErrorReturnCode as error: output = error.stderr.decode() if (output and len(output) >= 2 and "pathspec '.' did not match any file(s) known to git" in output.strip()): debug_echo( "Restoring git index failed due to total repository deletion; skipping checkout" ) else: raise ActionFailure( f"Fatal error restoring Git state; please restore your repository state manually:\n{output}" ) if self._status.removed: # Need to check if file exists since it is possible file was deleted # in both the base and head. Only call if there are files to delete to_remove = [r for r in self._status.removed if r.exists()] if to_remove: git.rm("-f", *(str(r) for r in to_remove))
def pull_layers(self): """ clone all downstream layers to be processed locally when doing charm builds """ if self.rebuild_cache: click.echo("- rebuild cache triggered, cleaning out cache.") shutil.rmtree(str(self.layers_dir)) shutil.rmtree(str(self.interfaces_dir)) os.mkdir(str(self.layers_dir)) os.mkdir(str(self.interfaces_dir)) layers_to_pull = [] for layer_map in self.layers: layer_name = list(layer_map.keys())[0] if layer_name == "layer:index": continue layers_to_pull.append(layer_name) pool = ThreadPool() pool.map(self.download, layers_to_pull) self.db["pull_layer_manifest"] = [] _paths_to_process = { "layer": glob("{}/*".format(str(self.layers_dir))), "interface": glob("{}/*".format(str(self.interfaces_dir))), } for prefix, paths in _paths_to_process.items(): for _path in paths: build_path = _path if not build_path: raise BuildException( f"Could not determine build path for {_path}") git.checkout(self.layer_branch, _cwd=build_path) layer_manifest = { "rev": git("rev-parse", "HEAD", _cwd=build_path).stdout.decode().strip(), "url": f"{prefix}:{Path(build_path).stem}", } self.db["pull_layer_manifest"].append(layer_manifest) click.echo( f"- {layer_manifest['url']} at commit: {layer_manifest['rev']}" )
def _baseline_context(self) -> Iterator[None]: """ Runs a block of code on files from the current branch HEAD. :raises RuntimeError: If git cannot detect a HEAD commit :raises RuntimeError: If unmerged files are detected """ repo = get_git_repo() if not repo: yield return self._abort_if_dirty() current_tree = git("write-tree").stdout.decode().strip() try: for a in self._status.added: a.unlink() git.checkout(self._base_commit, "--", ".") yield finally: # git checkout will fail if the checked-out index deletes all files in the repo # In this case, we still want to continue without error. # Note that we have no good way of detecting this issue without inspecting the checkout output # message, which means we are fragile with respect to git version here. try: git.checkout(current_tree.strip(), "--", ".") except sh.ErrorReturnCode as error: output = error.stderr.decode() if (output and len(output) >= 2 and "pathspec '.' did not match any file(s) known to git" in output.strip()): debug_echo( "Restoring git index failed due to total repository deletion; skipping checkout" ) else: raise error if self._status.removed: git.rm("-f", *(str(r) for r in self._status.removed))
def update_previous_head_commit(): # Update the latest noted commit now that patch notes have been created latest_patched_commit = git("rev-parse", "--short", "HEAD").rstrip("\n") commit_ledger = open(COMMIT_LEDGER, "w") commit_ledger.write(latest_patched_commit) commit_ledger.close()
# The master toctree document. master_doc = 'index' # General information about the project. project = 'treelite' copyright = '2017, DMLC. All rights reserved.' # pylint: disable=W0622 author = 'DMLC developers' # Read version info with open('../VERSION', 'r') as f: VERSION = f.readlines()[0] version = VERSION release = VERSION git_tag = str(git.tag('-l', '--points-at', 'HEAD')).rstrip('\n') git_commit = str(git('rev-parse', '--short', 'HEAD')).rstrip('\n') if git_tag: # tag exists intro_landing_release = 'You are currently browsing the documentation of ' +\ 'a stable version of treelite: **{}**.'.format(git_tag) nav_ver = git_tag else: # tag does not exist; part of "latest" intro_landing_release = 'You are currently browsing the documentation of ' +\ 'a development version of treelite: '+\ 'commit **{}**.'.format(git_commit) nav_ver = 'dev, commit {}'.format(git_commit) # Make dropdown menu for version selection with open('_templates/logo-text.html', 'w') as f: f.write(r""" <a href="{{ homepage() }}" class="text-logo"> {{ theme_project_nav_name or shorttitle }}
def find_repo_toplevel(path): """Return the repository's top level directory (the root of the repo).""" if not is_repo(path): return None return pathlib.Path( git("-C", path, "rev-parse", "--show-toplevel").strip())
book_library = os.environ['HOME'] + '/Google Drive/GitBooks/books' elif book_library.lower().startswith('d'): book_library = os.environ['HOME'] + '/Dropbox/GitBooks/books' else: print("Please specify 'Google Drive' or 'Dropbox'") prompt_booklib() prompt_booklib() print("Creating new book on GitHub") new_repo = g.get_user().create_repo(book_name, description=book_name) print("Created book at " + new_repo.ssh_url) print("Cloning prototype") git('clone', '--bare', PROTOTYPE_URL) os.chdir(PROTOTYPE_NAME + '.git') sh.git('push', '--mirror', new_repo.ssh_url, _in=sys.stdin, _out=sys.stdout) print("Removing Temporary Files") os.chdir(original_working_directory) sh.rm('-rf', 'gitbook-prototype.git') print("Adding repo to library") os.chdir(book_library) git('clone', new_repo.ssh_url) os.chdir(new_repo.name) print("Installing requirements. This may take a minute.") sh.gitbook('install', _out=sys.stdout)
def is_same_commit(head_a, head_b): sha_a = str(git('rev-parse', head_a)).strip() sha_b = str(git('rev-parse', head_b)).strip() return sha_a == sha_b
def _cut_stable_release(layer_list, charm_list, ancillary_list, filter_by_tag, dry_run): """This will merge each layers master onto the stable branches. PLEASE NOTE: This step should come after each stable branch has been tagged and references a current stable bundle revision. layer_list: YAML spec containing git repos and their upstream/downstream properties charm_list: YAML spec containing git repos and their upstream/downstream properties """ layer_list = yaml.safe_load(Path(layer_list).read_text(encoding="utf8")) charm_list = yaml.safe_load(Path(charm_list).read_text(encoding="utf8")) ancillary_list = yaml.safe_load( Path(ancillary_list).read_text(encoding="utf8")) new_env = os.environ.copy() failed_to_release = [] for layer_map in layer_list + charm_list + ancillary_list: for layer_name, repos in layer_map.items(): downstream = repos["downstream"] if not repos.get("needs_stable", True): continue tags = repos.get("tags", None) if tags: if not any(match in filter_by_tag for match in tags): continue auth = (new_env.get("CDKBOT_GH_USR"), new_env.get("CDKBOT_GH_PSW")) default_branch = repos.get("branch") or default_gh_branch( downstream, auth=auth) log.info( f"Releasing :: {layer_name:^35} :: from: {default_branch} to: stable" ) downstream = f"https://{':'.join(auth)}@github.com/{downstream}" identifier = str(uuid.uuid4()) os.makedirs(identifier) for line in git.clone(downstream, identifier, _iter=True): log.info(line) git_rev_default = (git("rev-parse", f"origin/{default_branch}", _cwd=identifier).stdout.decode().strip()) git_rev_stable = (git("rev-parse", "origin/stable", _cwd=identifier).stdout.decode().strip()) if git_rev_default == git_rev_stable: log.info( f"Skipping :: {layer_name:^35} :: {default_branch} == stable" ) continue log.info( f"Commits :: {layer_name:^35} :: {default_branch} != stable") log.info(f" {default_branch:10}= {git_rev_default:32}") log.info(f" {'stable':10}= {git_rev_stable:32}") for line in git("rev-list", f"origin/stable..origin/{default_branch}", _cwd=identifier): for line in git.show( "--format=%h %an '%s' %cr", "--no-patch", line.strip(), _cwd=identifier, ): log.info(" " + line.strip()) if not dry_run: git.config("user.email", "*****@*****.**", _cwd=identifier) git.config("user.name", "cdkbot", _cwd=identifier) git.config("--global", "push.default", "simple") git.checkout("-f", "stable", _cwd=identifier) git.reset(default_branch, _cwd=identifier) for line in git.push("origin", "stable", "-f", _cwd=identifier, _iter=True): log.info(line)
def _fix_head_for_github( base_ref_name: Optional[str] = None, head_ref: Optional[str] = None, ) -> Iterator[Optional[str]]: """ GHA can checkout the incorrect commit for a PR (it will create a fake merge commit), so we need to reset the head to the actual PR branch head before continuing. Note that this code is written in a generic manner, so that it becomes a no-op when the CI system has not artifically altered the HEAD ref. :return: The baseline ref as a commit hash """ debug_echo( f"Called _fix_head_for_github with base_ref_name: {base_ref_name} head_ref: {head_ref}" ) stashed_rev: Optional[str] = None base_ref: Optional[str] = base_ref_name if get_git_repo() is None: debug_echo("Yielding base_ref since get_git_repo was None") yield base_ref return if base_ref: # Preserve location of head^ after we possibly change location below try: debug_echo(f"Calling git rev-parse {base_ref}") process = git(["rev-parse", base_ref]) base_ref = process.stdout.decode("utf-8").rstrip() except sh.ErrorReturnCode as ex: raise ActionFailure( f"There is a problem with your git project:{ex}") if head_ref: debug_echo("Calling git branch --show-current") stashed_rev = git(["branch", "--show-current"]).stdout.decode("utf-8").rstrip() debug_echo(f"stashed_rev: {stashed_rev}") if not stashed_rev: debug_echo("Calling git rev-parse HEAD") rev_parse = git(["rev-parse", "HEAD"]) debug_echo(rev_parse.stderr.decode("utf-8").rstrip()) stashed_rev = rev_parse.stdout.decode("utf-8").rstrip() debug_echo(f"stashed_rev: {stashed_rev}") click.echo(f"| not on head ref {head_ref}; checking that out now...", err=True) git.checkout([head_ref], _timeout=GIT_SH_TIMEOUT, _out=debug_echo, _err=debug_echo) debug_echo(f"checked out {head_ref}") try: if base_ref is not None: merge_base = git("merge-base", base_ref, "HEAD").rstrip() # fmt:off click.echo("| reporting findings introduced by these commits:", err=True) print_git_log(f"{merge_base}..HEAD") if merge_base != git("rev-parse", base_ref).rstrip(): click.echo( "| also reporting findings fixed by these commits from the baseline branch:", err=True) print_git_log(f"{merge_base}..{base_ref}") click.echo("| to exclude these latter commits, run with", err=True) click.echo( f"| --baseline-ref $(git merge-base {base_ref_name} HEAD)", err=True) # fmt: on debug_echo(f"yielding {base_ref}") yield base_ref finally: if stashed_rev is not None: click.echo(f"| returning to original head revision {stashed_rev}", err=True) git.checkout([stashed_rev], _timeout=GIT_SH_TIMEOUT)