def is_repo(path):
"""Returns a boolean if the path is a Git repo."""
try:
git("-C", path, "rev-parse", "--is-inside-work-tree")
except sh.ErrorReturnCode:
return False
return True
def fix_head_for_github(
base_commit_ref: Optional[str] = None,
head_ref: Optional[str] = None,
) -> Iterator[Optional[str]]:
"""
GHA can checkout the incorrect commit for a PR (it will create a fake merge commit),
so we need to reset the head to the actual PR branch head before continuing.
Note that this code is written in a generic manner, so that it becomes a no-op when
the CI system has not artifically altered the HEAD ref.
:return: The baseline ref as a commit hash
"""
stashed_rev: Optional[str] = None
base_ref: Optional[str] = base_commit_ref
if get_git_repo() is None:
yield base_ref
return
if base_ref:
# Preserve location of head^ after we possibly change location below
base_ref = git(["rev-parse", base_ref]).stdout.decode("utf-8").rstrip()
if head_ref:
stashed_rev = git(["branch",
"--show-current"]).stdout.decode("utf-8").rstrip()
if not stashed_rev:
stashed_rev = git(["rev-parse",
"HEAD"]).stdout.decode("utf-8").rstrip()
click.echo(f"| not on head ref {head_ref}; checking that out now...",
err=True)
git.checkout([head_ref])
try:
if base_ref is not None:
merge_base = git("merge-base", base_ref, "HEAD").rstrip()
# fmt:off
click.echo("| reporting findings introduced by these commits:",
err=True)
print_git_log(f"{merge_base}..HEAD")
if merge_base != git("rev-parse", base_ref).rstrip():
click.echo(
"| also reporting findings fixed by these commits from the baseline branch:",
err=True)
print_git_log(f"{merge_base}..{base_ref}")
click.echo("| to exclude these latter commits, run with",
err=True)
click.echo(
f"| --baseline-ref $(git merge-base {base_commit_ref} HEAD)",
err=True)
# fmt: on
yield base_ref
finally:
if stashed_rev is not None:
click.echo(f"| returning to original head revision {stashed_rev}",
err=True)
git.checkout([stashed_rev])
def _cut_stable_release(layer_list, charm_list, ancillary_list, filter_by_tag,
dry_run):
"""This will merge each layers master onto the stable branches.
PLEASE NOTE: This step should come after each stable branch has been tagged
and references a current stable bundle revision.
layer_list: YAML spec containing git repos and their upstream/downstream properties
charm_list: YAML spec containing git repos and their upstream/downstream properties
"""
layer_list = yaml.safe_load(Path(layer_list).read_text(encoding="utf8"))
charm_list = yaml.safe_load(Path(charm_list).read_text(encoding="utf8"))
ancillary_list = yaml.safe_load(
Path(ancillary_list).read_text(encoding="utf8"))
new_env = os.environ.copy()
for layer_map in layer_list + charm_list + ancillary_list:
for layer_name, repos in layer_map.items():
downstream = repos["downstream"]
if not repos.get("needs_stable", True):
continue
tags = repos.get("tags", None)
if tags:
if not any(match in filter_by_tag for match in tags):
continue
log.info(
f"Releasing :: {layer_name:^35} :: from: master to: stable")
if not dry_run:
downstream = f"https://{new_env['CDKBOT_GH_USR']}:{new_env['CDKBOT_GH_PSW']}@github.com/{downstream}"
identifier = str(uuid.uuid4())
os.makedirs(identifier)
for line in git.clone(downstream, identifier, _iter=True):
log.info(line)
git_rev_master = git("rev-parse",
"origin/master",
_cwd=identifier).stdout.decode()
git_rev_stable = git("rev-parse",
"origin/stable",
_cwd=identifier).stdout.decode()
if git_rev_master == git_rev_stable:
log.info(
f"Skipping :: {layer_name:^35} :: master == stable")
continue
git.config("user.email",
"*****@*****.**",
_cwd=identifier)
git.config("user.name", "cdkbot", _cwd=identifier)
git.config("--global", "push.default", "simple")
git.checkout("-f", "stable", _cwd=identifier)
git.merge("master", "--no-ff", _cwd=identifier)
for line in git.push("origin",
"stable",
_cwd=identifier,
_iter=True):
log.info(line)
def base_commit_ref(self) -> Optional[str]:
target_branch = os.getenv("CI_MERGE_REQUEST_TARGET_BRANCH_NAME")
if not target_branch:
return None
head_sha = git("rev-parse", "HEAD").stdout.strip()
git.fetch(self._get_remote_url(), target_branch)
base_sha = (git("merge-base", "--all", head_sha,
"FETCH_HEAD").stdout.decode().strip())
return base_sha
def git_set_email_and_name(email, name):
text_helper.print_info('[GIT] Seteando email y usuario')
try:
git('config', '--global', 'user.email', email)
git('config', '--global', 'user.name', name)
except:
text_helper.print_error(
'[GIT] Hubo un error seteando el mail y usuario')
return False
text_helper.print_success(
'[GIT] Mail y usario seteados satisfactoriamente')
def commit(self, short=False):
"""Commit hash of downstream repo."""
if not Path(self.src_path).exists():
raise BuildException(f"Could not locate {self.src_path}")
if short:
git_commit = git("describe",
dirty=True,
always=True,
_cwd=self.src_path)
else:
git_commit = git("rev-parse", "HEAD", _cwd=self.src_path)
return git_commit.stdout.decode().strip()
def git_set_credentials(password, username):
text_helper.print_info('[GIT] Seteando credenciales')
try:
git('config', '--global', 'credential.https://github.com.username',
username)
git('config', '--global', 'credential.https://github.com.password',
password)
except:
text_helper.print_error(
'[GIT] Hubo un error seteando las credenciales')
return False
text_helper.print_success('[GIT] Credenciales seteadas satisfactoriamente')
def git_clone(repo):
text_helper.print_info('[GIT] Clonando: ' + utils.take_repo_name(repo))
try:
git('clone',
repo,
'./Workspace/' + utils.take_repo_name(repo),
_cwd=pathlib.Path.home())
except:
text_helper.print_error(
'[GIT] Hubo un error al intentar clonar el repositorio')
return False
text_helper.print_success('[GIT] Se clono satisfactoriamente: ' +
utils.take_repo_name(repo))
def base_commit_ref(self) -> Optional[str]:
if self.event_name == "pull_request":
pr_base = self.glom_event(T["pull_request"]["base"]["sha"])
# The pull request "base" that GitHub sends us is not necessarily the merge base,
# so we need to get the merge-base from Git
return git("merge-base", pr_base, "HEAD").stdout.decode().strip()
return None
def _find_branchoff_point(self, attempt_count: int = 0) -> str:
fetch_depth = 4 ** attempt_count # fetch 4, 16, 64, 256, 1024, ...
if attempt_count >= self.MAX_FETCH_ATTEMPT_COUNT: # get all commits on last try
fetch_depth = 2 ** 31 - 1 # git expects a signed 32-bit integer
if attempt_count: # skip fetching on first try
debug_echo(
f"fetching {fetch_depth} commits to find branch-off point of pull request"
)
git.fetch("origin", "--depth", fetch_depth, self.base_branch_tip)
git.fetch("origin", "--depth", fetch_depth, self.head_ref)
try: # check if both branches connect to the yet-unknown branch-off point now
process = git("merge-base", self.base_branch_tip, self.head_ref)
except sh.ErrorReturnCode as error:
output = error.stderr.decode().strip()
if (
output # output is empty when unable to find branch-off point
and "Not a valid " not in output # the error when a ref is missing
):
exit_with_sh_error(error)
if attempt_count >= self.MAX_FETCH_ATTEMPT_COUNT:
raise ActionFailure(
"Could not find branch-off point between "
f"the baseline tip {self.base_branch_tip} and current head '{self.head_ref}' "
)
return self._find_branchoff_point(attempt_count + 1)
else:
return process.stdout.decode().strip()
def get_version_info():
version = None
sha = git("rev-parse", "--short", "HEAD").stdout.decode("utf-8")
try:
version = git("describe", "--tags", "--exact-match").stdout.decode("utf-8").strip()
except sh.ErrorReturnCode_128:
# No exact match
pass
if "GITHUB_SHA" in os.environ:
sha = os.environ["GITHUB_SHA"]
if not version:
version = "{}-{}".format(date.today().strftime("%Y%m%d"), sha[:7])
return sha, version
def commit(self):
"""Commit hash of downstream repo"""
if not Path(self.src_path).exists():
raise BuildException(f"Could not locate {self.src_path}")
git_commit = git("rev-parse", "HEAD", _cwd=self.src_path)
return git_commit.stdout.decode().strip()
def get_version_info():
version = None
sha = git("rev-parse", "--short", "HEAD").stdout.decode("utf-8")
try:
version = git("describe", "--tags", "--exact-match").stdout.decode("utf-8").strip()
except sh.ErrorReturnCode_128:
# No exact match
pass
if "TRAVIS" in os.environ and os.environ["TRAVIS"] == "true":
sha = os.environ["TRAVIS_COMMIT"]
if os.environ["TRAVIS_PULL_REQUEST"] != "false":
sha = os.environ["TRAVIS_PULL_REQUEST_SHA"]
if not version:
version="{}-{}".format(date.today().strftime("%Y%m%d"), sha[:7])
return sha, version
def is_commit_pullable(ref):
try:
# git diff-tree --no-commit-id --name-only -r <>
files = git('diff-tree', '--no-commit-id', '--name-only', '-r',
ref.strip())
files = str(files).split()
return "noexec" not in files
except GitError as e:
log_exception(e)
return False
def _baseline_context(self) -> Iterator[None]:
"""
Runs a block of code on files from the current branch HEAD.
:raises ActionFailure: If git cannot detect a HEAD commit
:raises ActionFailure: If unmerged files are detected
"""
repo = get_git_repo()
if not repo:
yield
return
self._abort_on_pending_changes()
self._abort_on_conflicting_untracked_paths()
current_tree = git("write-tree").stdout.decode().strip()
try:
for a in self._status.added:
a.unlink()
git.checkout(self._base_commit, "--", ".")
yield
finally:
# git checkout will fail if the checked-out index deletes all files in the repo
# In this case, we still want to continue without error.
# Note that we have no good way of detecting this issue without inspecting the checkout output
# message, which means we are fragile with respect to git version here.
try:
git.checkout(current_tree.strip(), "--", ".")
except sh.ErrorReturnCode as error:
output = error.stderr.decode()
if (output and len(output) >= 2 and
"pathspec '.' did not match any file(s) known to git"
in output.strip()):
debug_echo(
"Restoring git index failed due to total repository deletion; skipping checkout"
)
else:
raise ActionFailure(
f"Fatal error restoring Git state; please restore your repository state manually:\n{output}"
)
if self._status.removed:
# Need to check if file exists since it is possible file was deleted
# in both the base and head. Only call if there are files to delete
to_remove = [r for r in self._status.removed if r.exists()]
if to_remove:
git.rm("-f", *(str(r) for r in to_remove))
def pull_layers(self):
""" clone all downstream layers to be processed locally when doing charm builds
"""
if self.rebuild_cache:
click.echo("- rebuild cache triggered, cleaning out cache.")
shutil.rmtree(str(self.layers_dir))
shutil.rmtree(str(self.interfaces_dir))
os.mkdir(str(self.layers_dir))
os.mkdir(str(self.interfaces_dir))
layers_to_pull = []
for layer_map in self.layers:
layer_name = list(layer_map.keys())[0]
if layer_name == "layer:index":
continue
layers_to_pull.append(layer_name)
pool = ThreadPool()
pool.map(self.download, layers_to_pull)
self.db["pull_layer_manifest"] = []
_paths_to_process = {
"layer": glob("{}/*".format(str(self.layers_dir))),
"interface": glob("{}/*".format(str(self.interfaces_dir))),
}
for prefix, paths in _paths_to_process.items():
for _path in paths:
build_path = _path
if not build_path:
raise BuildException(
f"Could not determine build path for {_path}")
git.checkout(self.layer_branch, _cwd=build_path)
layer_manifest = {
"rev":
git("rev-parse", "HEAD",
_cwd=build_path).stdout.decode().strip(),
"url":
f"{prefix}:{Path(build_path).stem}",
}
self.db["pull_layer_manifest"].append(layer_manifest)
click.echo(
f"- {layer_manifest['url']} at commit: {layer_manifest['rev']}"
)
def _baseline_context(self) -> Iterator[None]:
"""
Runs a block of code on files from the current branch HEAD.
:raises RuntimeError: If git cannot detect a HEAD commit
:raises RuntimeError: If unmerged files are detected
"""
repo = get_git_repo()
if not repo:
yield
return
self._abort_if_dirty()
current_tree = git("write-tree").stdout.decode().strip()
try:
for a in self._status.added:
a.unlink()
git.checkout(self._base_commit, "--", ".")
yield
finally:
# git checkout will fail if the checked-out index deletes all files in the repo
# In this case, we still want to continue without error.
# Note that we have no good way of detecting this issue without inspecting the checkout output
# message, which means we are fragile with respect to git version here.
try:
git.checkout(current_tree.strip(), "--", ".")
except sh.ErrorReturnCode as error:
output = error.stderr.decode()
if (output and len(output) >= 2 and
"pathspec '.' did not match any file(s) known to git"
in output.strip()):
debug_echo(
"Restoring git index failed due to total repository deletion; skipping checkout"
)
else:
raise error
if self._status.removed:
git.rm("-f", *(str(r) for r in self._status.removed))
def update_previous_head_commit():
# Update the latest noted commit now that patch notes have been created
latest_patched_commit = git("rev-parse", "--short", "HEAD").rstrip("\n")
commit_ledger = open(COMMIT_LEDGER, "w")
commit_ledger.write(latest_patched_commit)
commit_ledger.close()
# The master toctree document.
master_doc = 'index'
# General information about the project.
project = 'treelite'
copyright = '2017, DMLC. All rights reserved.' # pylint: disable=W0622
author = 'DMLC developers'
# Read version info
with open('../VERSION', 'r') as f:
VERSION = f.readlines()[0]
version = VERSION
release = VERSION
git_tag = str(git.tag('-l', '--points-at', 'HEAD')).rstrip('\n')
git_commit = str(git('rev-parse', '--short', 'HEAD')).rstrip('\n')
if git_tag: # tag exists
intro_landing_release = 'You are currently browsing the documentation of ' +\
'a stable version of treelite: **{}**.'.format(git_tag)
nav_ver = git_tag
else: # tag does not exist; part of "latest"
intro_landing_release = 'You are currently browsing the documentation of ' +\
'a development version of treelite: '+\
'commit **{}**.'.format(git_commit)
nav_ver = 'dev, commit {}'.format(git_commit)
# Make dropdown menu for version selection
with open('_templates/logo-text.html', 'w') as f:
f.write(r"""
<a href="{{ homepage() }}" class="text-logo">
{{ theme_project_nav_name or shorttitle }}
def find_repo_toplevel(path):
"""Return the repository's top level directory (the root of the repo)."""
if not is_repo(path):
return None
return pathlib.Path(
git("-C", path, "rev-parse", "--show-toplevel").strip())
book_library = os.environ['HOME'] + '/Google Drive/GitBooks/books'
elif book_library.lower().startswith('d'):
book_library = os.environ['HOME'] + '/Dropbox/GitBooks/books'
else:
print("Please specify 'Google Drive' or 'Dropbox'")
prompt_booklib()
prompt_booklib()
print("Creating new book on GitHub")
new_repo = g.get_user().create_repo(book_name, description=book_name)
print("Created book at " + new_repo.ssh_url)
print("Cloning prototype")
git('clone', '--bare', PROTOTYPE_URL)
os.chdir(PROTOTYPE_NAME + '.git')
sh.git('push', '--mirror', new_repo.ssh_url, _in=sys.stdin, _out=sys.stdout)
print("Removing Temporary Files")
os.chdir(original_working_directory)
sh.rm('-rf', 'gitbook-prototype.git')
print("Adding repo to library")
os.chdir(book_library)
git('clone', new_repo.ssh_url)
os.chdir(new_repo.name)
print("Installing requirements. This may take a minute.")
sh.gitbook('install', _out=sys.stdout)
def is_same_commit(head_a, head_b):
sha_a = str(git('rev-parse', head_a)).strip()
sha_b = str(git('rev-parse', head_b)).strip()
return sha_a == sha_b
def _cut_stable_release(layer_list, charm_list, ancillary_list, filter_by_tag,
dry_run):
"""This will merge each layers master onto the stable branches.
PLEASE NOTE: This step should come after each stable branch has been tagged
and references a current stable bundle revision.
layer_list: YAML spec containing git repos and their upstream/downstream properties
charm_list: YAML spec containing git repos and their upstream/downstream properties
"""
layer_list = yaml.safe_load(Path(layer_list).read_text(encoding="utf8"))
charm_list = yaml.safe_load(Path(charm_list).read_text(encoding="utf8"))
ancillary_list = yaml.safe_load(
Path(ancillary_list).read_text(encoding="utf8"))
new_env = os.environ.copy()
failed_to_release = []
for layer_map in layer_list + charm_list + ancillary_list:
for layer_name, repos in layer_map.items():
downstream = repos["downstream"]
if not repos.get("needs_stable", True):
continue
tags = repos.get("tags", None)
if tags:
if not any(match in filter_by_tag for match in tags):
continue
auth = (new_env.get("CDKBOT_GH_USR"), new_env.get("CDKBOT_GH_PSW"))
default_branch = repos.get("branch") or default_gh_branch(
downstream, auth=auth)
log.info(
f"Releasing :: {layer_name:^35} :: from: {default_branch} to: stable"
)
downstream = f"https://{':'.join(auth)}@github.com/{downstream}"
identifier = str(uuid.uuid4())
os.makedirs(identifier)
for line in git.clone(downstream, identifier, _iter=True):
log.info(line)
git_rev_default = (git("rev-parse",
f"origin/{default_branch}",
_cwd=identifier).stdout.decode().strip())
git_rev_stable = (git("rev-parse",
"origin/stable",
_cwd=identifier).stdout.decode().strip())
if git_rev_default == git_rev_stable:
log.info(
f"Skipping :: {layer_name:^35} :: {default_branch} == stable"
)
continue
log.info(
f"Commits :: {layer_name:^35} :: {default_branch} != stable")
log.info(f" {default_branch:10}= {git_rev_default:32}")
log.info(f" {'stable':10}= {git_rev_stable:32}")
for line in git("rev-list",
f"origin/stable..origin/{default_branch}",
_cwd=identifier):
for line in git.show(
"--format=%h %an '%s' %cr",
"--no-patch",
line.strip(),
_cwd=identifier,
):
log.info(" " + line.strip())
if not dry_run:
git.config("user.email",
"*****@*****.**",
_cwd=identifier)
git.config("user.name", "cdkbot", _cwd=identifier)
git.config("--global", "push.default", "simple")
git.checkout("-f", "stable", _cwd=identifier)
git.reset(default_branch, _cwd=identifier)
for line in git.push("origin",
"stable",
"-f",
_cwd=identifier,
_iter=True):
log.info(line)
def _fix_head_for_github(
base_ref_name: Optional[str] = None,
head_ref: Optional[str] = None,
) -> Iterator[Optional[str]]:
"""
GHA can checkout the incorrect commit for a PR (it will create a fake merge commit),
so we need to reset the head to the actual PR branch head before continuing.
Note that this code is written in a generic manner, so that it becomes a no-op when
the CI system has not artifically altered the HEAD ref.
:return: The baseline ref as a commit hash
"""
debug_echo(
f"Called _fix_head_for_github with base_ref_name: {base_ref_name} head_ref: {head_ref}"
)
stashed_rev: Optional[str] = None
base_ref: Optional[str] = base_ref_name
if get_git_repo() is None:
debug_echo("Yielding base_ref since get_git_repo was None")
yield base_ref
return
if base_ref:
# Preserve location of head^ after we possibly change location below
try:
debug_echo(f"Calling git rev-parse {base_ref}")
process = git(["rev-parse", base_ref])
base_ref = process.stdout.decode("utf-8").rstrip()
except sh.ErrorReturnCode as ex:
raise ActionFailure(
f"There is a problem with your git project:{ex}")
if head_ref:
debug_echo("Calling git branch --show-current")
stashed_rev = git(["branch",
"--show-current"]).stdout.decode("utf-8").rstrip()
debug_echo(f"stashed_rev: {stashed_rev}")
if not stashed_rev:
debug_echo("Calling git rev-parse HEAD")
rev_parse = git(["rev-parse", "HEAD"])
debug_echo(rev_parse.stderr.decode("utf-8").rstrip())
stashed_rev = rev_parse.stdout.decode("utf-8").rstrip()
debug_echo(f"stashed_rev: {stashed_rev}")
click.echo(f"| not on head ref {head_ref}; checking that out now...",
err=True)
git.checkout([head_ref],
_timeout=GIT_SH_TIMEOUT,
_out=debug_echo,
_err=debug_echo)
debug_echo(f"checked out {head_ref}")
try:
if base_ref is not None:
merge_base = git("merge-base", base_ref, "HEAD").rstrip()
# fmt:off
click.echo("| reporting findings introduced by these commits:",
err=True)
print_git_log(f"{merge_base}..HEAD")
if merge_base != git("rev-parse", base_ref).rstrip():
click.echo(
"| also reporting findings fixed by these commits from the baseline branch:",
err=True)
print_git_log(f"{merge_base}..{base_ref}")
click.echo("| to exclude these latter commits, run with",
err=True)
click.echo(
f"| --baseline-ref $(git merge-base {base_ref_name} HEAD)",
err=True)
# fmt: on
debug_echo(f"yielding {base_ref}")
yield base_ref
finally:
if stashed_rev is not None:
click.echo(f"| returning to original head revision {stashed_rev}",
err=True)
git.checkout([stashed_rev], _timeout=GIT_SH_TIMEOUT)
