def apply_patch(repo_directory, patch_filepath, repo, patch, flags, use_apply):
""" Apply the `patch` in `patch_filepath` to the `repo` in
`repo_directory` using git am or git apply. The commit
with the user running the script (adabot if credentials are set
for that).
When `use_apply` is true, the `--apply` flag is automatically added
to ensure that any passed flags that turn off apply (e.g. `--check`)
are overridden.
"""
if not os.getcwd() == repo_directory:
os.chdir(repo_directory)
if not use_apply:
try:
git.am(flags, patch_filepath)
except sh.ErrorReturnCode as Err:
apply_errors.append(
dict(repo_name=repo, patch_name=patch, error=Err.stderr))
return False
else:
apply_flags = ["--apply"]
for flag in flags:
if not flag == "--signoff":
apply_flags.append(flag)
try:
git.apply(apply_flags, patch_filepath)
except sh.ErrorReturnCode as Err:
apply_errors.append(
dict(repo_name=repo, patch_name=patch, error=Err.stderr))
return False
with open(patch_filepath) as f:
for line in f:
if "[PATCH]" in line:
message = '"' + line[(line.find("]") + 2):] + '"'
break
try:
git.commit("-a", "-m", message)
except sh.ErrorReturnCode as Err:
apply_errors.append(
dict(repo_name=repo, patch_name=patch, error=Err.stderr))
return False
try:
git.push()
except sh.ErrorReturnCode as Err:
apply_errors.append(
dict(repo_name=repo, patch_name=patch, error=Err.stderr))
return False
return True
def main():
with tempfile.TemporaryDirectory() as tmpdir:
info('Created tmp directory ' + tmpdir)
os.chdir(tmpdir)
git.clone(WIKI_DIR, 'openafs-wiki', _fg=True)
os.chdir('openafs-wiki')
git.remote('add', 'gerrit', 'ssh://gerrit.openafs.org/openafs-wiki.git')
git.fetch('gerrit', _fg=True)
git.reset('gerrit/master', '--hard', _fg=True)
update_page('devel/GerritsForMaster.mdwn', 'master')
update_page('devel/GerritsForStable.mdwn', 'openafs-stable-1_8_x')
update_page('devel/GerritsForOldStable.mdwn', 'openafs-stable-1_6_x')
try:
git.commit('-m', 'update gerrit list', _fg=True)
except ErrorReturnCode_1:
print('No changes')
else:
git.push('gerrit', 'HEAD:refs/heads/master', _fg=True)
def _cut_stable_release(layer_list, charm_list, ancillary_list, filter_by_tag,
dry_run):
"""This will merge each layers master onto the stable branches.
PLEASE NOTE: This step should come after each stable branch has been tagged
and references a current stable bundle revision.
layer_list: YAML spec containing git repos and their upstream/downstream properties
charm_list: YAML spec containing git repos and their upstream/downstream properties
"""
layer_list = yaml.safe_load(Path(layer_list).read_text(encoding="utf8"))
charm_list = yaml.safe_load(Path(charm_list).read_text(encoding="utf8"))
ancillary_list = yaml.safe_load(
Path(ancillary_list).read_text(encoding="utf8"))
new_env = os.environ.copy()
for layer_map in layer_list + charm_list + ancillary_list:
for layer_name, repos in layer_map.items():
downstream = repos["downstream"]
if not repos.get("needs_stable", True):
continue
tags = repos.get("tags", None)
if tags:
if not any(match in filter_by_tag for match in tags):
continue
log.info(
f"Releasing :: {layer_name:^35} :: from: master to: stable")
if not dry_run:
downstream = f"https://{new_env['CDKBOT_GH_USR']}:{new_env['CDKBOT_GH_PSW']}@github.com/{downstream}"
identifier = str(uuid.uuid4())
os.makedirs(identifier)
for line in git.clone(downstream, identifier, _iter=True):
log.info(line)
git_rev_master = git("rev-parse",
"origin/master",
_cwd=identifier).stdout.decode()
git_rev_stable = git("rev-parse",
"origin/stable",
_cwd=identifier).stdout.decode()
if git_rev_master == git_rev_stable:
log.info(
f"Skipping :: {layer_name:^35} :: master == stable")
continue
git.config("user.email",
"*****@*****.**",
_cwd=identifier)
git.config("user.name", "cdkbot", _cwd=identifier)
git.config("--global", "push.default", "simple")
git.checkout("-f", "stable", _cwd=identifier)
git.merge("master", "--no-ff", _cwd=identifier)
for line in git.push("origin",
"stable",
_cwd=identifier,
_iter=True):
log.info(line)
def push_repo(data, app_name):
repo_name = data['repository']['name']
repo_owner = data['repository']['owner']['name']
head_commit = data['head_commit']['id']
repo_path = get_repo_path(repo_owner, repo_name)
dokku_host = f'{SSH_CONNECT_STRING}:{settings.SSH_DOKKU_PORT}'
if not settings.DEPLOY_LOGS_BASE_PATH.exists():
settings.DEPLOY_LOGS_BASE_PATH.mkdir()
deploy_log_file = settings.DEPLOY_LOGS_BASE_PATH / f'{app_name}.txt'
with deploy_log_file.open('wb') as dlfo:
dlfo.write(
dedent(f"""\
=====================================================
Deployment: {datetime.utcnow().isoformat()}
Repository: {data['repository']['full_name']}
Branch name: {data['ref']}
App name: {app_name}
Github user: {data['pusher']['name']}
Commit: {head_commit}
=====================================================
""").encode('utf-8'))
with pushd(repo_path):
try:
app_json = str(git.show(f'{head_commit}:app.json'))
except ErrorReturnCode:
app_json = None
config_set(app_name, app_json, dlfo)
git.push('--force',
f'ssh://{dokku_host}/{app_name}',
f'{head_commit}:refs/heads/master',
_err_to_out=True,
_out=dlfo)
if settings.APPS_LETSENCRYPT:
dlfo.write(
dedent(f"""\
================================
Let's Encrypt: Add or renew cert
================================
""").encode('utf-8'))
dokku('letsencrypt:auto-renew', app_name, _out=dlfo)
def update_json_file(working_directory, cp_org_dir, output_filename,
json_string):
""" Clone the circuitpython-org repo, update libraries.json, and push the updates
in a commit.
"""
if "TRAIVS" in os.environ:
if not os.path.isdir(cp_org_dir):
os.makedirs(cp_org_dir, exist_ok=True)
git_url = "https://" + os.environ[
"ADABOT_GITHUB_ACCESS_TOKEN"] + "@github.com/adafruit/circuitpython-org.git"
git.clone("-o", "adafruit", git_url, cp_org_dir)
os.chdir(cp_org_dir)
git.pull()
git.submodule("update", "--init", "--recursive")
with open(output_filename, "w") as json_file:
json.dump(json_string, json_file, indent=2)
commit_day = date.date.strftime(datetime.datetime.today(), "%Y-%m-%d")
commit_msg = "adabot: auto-update of libraries.json ({})".format(
commit_day)
git.commit("-a", "-m", commit_msg)
git_push = git.push("adafruit", "master")
print(git_push)
def _tag_stable_forks(layer_list, charm_list, k8s_version, bundle_rev,
filter_by_tag, bugfix, dry_run):
"""Tags stable forks to a certain bundle revision for a k8s version
layer_list: YAML spec containing git repos and their upstream/downstream properties
bundle_rev: bundle revision to tag for a particular version of k8s
git tag (ie. ck-{bundle_rev}), this would mean we tagged current
stable branches for 1.14 with the latest charmed kubernetes(ck) bundle rev
of {bundle_rev}
TODO: Switch to different merge strategy
git checkout master
git checkout -b staging
git merge stable -s ours
git checkout stable
git reset staging
"""
layer_list = yaml.safe_load(Path(layer_list).read_text(encoding="utf8"))
charm_list = yaml.safe_load(Path(charm_list).read_text(encoding="utf8"))
new_env = os.environ.copy()
for layer_map in layer_list + charm_list:
for layer_name, repos in layer_map.items():
tags = repos.get("tags", None)
if tags:
if not any(match in filter_by_tag for match in tags):
continue
downstream = repos["downstream"]
if bugfix:
tag = f"{k8s_version}+{bundle_rev}"
else:
tag = f"ck-{k8s_version}-{bundle_rev}"
if not repos.get("needs_tagging", True):
log.info(f"Skipping {layer_name} :: does not require tagging")
continue
log.info(f"Tagging {layer_name} ({tag}) :: {repos['downstream']}")
if not dry_run:
downstream = f"https://{new_env['CDKBOT_GH_USR']}:{new_env['CDKBOT_GH_PSW']}@github.com/{downstream}"
identifier = str(uuid.uuid4())
os.makedirs(identifier)
for line in git.clone(downstream, identifier, _iter=True):
log.info(line)
git.config("user.email",
"*****@*****.**",
_cwd=identifier)
git.config("user.name", "cdkbot", _cwd=identifier)
git.config("--global", "push.default", "simple")
git.checkout("stable", _cwd=identifier)
try:
for line in git.tag("--force",
tag,
_cwd=identifier,
_iter=True,
_bg_exc=False):
log.info(line)
for line in git.push(
"--force",
"origin",
tag,
_cwd=identifier,
_bg_exc=False,
_iter=True,
):
log.info(line)
except sh.ErrorReturnCode as error:
log.info(
f"Problem tagging: {error.stderr.decode().strip()}, will skip for now.."
)
def push_updates(bundle_path):
working_directory = os.path.abspath(os.getcwd())
os.chdir(bundle_path)
git.push()
os.chdir(working_directory)
def _cut_stable_release(layer_list, charm_list, ancillary_list, filter_by_tag,
dry_run):
"""This will merge each layers master onto the stable branches.
PLEASE NOTE: This step should come after each stable branch has been tagged
and references a current stable bundle revision.
layer_list: YAML spec containing git repos and their upstream/downstream properties
charm_list: YAML spec containing git repos and their upstream/downstream properties
"""
layer_list = yaml.safe_load(Path(layer_list).read_text(encoding="utf8"))
charm_list = yaml.safe_load(Path(charm_list).read_text(encoding="utf8"))
ancillary_list = yaml.safe_load(
Path(ancillary_list).read_text(encoding="utf8"))
new_env = os.environ.copy()
failed_to_release = []
for layer_map in layer_list + charm_list + ancillary_list:
for layer_name, repos in layer_map.items():
downstream = repos["downstream"]
if not repos.get("needs_stable", True):
continue
tags = repos.get("tags", None)
if tags:
if not any(match in filter_by_tag for match in tags):
continue
auth = (new_env.get("CDKBOT_GH_USR"), new_env.get("CDKBOT_GH_PSW"))
default_branch = repos.get("branch") or default_gh_branch(
downstream, auth=auth)
log.info(
f"Releasing :: {layer_name:^35} :: from: {default_branch} to: stable"
)
downstream = f"https://{':'.join(auth)}@github.com/{downstream}"
identifier = str(uuid.uuid4())
os.makedirs(identifier)
for line in git.clone(downstream, identifier, _iter=True):
log.info(line)
git_rev_default = (git("rev-parse",
f"origin/{default_branch}",
_cwd=identifier).stdout.decode().strip())
git_rev_stable = (git("rev-parse",
"origin/stable",
_cwd=identifier).stdout.decode().strip())
if git_rev_default == git_rev_stable:
log.info(
f"Skipping :: {layer_name:^35} :: {default_branch} == stable"
)
continue
log.info(
f"Commits :: {layer_name:^35} :: {default_branch} != stable")
log.info(f" {default_branch:10}= {git_rev_default:32}")
log.info(f" {'stable':10}= {git_rev_stable:32}")
for line in git("rev-list",
f"origin/stable..origin/{default_branch}",
_cwd=identifier):
for line in git.show(
"--format=%h %an '%s' %cr",
"--no-patch",
line.strip(),
_cwd=identifier,
):
log.info(" " + line.strip())
if not dry_run:
git.config("user.email",
"*****@*****.**",
_cwd=identifier)
git.config("user.name", "cdkbot", _cwd=identifier)
git.config("--global", "push.default", "simple")
git.checkout("-f", "stable", _cwd=identifier)
git.reset(default_branch, _cwd=identifier)
for line in git.push("origin",
"stable",
"-f",
_cwd=identifier,
_iter=True):
log.info(line)
def sync_branches(self, repo, bitbucket_account_id, bitbucket_access_token,
github_account_id, github_access_token):
repo_name = repo['name']
prefixed_repo_name = self.prefix + repo_name
github_link = repo['github_link']
bitbucket_link = repo['bitbucket_link']
# Boolean: whether the repo is a new migration to GitHub
new_migration = repo['new_migration']
# Use this instead of setting the authenticated link as a new remote.
# Remote links get stored in git config
github_link_domain = github_link.split("//")[1]
authenticated_github_link = f"https://{github_account_id}:{github_access_token}@{github_link_domain}"
bitbucket_link_domain = bitbucket_link.split("//")[1]
authenticated_bitbucket_link = f"https://{bitbucket_account_id}:{bitbucket_access_token}@{bitbucket_link_domain}"
# Set remote to bitbucket
git.remote('set-url', 'origin', bitbucket_link)
self.log.debug("Syncing branches. Set origin to Bitbucket",
repo_name=repo_name,
bitbucket_link=bitbucket_link)
# Fetch branches from origin (bitbucket)
self.log.info("Fetching refs (branches) from origin",
repo_name=repo_name)
# git.fetch('origin')
git.fetch(authenticated_bitbucket_link)
self.log.debug("Fetched refs (branches) from BitBucket",
result="SUCCESS",
repo_name=repo_name)
# List remote branches
remote_branches = git.branch("-r").split("\n")
remote_branches = [
remote.lstrip().rstrip() for remote in remote_branches
if (remote and not re.match("^.*/HEAD -> .*$", remote))
]
try:
# if master exists on origin, move that to the start of the array
# pushing 'master' first makes it the default branch on github
remote_branches.remove('origin/master')
remote_branches = ['origin/master'] + remote_branches
except Exception:
# 'master' did not exist on origin
pass
success_branches = []
failed_branches = []
# Push changes to each branch individually, log error if any fails and continue to next branch
for remote in remote_branches:
[remote_name, branch_name] = remote.split('/', 1)
self.log.info("Syncing branch for repository",
repo_name=repo_name,
branch_name=branch_name)
if (remote_name == 'origin'):
# Different way to handle master branches, support prefixing.
if (branch_name == "master"):
master_branch_refspecs = []
prefix_exists = self.master_branch_prefix != ""
if (prefix_exists):
# Order is IMPORTANT, 'master' should be added before prefixed_master.
# Default branch is the first branch that is pushed to GitHub
if (new_migration):
master_branch_refspecs.append(
f"refs/remotes/origin/{branch_name}:refs/heads/{branch_name}"
)
prefixed_master_branch_name = self.master_branch_prefix + branch_name
master_branch_refspecs.append(
f"refs/remotes/origin/{branch_name}:refs/heads/{prefixed_master_branch_name}"
)
else:
master_branch_refspecs.append(
f"refs/remotes/origin/{branch_name}:refs/heads/{branch_name}"
)
for branch_refspec in master_branch_refspecs:
target_branch_name = branch_refspec.split('/')[-1]
try:
self.log.info(
"Pushing branch for repository",
repo_name=prefixed_repo_name,
repo_prefix=self.prefix,
branch_name=branch_name,
target_branch_name=target_branch_name)
git.push(authenticated_github_link, branch_refspec)
# Success on syncing current branch
self.log.debug(
"Successfully synced branch for repository",
result="SUCCESS",
repo_name=prefixed_repo_name,
repo_prefix=self.prefix,
branch_name=branch_name,
target_branch_name=target_branch_name)
success_branches.append(branch_name)
except ErrorReturnCode as e:
# Redact or remove the access token before logging
stderr = utils.StringUtils.redact_error(
e.stderr, github_access_token,
"<ACCESS-TOKEN>")
self.log.error(
"Failed to push changes to origin branch",
result="FAILED",
repo_name=prefixed_repo_name,
repo_prefix=self.prefix,
branch_name=branch_name,
target_branch_name=target_branch_name,
exit_code=e.exit_code,
stderr=stderr)
failed_branches.append(branch_name)
continue # Continue to the next master_branch_refspec
continue # Continue to the next branch
branch_refspec = f"refs/remotes/origin/{branch_name}:refs/heads/{branch_name}"
try:
self.log.info("Pushing branch for repository",
repo_name=prefixed_repo_name,
repo_prefix=self.prefix,
branch_name=branch_name,
target_branch_name=branch_name)
git.push(authenticated_github_link, branch_refspec)
# Success on syncing current branch
self.log.debug("Successfully synced branch for repository",
result="SUCCESS",
repo_name=prefixed_repo_name,
repo_prefix=self.prefix,
branch_name=branch_name,
target_branch_name=branch_name)
success_branches.append(branch_name)
except ErrorReturnCode as e:
# Redact or remove the access token before logging
stderr = utils.StringUtils.redact_error(
e.stderr, github_access_token, "<ACCESS-TOKEN>")
self.log.error("Failed to push changes to origin branch",
result="FAILED",
repo_name=prefixed_repo_name,
repo_prefix=self.prefix,
branch_name=branch_name,
target_branch_name=branch_name,
exit_code=e.exit_code,
stderr=stderr)
failed_branches.append(branch_name)
continue
else:
continue
all_remote_branches = [
branch_name.split('origin/')[1] for branch_name in remote_branches
]
branches_sync_success = set(all_remote_branches) == set(
success_branches)
return branches_sync_success, all_remote_branches, failed_branches
def sync_tags(self, repo, bitbucket_account_id, bitbucket_access_token,
github_account_id, github_access_token):
# Everytime, tags are fetched from remote (bitbucket) and then pushed to github
repo_name = repo['name']
prefixed_repo_name = self.prefix + repo_name
github_link = repo['github_link']
bitbucket_link = repo['bitbucket_link']
# Use this instead of setting the authenticated link as a new remote.
# Remote links get stored in git config
github_link_domain = github_link.split("//")[1]
authenticated_github_link = f"https://{github_account_id}:{github_access_token}@{github_link_domain}"
bitbucket_link_domain = bitbucket_link.split("//")[1]
authenticated_bitbucket_link = f"https://{bitbucket_account_id}:{bitbucket_access_token}@{bitbucket_link_domain}"
git.remote('set-url', 'origin', bitbucket_link)
self.log.debug("Syncing Tags. Set origin to BitBucket",
repo_name=repo_name,
bitbucket_link=bitbucket_link)
# Fetch tags from origin (bitbucket)
self.log.info("Fetching refs (tags) from origin", repo_name=repo_name)
# git.fetch('origin')
git.fetch(authenticated_bitbucket_link)
self.log.debug("Fetched refs (tags) from BitBucket",
result="SUCCESS",
repo_name=repo_name)
# List all tags
tags = git.tag().split('\n')
tags = [tag.lstrip().rstrip() for tag in tags if tag]
success_tags = []
failed_tags = []
# Set origin to github
# git.remote('set-url', 'origin', github_link)
self.log.debug("Syncing tags. Set origin to Github",
repo_name=prefixed_repo_name,
repo_prefix=self.prefix,
github_link=github_link)
# Push each tag individually, log error if any fails and continue to next tag
for tag_name in tags:
self.log.info("Syncing tag for repository",
repo_name=repo_name,
tag_name=tag_name)
try:
tag_refspec = f"refs/tags/{tag_name}:refs/tags/{tag_name}"
git.push(authenticated_github_link, tag_refspec)
self.log.debug("Pushed tag for repository",
result="SUCCESS",
repo_name=prefixed_repo_name,
repo_prefix=self.prefix,
tag_name=tag_name)
success_tags.append(tag_name)
except ErrorReturnCode as e:
# Redact or remove the access token before logging
stderr = utils.StringUtils.redact_error(
e.stderr, github_access_token, "<ACCESS-TOKEN>")
self.log.error("Failed to push tag to github",
result="FAILED",
repo_name=prefixed_repo_name,
repo_prefix=self.prefix,
tag_name=tag_name,
exit_code=e.exit_code,
stderr=stderr)
failed_tags.append(tag_name)
continue
tags_sync_success = set(tags) == set(success_tags)
return tags_sync_success, tags, failed_tags
def empty_commit(repo):
"""Commits an empty commit and pushes."""
with sh.pushd(repo):
git.commit("--allow-empty", "--only", "--message",
"Initial empty commit.")
git.push()
