def apply_patch(repo_directory, patch_filepath, repo, patch, flags, use_apply): """ Apply the `patch` in `patch_filepath` to the `repo` in `repo_directory` using git am or git apply. The commit with the user running the script (adabot if credentials are set for that). When `use_apply` is true, the `--apply` flag is automatically added to ensure that any passed flags that turn off apply (e.g. `--check`) are overridden. """ if not os.getcwd() == repo_directory: os.chdir(repo_directory) if not use_apply: try: git.am(flags, patch_filepath) except sh.ErrorReturnCode as Err: apply_errors.append( dict(repo_name=repo, patch_name=patch, error=Err.stderr)) return False else: apply_flags = ["--apply"] for flag in flags: if not flag == "--signoff": apply_flags.append(flag) try: git.apply(apply_flags, patch_filepath) except sh.ErrorReturnCode as Err: apply_errors.append( dict(repo_name=repo, patch_name=patch, error=Err.stderr)) return False with open(patch_filepath) as f: for line in f: if "[PATCH]" in line: message = '"' + line[(line.find("]") + 2):] + '"' break try: git.commit("-a", "-m", message) except sh.ErrorReturnCode as Err: apply_errors.append( dict(repo_name=repo, patch_name=patch, error=Err.stderr)) return False try: git.push() except sh.ErrorReturnCode as Err: apply_errors.append( dict(repo_name=repo, patch_name=patch, error=Err.stderr)) return False return True
def main(): with tempfile.TemporaryDirectory() as tmpdir: info('Created tmp directory ' + tmpdir) os.chdir(tmpdir) git.clone(WIKI_DIR, 'openafs-wiki', _fg=True) os.chdir('openafs-wiki') git.remote('add', 'gerrit', 'ssh://gerrit.openafs.org/openafs-wiki.git') git.fetch('gerrit', _fg=True) git.reset('gerrit/master', '--hard', _fg=True) update_page('devel/GerritsForMaster.mdwn', 'master') update_page('devel/GerritsForStable.mdwn', 'openafs-stable-1_8_x') update_page('devel/GerritsForOldStable.mdwn', 'openafs-stable-1_6_x') try: git.commit('-m', 'update gerrit list', _fg=True) except ErrorReturnCode_1: print('No changes') else: git.push('gerrit', 'HEAD:refs/heads/master', _fg=True)
def _cut_stable_release(layer_list, charm_list, ancillary_list, filter_by_tag, dry_run): """This will merge each layers master onto the stable branches. PLEASE NOTE: This step should come after each stable branch has been tagged and references a current stable bundle revision. layer_list: YAML spec containing git repos and their upstream/downstream properties charm_list: YAML spec containing git repos and their upstream/downstream properties """ layer_list = yaml.safe_load(Path(layer_list).read_text(encoding="utf8")) charm_list = yaml.safe_load(Path(charm_list).read_text(encoding="utf8")) ancillary_list = yaml.safe_load( Path(ancillary_list).read_text(encoding="utf8")) new_env = os.environ.copy() for layer_map in layer_list + charm_list + ancillary_list: for layer_name, repos in layer_map.items(): downstream = repos["downstream"] if not repos.get("needs_stable", True): continue tags = repos.get("tags", None) if tags: if not any(match in filter_by_tag for match in tags): continue log.info( f"Releasing :: {layer_name:^35} :: from: master to: stable") if not dry_run: downstream = f"https://{new_env['CDKBOT_GH_USR']}:{new_env['CDKBOT_GH_PSW']}@github.com/{downstream}" identifier = str(uuid.uuid4()) os.makedirs(identifier) for line in git.clone(downstream, identifier, _iter=True): log.info(line) git_rev_master = git("rev-parse", "origin/master", _cwd=identifier).stdout.decode() git_rev_stable = git("rev-parse", "origin/stable", _cwd=identifier).stdout.decode() if git_rev_master == git_rev_stable: log.info( f"Skipping :: {layer_name:^35} :: master == stable") continue git.config("user.email", "*****@*****.**", _cwd=identifier) git.config("user.name", "cdkbot", _cwd=identifier) git.config("--global", "push.default", "simple") git.checkout("-f", "stable", _cwd=identifier) git.merge("master", "--no-ff", _cwd=identifier) for line in git.push("origin", "stable", _cwd=identifier, _iter=True): log.info(line)
def push_repo(data, app_name): repo_name = data['repository']['name'] repo_owner = data['repository']['owner']['name'] head_commit = data['head_commit']['id'] repo_path = get_repo_path(repo_owner, repo_name) dokku_host = f'{SSH_CONNECT_STRING}:{settings.SSH_DOKKU_PORT}' if not settings.DEPLOY_LOGS_BASE_PATH.exists(): settings.DEPLOY_LOGS_BASE_PATH.mkdir() deploy_log_file = settings.DEPLOY_LOGS_BASE_PATH / f'{app_name}.txt' with deploy_log_file.open('wb') as dlfo: dlfo.write( dedent(f"""\ ===================================================== Deployment: {datetime.utcnow().isoformat()} Repository: {data['repository']['full_name']} Branch name: {data['ref']} App name: {app_name} Github user: {data['pusher']['name']} Commit: {head_commit} ===================================================== """).encode('utf-8')) with pushd(repo_path): try: app_json = str(git.show(f'{head_commit}:app.json')) except ErrorReturnCode: app_json = None config_set(app_name, app_json, dlfo) git.push('--force', f'ssh://{dokku_host}/{app_name}', f'{head_commit}:refs/heads/master', _err_to_out=True, _out=dlfo) if settings.APPS_LETSENCRYPT: dlfo.write( dedent(f"""\ ================================ Let's Encrypt: Add or renew cert ================================ """).encode('utf-8')) dokku('letsencrypt:auto-renew', app_name, _out=dlfo)
def update_json_file(working_directory, cp_org_dir, output_filename, json_string): """ Clone the circuitpython-org repo, update libraries.json, and push the updates in a commit. """ if "TRAIVS" in os.environ: if not os.path.isdir(cp_org_dir): os.makedirs(cp_org_dir, exist_ok=True) git_url = "https://" + os.environ[ "ADABOT_GITHUB_ACCESS_TOKEN"] + "@github.com/adafruit/circuitpython-org.git" git.clone("-o", "adafruit", git_url, cp_org_dir) os.chdir(cp_org_dir) git.pull() git.submodule("update", "--init", "--recursive") with open(output_filename, "w") as json_file: json.dump(json_string, json_file, indent=2) commit_day = date.date.strftime(datetime.datetime.today(), "%Y-%m-%d") commit_msg = "adabot: auto-update of libraries.json ({})".format( commit_day) git.commit("-a", "-m", commit_msg) git_push = git.push("adafruit", "master") print(git_push)
def _tag_stable_forks(layer_list, charm_list, k8s_version, bundle_rev, filter_by_tag, bugfix, dry_run): """Tags stable forks to a certain bundle revision for a k8s version layer_list: YAML spec containing git repos and their upstream/downstream properties bundle_rev: bundle revision to tag for a particular version of k8s git tag (ie. ck-{bundle_rev}), this would mean we tagged current stable branches for 1.14 with the latest charmed kubernetes(ck) bundle rev of {bundle_rev} TODO: Switch to different merge strategy git checkout master git checkout -b staging git merge stable -s ours git checkout stable git reset staging """ layer_list = yaml.safe_load(Path(layer_list).read_text(encoding="utf8")) charm_list = yaml.safe_load(Path(charm_list).read_text(encoding="utf8")) new_env = os.environ.copy() for layer_map in layer_list + charm_list: for layer_name, repos in layer_map.items(): tags = repos.get("tags", None) if tags: if not any(match in filter_by_tag for match in tags): continue downstream = repos["downstream"] if bugfix: tag = f"{k8s_version}+{bundle_rev}" else: tag = f"ck-{k8s_version}-{bundle_rev}" if not repos.get("needs_tagging", True): log.info(f"Skipping {layer_name} :: does not require tagging") continue log.info(f"Tagging {layer_name} ({tag}) :: {repos['downstream']}") if not dry_run: downstream = f"https://{new_env['CDKBOT_GH_USR']}:{new_env['CDKBOT_GH_PSW']}@github.com/{downstream}" identifier = str(uuid.uuid4()) os.makedirs(identifier) for line in git.clone(downstream, identifier, _iter=True): log.info(line) git.config("user.email", "*****@*****.**", _cwd=identifier) git.config("user.name", "cdkbot", _cwd=identifier) git.config("--global", "push.default", "simple") git.checkout("stable", _cwd=identifier) try: for line in git.tag("--force", tag, _cwd=identifier, _iter=True, _bg_exc=False): log.info(line) for line in git.push( "--force", "origin", tag, _cwd=identifier, _bg_exc=False, _iter=True, ): log.info(line) except sh.ErrorReturnCode as error: log.info( f"Problem tagging: {error.stderr.decode().strip()}, will skip for now.." )
def push_updates(bundle_path): working_directory = os.path.abspath(os.getcwd()) os.chdir(bundle_path) git.push() os.chdir(working_directory)
def _cut_stable_release(layer_list, charm_list, ancillary_list, filter_by_tag, dry_run): """This will merge each layers master onto the stable branches. PLEASE NOTE: This step should come after each stable branch has been tagged and references a current stable bundle revision. layer_list: YAML spec containing git repos and their upstream/downstream properties charm_list: YAML spec containing git repos and their upstream/downstream properties """ layer_list = yaml.safe_load(Path(layer_list).read_text(encoding="utf8")) charm_list = yaml.safe_load(Path(charm_list).read_text(encoding="utf8")) ancillary_list = yaml.safe_load( Path(ancillary_list).read_text(encoding="utf8")) new_env = os.environ.copy() failed_to_release = [] for layer_map in layer_list + charm_list + ancillary_list: for layer_name, repos in layer_map.items(): downstream = repos["downstream"] if not repos.get("needs_stable", True): continue tags = repos.get("tags", None) if tags: if not any(match in filter_by_tag for match in tags): continue auth = (new_env.get("CDKBOT_GH_USR"), new_env.get("CDKBOT_GH_PSW")) default_branch = repos.get("branch") or default_gh_branch( downstream, auth=auth) log.info( f"Releasing :: {layer_name:^35} :: from: {default_branch} to: stable" ) downstream = f"https://{':'.join(auth)}@github.com/{downstream}" identifier = str(uuid.uuid4()) os.makedirs(identifier) for line in git.clone(downstream, identifier, _iter=True): log.info(line) git_rev_default = (git("rev-parse", f"origin/{default_branch}", _cwd=identifier).stdout.decode().strip()) git_rev_stable = (git("rev-parse", "origin/stable", _cwd=identifier).stdout.decode().strip()) if git_rev_default == git_rev_stable: log.info( f"Skipping :: {layer_name:^35} :: {default_branch} == stable" ) continue log.info( f"Commits :: {layer_name:^35} :: {default_branch} != stable") log.info(f" {default_branch:10}= {git_rev_default:32}") log.info(f" {'stable':10}= {git_rev_stable:32}") for line in git("rev-list", f"origin/stable..origin/{default_branch}", _cwd=identifier): for line in git.show( "--format=%h %an '%s' %cr", "--no-patch", line.strip(), _cwd=identifier, ): log.info(" " + line.strip()) if not dry_run: git.config("user.email", "*****@*****.**", _cwd=identifier) git.config("user.name", "cdkbot", _cwd=identifier) git.config("--global", "push.default", "simple") git.checkout("-f", "stable", _cwd=identifier) git.reset(default_branch, _cwd=identifier) for line in git.push("origin", "stable", "-f", _cwd=identifier, _iter=True): log.info(line)
def sync_branches(self, repo, bitbucket_account_id, bitbucket_access_token, github_account_id, github_access_token): repo_name = repo['name'] prefixed_repo_name = self.prefix + repo_name github_link = repo['github_link'] bitbucket_link = repo['bitbucket_link'] # Boolean: whether the repo is a new migration to GitHub new_migration = repo['new_migration'] # Use this instead of setting the authenticated link as a new remote. # Remote links get stored in git config github_link_domain = github_link.split("//")[1] authenticated_github_link = f"https://{github_account_id}:{github_access_token}@{github_link_domain}" bitbucket_link_domain = bitbucket_link.split("//")[1] authenticated_bitbucket_link = f"https://{bitbucket_account_id}:{bitbucket_access_token}@{bitbucket_link_domain}" # Set remote to bitbucket git.remote('set-url', 'origin', bitbucket_link) self.log.debug("Syncing branches. Set origin to Bitbucket", repo_name=repo_name, bitbucket_link=bitbucket_link) # Fetch branches from origin (bitbucket) self.log.info("Fetching refs (branches) from origin", repo_name=repo_name) # git.fetch('origin') git.fetch(authenticated_bitbucket_link) self.log.debug("Fetched refs (branches) from BitBucket", result="SUCCESS", repo_name=repo_name) # List remote branches remote_branches = git.branch("-r").split("\n") remote_branches = [ remote.lstrip().rstrip() for remote in remote_branches if (remote and not re.match("^.*/HEAD -> .*$", remote)) ] try: # if master exists on origin, move that to the start of the array # pushing 'master' first makes it the default branch on github remote_branches.remove('origin/master') remote_branches = ['origin/master'] + remote_branches except Exception: # 'master' did not exist on origin pass success_branches = [] failed_branches = [] # Push changes to each branch individually, log error if any fails and continue to next branch for remote in remote_branches: [remote_name, branch_name] = remote.split('/', 1) self.log.info("Syncing branch for repository", repo_name=repo_name, branch_name=branch_name) if (remote_name == 'origin'): # Different way to handle master branches, support prefixing. if (branch_name == "master"): master_branch_refspecs = [] prefix_exists = self.master_branch_prefix != "" if (prefix_exists): # Order is IMPORTANT, 'master' should be added before prefixed_master. # Default branch is the first branch that is pushed to GitHub if (new_migration): master_branch_refspecs.append( f"refs/remotes/origin/{branch_name}:refs/heads/{branch_name}" ) prefixed_master_branch_name = self.master_branch_prefix + branch_name master_branch_refspecs.append( f"refs/remotes/origin/{branch_name}:refs/heads/{prefixed_master_branch_name}" ) else: master_branch_refspecs.append( f"refs/remotes/origin/{branch_name}:refs/heads/{branch_name}" ) for branch_refspec in master_branch_refspecs: target_branch_name = branch_refspec.split('/')[-1] try: self.log.info( "Pushing branch for repository", repo_name=prefixed_repo_name, repo_prefix=self.prefix, branch_name=branch_name, target_branch_name=target_branch_name) git.push(authenticated_github_link, branch_refspec) # Success on syncing current branch self.log.debug( "Successfully synced branch for repository", result="SUCCESS", repo_name=prefixed_repo_name, repo_prefix=self.prefix, branch_name=branch_name, target_branch_name=target_branch_name) success_branches.append(branch_name) except ErrorReturnCode as e: # Redact or remove the access token before logging stderr = utils.StringUtils.redact_error( e.stderr, github_access_token, "<ACCESS-TOKEN>") self.log.error( "Failed to push changes to origin branch", result="FAILED", repo_name=prefixed_repo_name, repo_prefix=self.prefix, branch_name=branch_name, target_branch_name=target_branch_name, exit_code=e.exit_code, stderr=stderr) failed_branches.append(branch_name) continue # Continue to the next master_branch_refspec continue # Continue to the next branch branch_refspec = f"refs/remotes/origin/{branch_name}:refs/heads/{branch_name}" try: self.log.info("Pushing branch for repository", repo_name=prefixed_repo_name, repo_prefix=self.prefix, branch_name=branch_name, target_branch_name=branch_name) git.push(authenticated_github_link, branch_refspec) # Success on syncing current branch self.log.debug("Successfully synced branch for repository", result="SUCCESS", repo_name=prefixed_repo_name, repo_prefix=self.prefix, branch_name=branch_name, target_branch_name=branch_name) success_branches.append(branch_name) except ErrorReturnCode as e: # Redact or remove the access token before logging stderr = utils.StringUtils.redact_error( e.stderr, github_access_token, "<ACCESS-TOKEN>") self.log.error("Failed to push changes to origin branch", result="FAILED", repo_name=prefixed_repo_name, repo_prefix=self.prefix, branch_name=branch_name, target_branch_name=branch_name, exit_code=e.exit_code, stderr=stderr) failed_branches.append(branch_name) continue else: continue all_remote_branches = [ branch_name.split('origin/')[1] for branch_name in remote_branches ] branches_sync_success = set(all_remote_branches) == set( success_branches) return branches_sync_success, all_remote_branches, failed_branches
def sync_tags(self, repo, bitbucket_account_id, bitbucket_access_token, github_account_id, github_access_token): # Everytime, tags are fetched from remote (bitbucket) and then pushed to github repo_name = repo['name'] prefixed_repo_name = self.prefix + repo_name github_link = repo['github_link'] bitbucket_link = repo['bitbucket_link'] # Use this instead of setting the authenticated link as a new remote. # Remote links get stored in git config github_link_domain = github_link.split("//")[1] authenticated_github_link = f"https://{github_account_id}:{github_access_token}@{github_link_domain}" bitbucket_link_domain = bitbucket_link.split("//")[1] authenticated_bitbucket_link = f"https://{bitbucket_account_id}:{bitbucket_access_token}@{bitbucket_link_domain}" git.remote('set-url', 'origin', bitbucket_link) self.log.debug("Syncing Tags. Set origin to BitBucket", repo_name=repo_name, bitbucket_link=bitbucket_link) # Fetch tags from origin (bitbucket) self.log.info("Fetching refs (tags) from origin", repo_name=repo_name) # git.fetch('origin') git.fetch(authenticated_bitbucket_link) self.log.debug("Fetched refs (tags) from BitBucket", result="SUCCESS", repo_name=repo_name) # List all tags tags = git.tag().split('\n') tags = [tag.lstrip().rstrip() for tag in tags if tag] success_tags = [] failed_tags = [] # Set origin to github # git.remote('set-url', 'origin', github_link) self.log.debug("Syncing tags. Set origin to Github", repo_name=prefixed_repo_name, repo_prefix=self.prefix, github_link=github_link) # Push each tag individually, log error if any fails and continue to next tag for tag_name in tags: self.log.info("Syncing tag for repository", repo_name=repo_name, tag_name=tag_name) try: tag_refspec = f"refs/tags/{tag_name}:refs/tags/{tag_name}" git.push(authenticated_github_link, tag_refspec) self.log.debug("Pushed tag for repository", result="SUCCESS", repo_name=prefixed_repo_name, repo_prefix=self.prefix, tag_name=tag_name) success_tags.append(tag_name) except ErrorReturnCode as e: # Redact or remove the access token before logging stderr = utils.StringUtils.redact_error( e.stderr, github_access_token, "<ACCESS-TOKEN>") self.log.error("Failed to push tag to github", result="FAILED", repo_name=prefixed_repo_name, repo_prefix=self.prefix, tag_name=tag_name, exit_code=e.exit_code, stderr=stderr) failed_tags.append(tag_name) continue tags_sync_success = set(tags) == set(success_tags) return tags_sync_success, tags, failed_tags
def empty_commit(repo): """Commits an empty commit and pushes.""" with sh.pushd(repo): git.commit("--allow-empty", "--only", "--message", "Initial empty commit.") git.push()