def remove_patches(directory=None): # Get the current branch current_branch = get_current_branch(directory) # Ensure the current branch is valid if current_branch is None: error("Could not determine current branch, are you in a git repo?") return 1 # Construct the patches branch patches_branch = 'patches/' + current_branch try: # See if the patches branch exists if branch_exists(patches_branch, False, directory=directory): if not branch_exists(patches_branch, True, directory=directory): track_branches(patches_branch, directory) else: error("No patches branch (" + patches_branch + ") found, cannot " "remove patches.") return 1 # Get the parent branch from the patches branch config = get_patch_config(patches_branch, directory=directory) parent, spec = config['parent'], config['base'] if None in [parent, spec]: error("Could not retrieve patches info.") return 1 debug("Removing patches from " + current_branch + " back to base " "commit " + spec) # Reset this branch using git reset --hard spec execute_command('git reset --hard ' + spec, cwd=directory) finally: if current_branch: checkout(current_branch, directory=directory) return 0
def upconvert_bloom_to_config_branch(): global _has_checked_bloom_branch if _has_checked_bloom_branch: return # Assert that this repository does not have multiple remotes check_for_multiple_remotes() if get_root() is None: # Not a git repository return track_branches(['bloom', BLOOM_CONFIG_BRANCH]) if show('bloom', PLACEHOLDER_FILE) is not None: return if show('bloom', 'bloom.conf') is not None: # Wait for the bloom.conf upconvert... return if not branch_exists('bloom'): return _has_checked_bloom_branch = True info("Moving configurations from deprecated 'bloom' branch " "to the '{0}' branch.".format(BLOOM_CONFIG_BRANCH)) tmp_dir = mkdtemp() git_root = get_root() try: # Copy the new upstream source into the temporary directory with inbranch('bloom'): ignores = ('.git', '.gitignore', '.svn', '.hgignore', '.hg', 'CVS') configs = os.path.join(tmp_dir, 'configs') my_copytree(git_root, configs, ignores) if [x for x in os.listdir(os.getcwd()) if x not in ignores]: execute_command('git rm -rf ./*') with open(PLACEHOLDER_FILE, 'w') as f: f.write("""\ This branch ('bloom') has been deprecated in favor of storing settings and overlay files in the master branch. Please goto the master branch for anything which referenced the bloom branch. You can delete this branch at your convenience. """) execute_command('git add ' + PLACEHOLDER_FILE) if has_changes(): execute_command('git commit -m "DEPRECATING BRANCH"') if not branch_exists(BLOOM_CONFIG_BRANCH): info("Creating '{0}' branch.".format(BLOOM_CONFIG_BRANCH)) create_branch(BLOOM_CONFIG_BRANCH, orphaned=True) with inbranch(BLOOM_CONFIG_BRANCH): my_copytree(configs, git_root) execute_command('git add ./*') if has_changes(): execute_command('git commit -m ' '"Moving configs from bloom branch"') finally: # Clean up if os.path.exists(tmp_dir): shutil.rmtree(tmp_dir)
def import_patches(directory=None): # Get current branch current_branch = get_current_branch(directory) # Construct the patches branch name patches_branch = 'patches/' + current_branch # Ensure the patches branch exists and is tracked if branch_exists(patches_branch, False, directory=directory): if not branch_exists(patches_branch, True, directory=directory): track_branches(patches_branch, directory) else: error("The patches branch ({0}) does not ".format(patches_branch) + \ "exist, did you use git-bloom-branch?") return code.BRANCH_DOES_NOT_EXIST # Create a swap space tmp_dir = tempfile.mkdtemp() try: # Get parent branch and base commit from patches branch config = get_patch_config(patches_branch, directory) parent_branch, commit = config['parent'], config['base'] if commit != get_commit_hash(current_branch, directory): warning("The current commit is not the same as the most recent " "rebase commit. This might mean that you have committed " "since the last time you did 'git-bloom-patch export'.") return code.PATCHES_NOT_EXPORTED # Checkout to the patches branch checkout(patches_branch, directory=directory) # Copy the patches to a temp location patches = list_patches(directory) if len(patches) == 0: debug("No patches in the patches branch, nothing to do") return code.NOTHING_TO_DO tmp_dir_patches = [] for patch in patches: tmp_dir_patches.append(os.path.join(tmp_dir, patch)) if directory is not None: patch = os.path.join(directory, patch) shutil.copy(patch, tmp_dir) # Now checkout back to the original branch and import them checkout(current_branch, directory=directory) cmd = 'git am {0}*.patch'.format(tmp_dir + os.sep) execute_command(cmd, cwd=directory) # Notify the user info("Applied {0} patches".format(len(patches))) finally: if current_branch: checkout(current_branch, directory=directory) if os.path.exists(tmp_dir): shutil.rmtree(tmp_dir) return 0
def trim(sub_dir=None, force=False, undo=False, directory=None): # Get the current branch current_branch = get_current_branch(directory) # Ensure the current branch is valid if current_branch is None: error("Could not determine current branch, are you in a git repo?") return code.NOT_ON_A_GIT_BRANCH # Construct the patches branch patches_branch = 'patches/' + current_branch try: # See if the patches branch exists if branch_exists(patches_branch, False, directory=directory): if not branch_exists(patches_branch, True, directory=directory): track_branches(patches_branch, directory) else: error("No patches branch (" + patches_branch + ") found, cannot " "perform trim.") return code.BRANCH_DOES_NOT_EXIST # Get the parent branch from the patches branch config = get_patch_config(patches_branch, directory=directory) if config is None: error("Could not retrieve patches info.") return code.COULD_NOT_GET_PATCH_INFO # If sub_dir is set, try to set it new_config = _set_trim_sub_dir(sub_dir, force, config, directory) if new_config is None: return code.COULD_NOT_TRIM # Perform trime or undo if undo: new_config = _undo(new_config, directory) if new_config is None: return code.NOTHING_TO_DO else: new_config = _trim(new_config, force, directory) if new_config is None: return code.COULD_NOT_TRIM # Commit the new config set_patch_config(patches_branch, new_config, directory) finally: if current_branch: checkout(current_branch, directory=directory) return code.OK
def trim(sub_dir=None, force=False, undo=False, directory=None): # Get the current branch current_branch = get_current_branch(directory) # Ensure the current branch is valid if current_branch is None: error("Could not determine current branch, are you in a git repo?", exit=True) # Construct the patches branch patches_branch = 'patches/' + current_branch try: # See if the patches branch exists if branch_exists(patches_branch, False, directory=directory): if not branch_exists(patches_branch, True, directory=directory): track_branches(patches_branch, directory) else: error("No patches branch (" + patches_branch + ") found, cannot " "perform trim.", exit=True) # Get the parent branch from the patches branch config = get_patch_config(patches_branch, directory=directory) if config is None: error("Could not retrieve patches info.", exit=True) # If sub_dir is set, try to set it new_config = _set_trim_sub_dir(sub_dir, force, config, directory) if new_config is None: sys.exit('Could not perform trim') # Perform trime or undo if undo: new_config = _undo(new_config, directory) if new_config is None: return -1 # Indicates that nothing was done else: new_config = _trim(new_config, force, directory) if new_config is None: sys.exit('Could not perform trim') # Commit the new config set_patch_config(patches_branch, new_config, directory) finally: if current_branch: checkout(current_branch, directory=directory)
def test_track_branches(): tmp_dir = mkdtemp() orig_dir = os.path.join(tmp_dir, 'orig') clone_dir = os.path.join(tmp_dir, 'clone') os.makedirs(orig_dir) os.makedirs(clone_dir) from subprocess import check_call, PIPE, check_output check_call('git init .', shell=True, cwd=orig_dir, stdout=PIPE) check_call('touch example.txt', shell=True, cwd=orig_dir, stdout=PIPE) check_call('git add *', shell=True, cwd=orig_dir, stdout=PIPE) check_call('git commit -m "Init"', shell=True, cwd=orig_dir, stdout=PIPE) check_call('git branch bloom', shell=True, cwd=orig_dir, stdout=PIPE) check_call('git branch upstream', shell=True, cwd=orig_dir, stdout=PIPE) check_call('git branch refactor', shell=True, cwd=orig_dir, stdout=PIPE) from vcstools import VcsClient clone = VcsClient('git', clone_dir) clone.checkout('file://{0}'.format(orig_dir), 'master') output = check_output('git branch --no-color', shell=True, cwd=clone_dir) assert output == '* master\n' from bloom.git import track_branches track_branches(['bloom', 'upstream'], clone_dir) output = check_output('git branch --no-color', shell=True, cwd=clone_dir) assert output == ' bloom\n* master\n upstream\n' track_branches(cwd=clone_dir) output = check_output('git branch --no-color', shell=True, cwd=clone_dir) assert output == ' bloom\n* master\n refactor\n upstream\n', \ output + ' == ` bloom\n* master\n refactor\n upstream\n`' track_branches(['fake'], clone_dir) output = check_output('git branch', shell=True, cwd=clone_dir) assert output.count('fake') == 0 rmtree(tmp_dir)
def remove_patches(directory=None): # Get the current branch current_branch = get_current_branch(directory) if current_branch is None: error("Could not determine current branch.", exit=True) # Ensure the current branch is valid if current_branch is None: error("Could not determine current branch, are you in a git repo?", exit=True) # Construct the patches branch patches_branch = 'patches/' + current_branch try: # See if the patches branch exists if branch_exists(patches_branch, False, directory=directory): if not branch_exists(patches_branch, True, directory=directory): track_branches(patches_branch, directory) else: error("No patches branch (" + patches_branch + ") found, cannot " "remove patches.", exit=True) # Get the parent branch from the patches branch config = get_patch_config(patches_branch, directory=directory) parent, spec = config['parent'], config['base'] if None in [parent, spec]: error("Could not retrieve patches info.", exit=True) debug("Removing patches from " + current_branch + " back to base " "commit " + spec) # Reset this branch using git revert --no-edit spec current_commit = get_commit_hash(current_branch, directory) command_spec = spec + '..' + current_commit execute_command('git revert --no-edit -Xtheirs ' + command_spec, cwd=directory) # Update the base config['base'] = get_commit_hash(current_branch, directory) set_patch_config(patches_branch, config, directory=directory) finally: if current_branch: checkout(current_branch, directory=directory)
def remove_patches(directory=None): # Get the current branch current_branch = get_current_branch(directory) if current_branch is None: error("Could not determine current branch.", exit=True) # Ensure the current branch is valid if current_branch is None: error("Could not determine current branch, are you in a git repo?", exit=True) # Construct the patches branch patches_branch = 'patches/' + current_branch try: # See if the patches branch exists if branch_exists(patches_branch, False, directory=directory): if not branch_exists(patches_branch, True, directory=directory): track_branches(patches_branch, directory) else: error("No patches branch (" + patches_branch + ") found, cannot " "remove patches.", exit=True) # Get the parent branch from the patches branch config = get_patch_config(patches_branch, directory=directory) parent, spec = config['parent'], config['base'] if None in [parent, spec]: error("Could not retrieve patches info.", exit=True) debug("Removing patches from " + current_branch + " back to base " "commit " + spec) # Reset this branch using git revert --no-edit spec current_commit = get_commit_hash(current_branch, directory) command_spec = spec + '..' + current_commit execute_command( 'git revert --no-edit -Xtheirs ' + command_spec, cwd=directory ) # Update the base config['base'] = get_commit_hash(current_branch, directory) set_patch_config(patches_branch, config, directory=directory) finally: if current_branch: checkout(current_branch, directory=directory)
def import_patches(directory=None): # Get current branch current_branch = get_current_branch(directory) if current_branch is None: error("Could not determine current branch.", exit=True) # Construct the patches branch name patches_branch = 'patches/' + current_branch # Ensure the patches branch exists and is tracked if branch_exists(patches_branch, False, directory=directory): if not branch_exists(patches_branch, True, directory=directory): track_branches(patches_branch, directory) else: error("The patches branch ({0}) does not ".format(patches_branch) + "exist, did you use git-bloom-branch?", exit=True) # Create a swap space tmp_dir = tempfile.mkdtemp() try: # Get parent branch and base commit from patches branch config = get_patch_config(patches_branch, directory) parent_branch, commit = config['parent'], config['base'] if commit != get_commit_hash(current_branch, directory): debug( "commit != get_commit_hash(current_branch, directory)" ) debug( "{0} != get_commit_hash({1}, {2}) != {3}".format( commit, current_branch, directory, get_commit_hash(current_branch, directory) ) ) os.system('git log') warning( "The current commit is not the same as the most recent " "rebase commit." ) warning( "This might mean that you have committed since the last " "time you did:" ) warning( " 'git-bloom-patch rebase' or 'git-bloom-patch remove'" ) warning( "Make sure you export any commits you want to save first:" ) warning(" 'git-bloom-patch export'") error("Patches not exported", exit=True) # Checkout to the patches branch checkout(patches_branch, directory=directory) # Copy the patches to a temp location patches = list_patches(directory) if len(patches) == 0: debug("No patches in the patches branch, nothing to do") return -1 # Indicates that nothing was done tmp_dir_patches = [] for patch in patches: tmp_dir_patches.append(os.path.join(tmp_dir, patch)) if directory is not None: patch = os.path.join(directory, patch) shutil.copy(patch, tmp_dir) # Now checkout back to the original branch and import them checkout(current_branch, directory=directory) try: cmd = 'git am {0}*.patch'.format(tmp_dir + os.sep) execute_command(cmd, cwd=directory) except subprocess.CalledProcessError as e: warning("Failed to apply one or more patches for the " "'{0}' branch.".format(str(e))) print() print() print(">>> Resolve any conflicts and when you have resolved this " "problem run 'git am --resolved' and then exit the " "shell using 'exit 0'. <<<") print(" To abort use 'exit 1'") if 'bash' in os.environ['SHELL']: ret = subprocess.call([ "/bin/bash", "-l", "-c", """\ /bin/bash --rcfile <(echo "if [ -f /etc/bashrc ]; then source /etc/bashrc; fi; \ if [ -f ~/.bashrc ]; then source ~/.bashrc; fi;PS1='(bloom)$PS1'") -i""" ]) else: ret = subprocess.call("$SHELL", shell=True) if ret != 0: error("User failed to resolve patch conflicts, exiting.") sys.exit("'git-bloom-patch import' aborted.") info("User reports that conflicts have been resolved, continuing.") # Notify the user info("Applied {0} patches".format(len(patches))) finally: if current_branch: checkout(current_branch, directory=directory) if os.path.exists(tmp_dir): shutil.rmtree(tmp_dir)
def execute_branch(src, dst, interactive, directory=None): """ Changes to the destination branch, creates branch and patches/branch if they do not exist. If the dst branch does not exist yet, then it is created by branching the current working branch or the specified SRC_BRANCH. If the patches/dst branch branch does not exist yet then it is created. If the branches are created successful, then the working branch will be set to the dst branch, otherwise the working branch will remain unchanged. :param src: source branch from which to copy :param dst: destination branch :param interactive: if True actions are summarized before committing :param directory: directory in which to preform this action :raises: subprocess.CalledProcessError if any git calls fail """ # Determine if the srouce branch exists if src is None: error("No source specified and/or not a branch currently", exit=True) if branch_exists(src, local_only=False, directory=directory): if not branch_exists(src, local_only=True, directory=directory): debug("Tracking source branch: {0}".format(src)) track_branches(src, directory) elif tag_exists(src): pass else: error("Specified source branch does not exist: {0}".format(src), exit=True) # Determine if the destination branch needs to be created create_dst_branch = False if branch_exists(dst, local_only=False, directory=directory): if not branch_exists(dst, local_only=True, directory=directory): debug("Tracking destination branch: {0}".format(dst)) track_branches(dst, directory) else: create_dst_branch = True # Determine if the destination patches branch needs to be created create_dst_patches_branch = False dst_patches = "patches/" + dst if branch_exists(dst_patches, False, directory=directory): if not branch_exists(dst_patches, True, directory=directory): track_branches(dst_patches, directory) else: create_dst_patches_branch = True # Summarize if interactive: info("Summary of changes:") if create_dst_branch: info( " " * 22 + "- The specified destination branch, " + ansi("boldon") + dst + ansi("reset") + ", does not exist; it will be created from the source " "branch " + ansi("boldon") + src + ansi("reset") ) if create_dst_patches_branch: info( " " * 22 + "- The destination patches branch, " + ansi("boldon") + dst_patches + ansi("reset") + ", does not exist; it will be created" ) info(" " * 22 + "- The working branch will be set to " + ansi("boldon") + dst + ansi("reset")) if not maybe_continue(): error("Answered no to continue, aborting.", exit=True) # Make changes to the layout current_branch = get_current_branch(directory) try: # Change to the src branch checkout(src, directory=directory) # Create the dst branch if needed if create_dst_branch: create_branch(dst, changeto=True, directory=directory) else: checkout(dst, directory=directory) # Create the dst patches branch if needed if create_dst_patches_branch: create_branch(dst_patches, orphaned=True, directory=directory) # Create the starting config data if it does not exist patches_ls = ls_tree(dst_patches, directory=directory) if "patches.conf" not in patches_ls: # Patches config not setup, set it up config = { "parent": src, "previous": "", "base": get_commit_hash(dst, directory=directory), "trim": "", "trimbase": "", } set_patch_config(dst_patches, config, directory=directory) else: config = get_patch_config(dst_patches, directory=directory) if config["parent"] != src: warning("Updated parent to '{0}' from '{1}'".format(src, config["parent"])) config["parent"] = src config["base"] = get_commit_hash(dst, directory=directory) set_patch_config(dst_patches, config, directory=directory) # Command successful, do not switch back to previous branch current_branch = None finally: if current_branch is not None: checkout(current_branch, directory=directory)
def import_patches(directory=None): # Get current branch current_branch = get_current_branch(directory) if current_branch is None: error("Could not determine current branch.", exit=True) # Construct the patches branch name patches_branch = 'patches/' + current_branch # Ensure the patches branch exists and is tracked if branch_exists(patches_branch, False, directory=directory): if not branch_exists(patches_branch, True, directory=directory): track_branches(patches_branch, directory) else: error("The patches branch ({0}) does not ".format(patches_branch) + "exist, did you use git-bloom-branch?", exit=True) # Create a swap space tmp_dir = tempfile.mkdtemp() try: # Get parent branch and base commit from patches branch config = get_patch_config(patches_branch, directory) parent_branch, commit = config['parent'], config['base'] if commit != get_commit_hash(current_branch, directory): debug( "commit != get_commit_hash(current_branch, directory)" ) debug( "{0} != get_commit_hash({1}, {2}) != {3}".format( commit, current_branch, directory, get_commit_hash(current_branch, directory) ) ) os.system('git log') warning( "The current commit is not the same as the most recent " "rebase commit." ) warning( "This might mean that you have committed since the last " "time you did:" ) warning( " 'git-bloom-patch rebase' or 'git-bloom-patch remove'" ) warning( "Make sure you export any commits you want to save first:" ) warning(" 'git-bloom-patch export'") error("Patches not exported", exit=True) # Checkout to the patches branch checkout(patches_branch, directory=directory) # Copy the patches to a temp location patches = list_patches(directory) if len(patches) == 0: debug("No patches in the patches branch, nothing to do") return -1 # Indicates that nothing was done tmp_dir_patches = [] for patch in patches: tmp_dir_patches.append(os.path.join(tmp_dir, patch)) if directory is not None: patch = os.path.join(directory, patch) shutil.copy(patch, tmp_dir) # Now checkout back to the original branch and import them checkout(current_branch, directory=directory) try: cmd = 'git am {0}*.patch'.format(tmp_dir + os.sep) execute_command(cmd, cwd=directory) except subprocess.CalledProcessError as e: warning("Failed to apply one or more patches for the " "'{0}' branch.".format(str(e))) info('', use_prefix=False) info('', use_prefix=False) info(">>> Resolve any conflicts and when you have resolved this " "problem run 'git am --resolved' and then exit the " "shell using 'exit 0'. <<<", use_prefix=False) info(" To abort use 'exit 1'", use_prefix=False) if 'bash' in os.environ['SHELL']: ret = subprocess.call([ "/bin/bash", "-l", "-c", """\ /bin/bash --rcfile <(echo "if [ -f /etc/bashrc ]; then source /etc/bashrc; fi; \ if [ -f ~/.bashrc ]; then source ~/.bashrc; fi;PS1='(bloom)$PS1'") -i""" ]) else: ret = subprocess.call("$SHELL", shell=True) if ret != 0: error("User failed to resolve patch conflicts, exiting.") sys.exit("'git-bloom-patch import' aborted.") info("User reports that conflicts have been resolved, continuing.") # Notify the user info("Applied {0} patches".format(len(patches))) finally: if current_branch: checkout(current_branch, directory=directory) if os.path.exists(tmp_dir): shutil.rmtree(tmp_dir)
def open_pull_request(track, repository, distro, distro_file_url=ROS_DISTRO_FILE): # Get the diff distro_file_url = distro_file_url.format(distro) distro_file_raw = fetch_distro_file(distro_file_url) distro_file = yaml.load(distro_file_raw) if repository in distro_file['repositories']: orig_version = distro_file['repositories'][repository].get('version', None) else: orig_version = None udiff_patch_file, updated_distro_file = generate_ros_distro_diff(track, repository, distro, distro_file_url, distro_file, distro_file_raw) if None in [udiff_patch_file, updated_distro_file]: # There were no changes, no pull request required return None version = distro_file['repositories'][repository]['version'] # Determine if the distro file is hosted on github... distro_file_url = distro_file_url.format(distro) gh_org, gh_repo, gh_branch, gh_path = get_gh_info(distro_file_url) if None in [gh_org, gh_repo, gh_branch, gh_path]: warning("Automated pull request only available via github.com") return # Determine if we have a .netrc file gh_username = None try: netrc_hosts = netrc.netrc().hosts except Exception as e: error("Failed to parse ~/.netrc file: {0}".format(e)) warning("Skipping the pull request...") return for host in netrc_hosts.keys(): if 'github.com' in host: gh_username = netrc_hosts[host][0] gh_password = netrc_hosts[host][2] if None in [gh_username, gh_password]: error("Either the github username or github password is not set in the ~/.netrc file.") warning("Skipping the pull request...") return # Check for fork info(fmt("@{bf}@!==> @|@!Checking for rosdistro fork on github...")) gh_user_repos = fetch_github_api('https://api.github.com/users/{0}/repos'.format(gh_username)) if gh_user_repos is None: error("Failed to get a list of repositories for user: '******'".format(gh_username)) warning("Skipping the pull request...") return if 'rosdistro' not in [x['name'] for x in gh_user_repos if 'name' in x]: warning("Github user '{0}' does not have a fork ".format(gh_username) + "of the {0}:{1} repository, create one?".format(gh_org, gh_repo)) if not maybe_continue(): warning("Skipping the pull request...") return # Create a fork create_fork(gh_org, gh_repo, gh_username, gh_password) # Clone the fork info(fmt("@{bf}@!==> @|@!" + "Cloning {0}/{1}...".format(gh_username, gh_repo))) temp_dir = tempfile.mkdtemp() new_branch = None title = "{0}: {1} in '{2}' [bloom]".format(repository, version, os.path.basename(gh_path)) body = """\ Increasing version of package(s) in repository `{0}`: - previous version: `{1}` - new version: `{2}` - distro file: `{3}` - bloom version: `{4}` """.format(repository, orig_version or 'null', version, gh_path, bloom.__version__) with change_directory(temp_dir): def _my_run(cmd): info(fmt("@{bf}@!==> @|@!" + str(cmd))) out = check_output(cmd, stderr=subprocess.STDOUT, shell=True) if out: info(out, use_prefix=False) _my_run('git clone https://github.com/{0}/{1}.git'.format(gh_username, gh_repo)) with change_directory(gh_repo): _my_run('git remote add bloom https://github.com/{0}/{1}.git'.format(gh_org, gh_repo)) _my_run('git remote update') _my_run('git fetch') track_branches() branches = get_branches() new_branch = 'bloom-patch-{0}' count = 0 while new_branch.format(count) in branches: count += 1 new_branch = new_branch.format(count) # Final check info(fmt("@{cf}Pull Request Title: @{yf}" + title)) info(fmt("@{cf}Pull Request Body : \n@{yf}" + body)) msg = fmt("@!Open a @|@{cf}pull request@| @!@{kf}from@| @!'@|@!@{bf}" + "{gh_username}/{gh_repo}:{new_branch}".format(**locals()) + "@|@!' @!@{kf}into@| @!'@|@!@{bf}" + "{gh_org}/{gh_repo}:{gh_branch}".format(**locals()) + "@|@!'?") info(msg) if not maybe_continue(): warning("Skipping the pull request...") return _my_run('git checkout -b {0} bloom/{1}'.format(new_branch, gh_branch)) with open('{0}'.format(gh_path), 'w') as f: info(fmt("@{bf}@!==> @|@!Writing new distribution file: ") + str(gh_path)) f.write(updated_distro_file) _my_run('git add {0}'.format(gh_path)) _my_run('git commit -m "{0}"'.format(title)) _my_run('git push origin {0}'.format(new_branch)) # Open the pull request return create_pull_request(gh_org, gh_repo, gh_username, gh_password, gh_branch, new_branch, title, body)
def open_pull_request(track, repository, distro): # Get the diff release_file = get_release_file(distro) if repository in release_file.repositories: orig_version = release_file.repositories[repository].version else: orig_version = None updated_release_file = generate_ros_distro_diff(track, repository, distro) if updated_release_file is None: # There were no changes, no pull request required return None version = updated_release_file.repositories[repository].version updated_distro_file = yaml_from_release_file(updated_release_file) # Determine if the distro file is hosted on github... gh_org, gh_repo, gh_branch, gh_path = get_gh_info( get_release_file_url(distro)) if None in [gh_org, gh_repo, gh_branch, gh_path]: warning("Automated pull request only available via github.com") return # Get the github user name gh_username = None bloom_user_path = os.path.join(os.path.expanduser('~'), '.bloom_user') if os.path.exists(bloom_user_path): with open(bloom_user_path, 'r') as f: gh_username = f.read().strip() gh_username = gh_username or getpass.getuser() response = raw_input("github user name [{0}]: ".format(gh_username)) if response: gh_username = response info( "Would you like bloom to store your github user name (~/.bloom_user)?" ) if maybe_continue(): with open(bloom_user_path, 'w') as f: f.write(gh_username) else: with open(bloom_user_path, 'w') as f: f.write(' ') warning( "If you want to have bloom store it in the future remove the ~/.bloom_user file." ) # Get the github password gh_password = getpass.getpass("github password (This is not stored):") if not gh_password or not gh_username: error("Either the github username or github password is not set.") warning("Skipping the pull request...") return # Check for fork info(fmt("@{bf}@!==> @|@!Checking for rosdistro fork on github...")) gh_user_repos = fetch_github_api( 'https://api.github.com/users/{0}/repos'.format(gh_username), use_pagination=True) if gh_user_repos is None: error("Failed to get a list of repositories for user: '******'".format( gh_username)) warning("Skipping the pull request...") return if 'rosdistro' not in [x['name'] for x in gh_user_repos if 'name' in x]: warning( "Github user '{0}' does not have a fork ".format(gh_username) + "of the {0}:{1} repository, create one?".format(gh_org, gh_repo)) if not maybe_continue(): warning("Skipping the pull request...") return # Create a fork create_fork(gh_org, gh_repo, gh_username, gh_password) # Clone the fork info( fmt("@{bf}@!==> @|@!" + "Cloning {0}/{1}...".format(gh_username, gh_repo))) temp_dir = tempfile.mkdtemp() new_branch = None title = "{0}: {1} in '{2}' [bloom]".format(repository, version, gh_path) body = """\ Increasing version of package(s) in repository `{0}`: - previous version: `{1}` - new version: `{2}` - distro file: `{3}` - bloom version: `{4}` """.format(repository, orig_version or 'null', version, gh_path, bloom.__version__) with change_directory(temp_dir): def _my_run(cmd): info(fmt("@{bf}@!==> @|@!" + str(cmd))) # out = check_output(cmd, stderr=subprocess.STDOUT, shell=True) out = None from subprocess import call call(cmd, shell=True) if out: info(out, use_prefix=False) _my_run('git clone https://github.com/{0}/{1}.git'.format( gh_username, gh_repo)) with change_directory(gh_repo): _my_run( 'git remote add bloom https://github.com/{0}/{1}.git'.format( gh_org, gh_repo)) _my_run('git remote update') _my_run('git fetch') track_branches() branches = get_branches() new_branch = 'bloom-{repository}-{count}' count = 0 while new_branch.format(repository=repository, count=count) in branches: count += 1 new_branch = new_branch.format(repository=repository, count=count) # Final check info(fmt("@{cf}Pull Request Title: @{yf}" + title)) info(fmt("@{cf}Pull Request Body : \n@{yf}" + body)) msg = fmt( "@!Open a @|@{cf}pull request@| @!@{kf}from@| @!'@|@!@{bf}" + "{gh_username}/{gh_repo}:{new_branch}".format(**locals()) + "@|@!' @!@{kf}into@| @!'@|@!@{bf}" + "{gh_org}/{gh_repo}:{gh_branch}".format(**locals()) + "@|@!'?") info(msg) if not maybe_continue(): warning("Skipping the pull request...") return _my_run('git checkout -b {0} bloom/{1}'.format( new_branch, gh_branch)) with open('{0}'.format(gh_path), 'w') as f: info( fmt("@{bf}@!==> @|@!Writing new distribution file: ") + str(gh_path)) f.write(updated_distro_file) _my_run('git add {0}'.format(gh_path)) _my_run('git commit -m "{0}"'.format(title)) _my_run('git push origin {0}'.format(new_branch)) # Open the pull request return create_pull_request(gh_org, gh_repo, gh_username, gh_password, gh_branch, new_branch, title, body)
def import_upstream(tarball_path, patches_path, version, name, replace): # If there is not tarball at the given path, fail if not os.path.exists(tarball_path): error("Specified archive does not exists: '{0}'".format(tarball_path), exit=True) # If either version or name are not provided, guess from archive name if not version or not name: # Parse tarball name tarball_file = os.path.basename(tarball_path) ending = None if tarball_file.endswith(".tar.gz"): ending = ".tar.gz" elif tarball_file.endswith(".zip"): ending = ".zip" else: error("Cannot detect type of archive: '{0}'".format(tarball_file), exit=True) tarball_file = tarball_file[: -len(ending)] split_tarball_file = tarball_file.split("-") if len(split_tarball_file) < 2 and not version or len(split_tarball_file) < 1: error("Cannot detect name and/or version from archive: '{0}'".format(tarball_file), exit=True) if not name and len(split_tarball_file) == 1: name = split_tarball_file[0] elif not name and len(split_tarball_file) == 1: name = "-".join(split_tarball_file[:-1]) if not version and len(split_tarball_file) < 2: error( "Cannot detect version from archive: '{0}'".format(tarball_file) + " and the version was not spcified.", exit=True, ) version = version if version else split_tarball_file[-1] # Check if the patches_path (if given) exists patches_path_dict = None if patches_path: patches_path_dict = ls_tree("bloom", patches_path) if not patches_path_dict: error("Given patches path '{0}' does not exist in bloom branch.".format(patches_path), exit=True) # Do version checking version_check(version) # Check for existing tags upstream_tag = "upstream/{0}".format(version) if tag_exists(upstream_tag): if not replace: error("Tag '{0}' already exists, use --replace to override it.".format(upstream_tag), exit=True) warning("Removing tag: '{0}'".format(upstream_tag)) delete_tag(upstream_tag) if not get_git_clone_state(): delete_remote_tag(upstream_tag) name_tag = "{0}/{1}".format(name or "upstream", version) if name_tag != upstream_tag and tag_exists(name_tag): if not replace: error("Tag '{0}' already exists, use --replace to override it.".format(name_tag), exit=True) warning("Removing tag: '{0}'".format(name_tag)) delete_tag(name_tag) if not get_git_clone_state(): delete_remote_tag(name_tag) # If there is not upstream branch, create one if not branch_exists("upstream"): info("Creating upstream branch.") create_branch("upstream", orphaned=True) else: track_branches(["upstream"]) # Import the given tarball info("Importing archive into upstream branch...") import_tarball(tarball_path, "upstream", version, name) # Handle patches_path if patches_path: import_patches(patches_path, patches_path_dict, "upstream", version) # Create tags with inbranch("upstream"): info("Creating tag: '{0}'".format(upstream_tag)) create_tag(upstream_tag) if name_tag != upstream_tag: info("Creating tag: '{0}'".format(name_tag)) create_tag(name_tag)
def open_pull_request(track, repository, distro): # Get the diff release_file = get_release_file(distro) if repository in release_file.repositories: orig_version = release_file.repositories[repository].version else: orig_version = None updated_release_file = generate_ros_distro_diff(track, repository, distro) if updated_release_file is None: # There were no changes, no pull request required return None version = updated_release_file.repositories[repository].version updated_distro_file = yaml_from_release_file(updated_release_file) # Determine if the distro file is hosted on github... gh_org, gh_repo, gh_branch, gh_path = get_gh_info(get_release_file_url(distro)) if None in [gh_org, gh_repo, gh_branch, gh_path]: warning("Automated pull request only available via github.com") return # Get the github user name gh_username = None bloom_user_path = os.path.join(os.path.expanduser("~"), ".bloom_user") if os.path.exists(bloom_user_path): with open(bloom_user_path, "r") as f: gh_username = f.read().strip() gh_username = gh_username or getpass.getuser() response = raw_input("github user name [{0}]: ".format(gh_username)) if response: gh_username = response info("Would you like bloom to store your github user name (~/.bloom_user)?") if maybe_continue(): with open(bloom_user_path, "w") as f: f.write(gh_username) else: with open(bloom_user_path, "w") as f: f.write(" ") warning("If you want to have bloom store it in the future remove the ~/.bloom_user file.") # Get the github password gh_password = getpass.getpass("github password (This is not stored):") if not gh_password or not gh_username: error("Either the github username or github password is not set.") warning("Skipping the pull request...") return # Check for fork info(fmt("@{bf}@!==> @|@!Checking for rosdistro fork on github...")) gh_user_repos = fetch_github_api("https://api.github.com/users/{0}/repos".format(gh_username), use_pagination=True) if gh_user_repos is None: error("Failed to get a list of repositories for user: '******'".format(gh_username)) warning("Skipping the pull request...") return if "rosdistro" not in [x["name"] for x in gh_user_repos if "name" in x]: warning( "Github user '{0}' does not have a fork ".format(gh_username) + "of the {0}:{1} repository, create one?".format(gh_org, gh_repo) ) if not maybe_continue(): warning("Skipping the pull request...") return # Create a fork create_fork(gh_org, gh_repo, gh_username, gh_password) # Clone the fork info(fmt("@{bf}@!==> @|@!" + "Cloning {0}/{1}...".format(gh_username, gh_repo))) temp_dir = tempfile.mkdtemp() new_branch = None title = "{0}: {1} in '{2}' [bloom]".format(repository, version, gh_path) body = """\ Increasing version of package(s) in repository `{0}`: - previous version: `{1}` - new version: `{2}` - distro file: `{3}` - bloom version: `{4}` """.format( repository, orig_version or "null", version, gh_path, bloom.__version__ ) with change_directory(temp_dir): def _my_run(cmd): info(fmt("@{bf}@!==> @|@!" + str(cmd))) # out = check_output(cmd, stderr=subprocess.STDOUT, shell=True) out = None from subprocess import call call(cmd, shell=True) if out: info(out, use_prefix=False) _my_run("git clone https://github.com/{0}/{1}.git".format(gh_username, gh_repo)) with change_directory(gh_repo): _my_run("git remote add bloom https://github.com/{0}/{1}.git".format(gh_org, gh_repo)) _my_run("git remote update") _my_run("git fetch") track_branches() branches = get_branches() new_branch = "bloom-{repository}-{count}" count = 0 while new_branch.format(repository=repository, count=count) in branches: count += 1 new_branch = new_branch.format(repository=repository, count=count) # Final check info(fmt("@{cf}Pull Request Title: @{yf}" + title)) info(fmt("@{cf}Pull Request Body : \n@{yf}" + body)) msg = fmt( "@!Open a @|@{cf}pull request@| @!@{kf}from@| @!'@|@!@{bf}" + "{gh_username}/{gh_repo}:{new_branch}".format(**locals()) + "@|@!' @!@{kf}into@| @!'@|@!@{bf}" + "{gh_org}/{gh_repo}:{gh_branch}".format(**locals()) + "@|@!'?" ) info(msg) if not maybe_continue(): warning("Skipping the pull request...") return _my_run("git checkout -b {0} bloom/{1}".format(new_branch, gh_branch)) with open("{0}".format(gh_path), "w") as f: info(fmt("@{bf}@!==> @|@!Writing new distribution file: ") + str(gh_path)) f.write(updated_distro_file) _my_run("git add {0}".format(gh_path)) _my_run('git commit -m "{0}"'.format(title)) _my_run("git push origin {0}".format(new_branch)) # Open the pull request return create_pull_request(gh_org, gh_repo, gh_username, gh_password, gh_branch, new_branch, title, body)
def import_upstream(cwd, tmp_dir, args): # Ensure the bloom and upstream branches are tracked locally track_branches(['bloom', 'upstream']) # Create a clone of the bloom_repo to help isolate the activity bloom_repo_clone_dir = os.path.join(tmp_dir, 'bloom_clone') os.makedirs(bloom_repo_clone_dir) os.chdir(bloom_repo_clone_dir) bloom_repo = get_vcs_client('git', bloom_repo_clone_dir) bloom_repo.checkout('file://{0}'.format(cwd)) # Ensure the bloom and upstream branches are tracked from the original track_branches(['bloom', 'upstream']) ### Fetch the upstream tag upstream_repo = None upstream_repo_dir = os.path.join(tmp_dir, 'upstream_repo') # If explicit svn url just export and git-import-orig if args.explicit_svn_url is not None: if args.upstream_version is None: error("'--explicit-svn-url' must be specified with " "'--upstream-version'") return 1 info("Checking out upstream at version " + ansi('boldon') + \ str(args.upstream_version) + ansi('reset') + \ " from repository at " + ansi('boldon') + \ str(args.explicit_svn_url) + ansi('reset')) upstream_repo = get_vcs_client('svn', upstream_repo_dir) retcode = try_vcstools_checkout(upstream_repo, args.explicit_svn_url) if retcode != 0: return retcode meta = { 'name': None, 'version': args.upstream_version, 'type': 'manual' } # Else fetching from bloom configs else: # Check for a bloom branch check_for_bloom() # Parse the bloom config file upstream_url, upstream_type, upstream_branch = parse_bloom_conf() # If the upstream_tag is specified, don't search just fetch upstream_repo = get_vcs_client(upstream_type, upstream_repo_dir) if args.upstream_tag is not None: warning("Using specified upstream tag '" + args.upstream_tag + "'") if upstream_type == 'svn': upstream_url += '/tags/' + args.upstream_tag upstream_tag = '' else: upstream_tag = args.upstream_tag retcode = try_vcstools_checkout(upstream_repo, upstream_url, upstream_tag) if retcode != 0: return retcode meta = { 'name': None, 'version': args.upstream_tag, 'type': 'manual' } # We have to search for the upstream tag else: if args.upstream_devel is not None: warning("Overriding the bloom.conf upstream branch with " + args.upstream_devel) devel_branch = args.upstream_devel else: devel_branch = upstream_branch if args.upstream_version is None: meta = auto_upstream_checkout( upstream_repo, upstream_url, devel_branch ) else: meta = { 'version': args.upstream_version } if type(meta) not in [dict] and meta != 0: return meta ### Export the repository version = meta['version'] # Export the repository to a tar ball tarball_prefix = 'upstream-' + str(version) info('Exporting version {0}'.format(version)) tarball_path = os.path.join(tmp_dir, tarball_prefix) if upstream_repo.get_vcs_type_name() == 'svn': upstream_repo.export_repository('', tarball_path) else: if args.upstream_tag is not None: upstream_repo.export_repository(args.upstream_tag, tarball_path) else: upstream_repo.export_repository(version, tarball_path) # Get the gbp version elements from either the last tag or the default last_tag = get_last_tag_by_version() if last_tag == '': gbp_major, gbp_minor, gbp_patch = segment_version(version) else: gbp_major, gbp_minor, gbp_patch = \ get_versions_from_upstream_tag(last_tag) info("The latest upstream tag in the release repository is " + ansi('boldon') + last_tag + ansi('reset')) # Ensure the new version is greater than the last tag last_tag_version = '.'.join([gbp_major, gbp_minor, gbp_patch]) if parse_version(version) < parse_version(last_tag_version): warning("""\ Version discrepancy: The upstream version, {0}, is not newer than the previous \ release version, {1}. """.format(version, last_tag_version)) if parse_version(version) <= parse_version(last_tag_version): if args.replace: # Remove the conflicting tag first warning("""\ The upstream version, {0}, is equal to or less than a previous \ import version. Removing conflicting tag before continuing \ because the '--replace' options was specified.\ """.format(version)) else: warning("""\ The upstream version, {0}, is equal to a previous import version. \ git-buildpackage will fail, if you want to replace the existing \ upstream import use the '--replace' option.\ """.format(version)) if args.replace: if tag_exists('upstream/' + version): execute_command('git tag -d {0}'.format('upstream/' + version)) execute_command('git push origin :refs/tags/' '{0}'.format('upstream/' + version)) # Look for upstream branch if not branch_exists('upstream', local_only=True): create_branch('upstream', orphaned=True, changeto=True) # Go to the bloom branch during import bloom_repo.update('bloom') # Detect if git-import-orig is installed tarball_path += '.tar.gz' import_orig(tarball_path, 'upstream', upstream_repo.get_url(), version) # Push changes back to the original bloom repo execute_command('git push --all -f') execute_command('git push --tags')
def import_upstream(tarball_path, patches_path, version, name, replace): # Check for a url and download it url = urlparse(tarball_path) if url.scheme: # Some scheme like http, https, or file... tmp_dir = tempfile.mkdtemp() try: info("Fetching file from url: '{0}'".format(tarball_path)) req = load_url_to_file_handle(tarball_path) tarball_path = os.path.join(tmp_dir, os.path.basename(url.path)) with open(tarball_path, 'wb') as f: chunk_size = 16 * 1024 while True: chunk = req.read(chunk_size) if not chunk: break f.write(chunk) return import_upstream(tarball_path, patches_path, version, name, replace) finally: shutil.rmtree(tmp_dir) # If there is not tarball at the given path, fail if not os.path.exists(tarball_path): error("Specified archive does not exists: '{0}'".format(tarball_path), exit=True) # If either version or name are not provided, guess from archive name if not version or not name: # Parse tarball name tarball_file = os.path.basename(tarball_path) ending = None if tarball_file.endswith('.tar.gz'): ending = '.tar.gz' elif tarball_file.endswith('.zip'): ending = '.zip' else: error("Cannot detect type of archive: '{0}'" .format(tarball_file), exit=True) tarball_file = tarball_file[:-len(ending)] split_tarball_file = tarball_file.split('-') if len(split_tarball_file) < 2 and not version or len(split_tarball_file) < 1: error("Cannot detect name and/or version from archive: '{0}'" .format(tarball_file), exit=True) if not name and len(split_tarball_file) == 1: name = split_tarball_file[0] elif not name and len(split_tarball_file) == 1: name = '-'.join(split_tarball_file[:-1]) if not version and len(split_tarball_file) < 2: error("Cannot detect version from archive: '{0}'" .format(tarball_file) + " and the version was not spcified.", exit=True) version = version if version else split_tarball_file[-1] # Check if the patches_path (if given) exists patches_path_dict = None if patches_path: patches_path_dict = ls_tree(BLOOM_CONFIG_BRANCH, patches_path) if not patches_path_dict: error("Given patches path '{0}' does not exist in bloom branch." .format(patches_path), exit=True) # Do version checking version_check(version) # Check for existing tags upstream_tag = 'upstream/{0}'.format(version) if tag_exists(upstream_tag): if not replace: error("Tag '{0}' already exists, use --replace to override it." .format(upstream_tag), exit=True) warning("Removing tag: '{0}'".format(upstream_tag)) delete_tag(upstream_tag) if not get_git_clone_state(): delete_remote_tag(upstream_tag) name_tag = '{0}/{1}'.format(name or 'upstream', version) if name_tag != upstream_tag and tag_exists(name_tag): if not replace: error("Tag '{0}' already exists, use --replace to override it." .format(name_tag), exit=True) warning("Removing tag: '{0}'".format(name_tag)) delete_tag(name_tag) if not get_git_clone_state(): delete_remote_tag(name_tag) # If there is not upstream branch, create one if not branch_exists('upstream'): info("Creating upstream branch.") create_branch('upstream', orphaned=True) else: track_branches(['upstream']) # Import the given tarball info("Importing archive into upstream branch...") import_tarball(tarball_path, 'upstream', version, name) # Handle patches_path if patches_path: import_patches(patches_path, patches_path_dict, 'upstream', version) # Create tags with inbranch('upstream'): info("Creating tag: '{0}'".format(upstream_tag)) create_tag(upstream_tag) if name_tag != upstream_tag: info("Creating tag: '{0}'".format(name_tag)) create_tag(name_tag)
def execute_branch(src, dst, interactive, directory=None): """ Changes to the destination branch, creates branch and patches/branch if they do not exist. If the dst branch does not exist yet, then it is created by branching the current working branch or the specified SRC_BRANCH. If the patches/dst branch branch does not exist yet then it is created. If the branches are created successful, then the working branch will be set to the dst branch, otherwise the working branch will remain unchanged. :param src: source branch from which to copy :param dst: destination branch :param interactive: if True actions are summarized before committing :param directory: directory in which to preform this action :raises: subprocess.CalledProcessError if any git calls fail """ # Determine if the srouce branch exists if src is None: error("No source specified and/or not a branch currently", exit=True) if branch_exists(src, local_only=False, directory=directory): if not branch_exists(src, local_only=True, directory=directory): debug("Tracking source branch: {0}".format(src)) track_branches(src, directory) elif tag_exists(src): pass else: error("Specified source branch does not exist: {0}".format(src), exit=True) # Determine if the destination branch needs to be created create_dst_branch = False if branch_exists(dst, local_only=False, directory=directory): if not branch_exists(dst, local_only=True, directory=directory): debug("Tracking destination branch: {0}".format(dst)) track_branches(dst, directory) else: create_dst_branch = True # Determine if the destination patches branch needs to be created create_dst_patches_branch = False dst_patches = 'patches/' + dst if branch_exists(dst_patches, False, directory=directory): if not branch_exists(dst_patches, True, directory=directory): track_branches(dst_patches, directory) else: create_dst_patches_branch = True # Summarize if interactive: info("Summary of changes:") if create_dst_branch: info(" " * 22 + "- The specified destination branch, " + ansi('boldon') + dst + ansi('reset') + ", does not exist; it will be created from the source " "branch " + ansi('boldon') + src + ansi('reset')) if create_dst_patches_branch: info(" " * 22 + "- The destination patches branch, " + ansi('boldon') + dst_patches + ansi('reset') + ", does not exist; it will be created") info(" " * 22 + "- The working branch will be set to " + ansi('boldon') + dst + ansi('reset')) if not maybe_continue(): error("Answered no to continue, aborting.", exit=True) # Make changes to the layout current_branch = get_current_branch(directory) try: # Change to the src branch checkout(src, directory=directory) # Create the dst branch if needed if create_dst_branch: create_branch(dst, changeto=True, directory=directory) else: checkout(dst, directory=directory) # Create the dst patches branch if needed if create_dst_patches_branch: create_branch(dst_patches, orphaned=True, directory=directory) # Create the starting config data if it does not exist patches_ls = ls_tree(dst_patches, directory=directory) if 'patches.conf' not in patches_ls: # Patches config not setup, set it up config = { 'parent': src, 'previous': '', 'base': get_commit_hash(dst, directory=directory), 'trim': '', 'trimbase': '' } set_patch_config(dst_patches, config, directory=directory) else: config = get_patch_config(dst_patches, directory=directory) if config['parent'] != src: warning("Updated parent to '{0}' from '{1}'".format( src, config['parent'])) config['parent'] = src config['base'] = get_commit_hash(dst, directory=directory) set_patch_config(dst_patches, config, directory=directory) # Command successful, do not switch back to previous branch current_branch = None finally: if current_branch is not None: checkout(current_branch, directory=directory)