def get_index_url(): global _rosdistro_index_commit index_url = rosdistro.get_index_url() pr = urlparse(index_url) if pr.netloc == 'raw.github.com': # Try to determine what the commit hash was tokens = [x for x in pr.path.split('/') if x] if len(tokens) <= 3: debug("Failed to get commit for rosdistro index file: index url") debug(tokens) return index_url owner = tokens[0] repo = tokens[1] branch = tokens[2] gh = get_github_interface(quiet=True) if gh is None: # Failed to get it with auth, try without auth (may fail) gh = Github(username=None, auth=None) try: data = gh.get_branch(owner, repo, branch) except GithubException: debug(traceback.format_exc()) debug("Failed to get commit for rosdistro index file: api") return index_url _rosdistro_index_commit = data.get('commit', {}).get('sha', None) if _rosdistro_index_commit is not None: info("ROS Distro index file associate with commit '{0}'" .format(_rosdistro_index_commit)) else: debug("Failed to get commit for rosdistro index file: json") return index_url
def merge_packages(pkgs_dict, get_subs_fn, os_name, os_version, ros_distro, install_prefix, native=False): all_subs = {} for path, pkg in pkgs_dict.items(): try: subs = get_subs_fn(pkg, os_name, os_version, ros_distro, install_prefix, native) all_subs[subs['Name']] = subs except Exception as exc: debug(traceback.format_exc()) error(type(exc).__name__ + ": " + str(exc), exit=True) except (KeyboardInterrupt, EOFError): sys.exit(1) repo_header = {} cnt = 0 for pkg, sub in all_subs.items(): try: if (0 == cnt): repo_header['Package'] = convertToUnicode( sanitize_package_name('tesseract_core')) repo_header['DebianInc'] = sub['DebianInc'] repo_header['format'] = sub['format'] repo_header['InstallationPrefix'] = sub['InstallationPrefix'] repo_header['Maintainer'] = sub['Maintainers'][0] repo_header['Maintainers'] = sub['Maintainers'] repo_header['BuildDepends'] = sub['BuildDepends'] repo_header['Homepage'] = sub['Homepage'] repo_header['Copyright'] = sub['Copyright'] repo_header['debhelper_version'] = sub['debhelper_version'] repo_header['changelogs'] = sub['changelogs'] repo_header['Distribution'] = sub['Distribution'] else: repo_header['Maintainers'].join(', '.join(sub['Maintainers'])) repo_header['BuildDepends'].extend(sub['BuildDepends']) repo_header['Copyright'].join(sub['Copyright']) cnt = cnt + 1 except Exception as exc: debug(traceback.format_exc()) error(type(exc).__name__ + ": " + str(exc), exit=True) except (KeyboardInterrupt, EOFError): sys.exit(1) # Remove build depends in this repository repo_header['BuildDepends'] = [ x for x in repo_header['BuildDepends'] if x not in all_subs.keys() ] # Remove duplicates repo_header['BuildDepends'] = list( dict.fromkeys(repo_header['BuildDepends'])) # TODO Remove Duplicates from repo_header['Maintainers'] all_subs[repo_header['Package']] = repo_header return all_subs
def set_upstream(upstream_repo, upstream_repo_type, upstream_repo_branch): # Check for a bloom branch if branch_exists('bloom', False): # Found a bloom branch debug("Found a bloom branch, checking out.") # Check out the bloom branch checkout('bloom') else: # No bloom branch found, create one create_branch('bloom', changeto=True) # Now set the upstream using the bloom config cmd = 'git config -f bloom.conf bloom.upstream "{0}"'.format(upstream_repo) execute_command(cmd) cmd = 'git config -f bloom.conf ' \ + 'bloom.upstreamtype "{0}"'.format(upstream_repo_type) execute_command(cmd) cmd = 'git config -f bloom.conf ' \ + 'bloom.upstreambranch "{0}"'.format(upstream_repo_branch) execute_command(cmd) execute_command('git add bloom.conf') if has_changes(): cmd = 'git commit -m "bloom branch update by git-bloom-config"' execute_command(cmd) else: debug("No chages, nothing to commit.")
def remove_patches(directory=None): # Get the current branch current_branch = get_current_branch(directory) # Ensure the current branch is valid if current_branch is None: error("Could not determine current branch, are you in a git repo?") return 1 # Construct the patches branch patches_branch = 'patches/' + current_branch try: # See if the patches branch exists if branch_exists(patches_branch, False, directory=directory): if not branch_exists(patches_branch, True, directory=directory): track_branches(patches_branch, directory) else: error("No patches branch (" + patches_branch + ") found, cannot " "remove patches.") return 1 # Get the parent branch from the patches branch config = get_patch_config(patches_branch, directory=directory) parent, spec = config['parent'], config['base'] if None in [parent, spec]: error("Could not retrieve patches info.") return 1 debug("Removing patches from " + current_branch + " back to base " "commit " + spec) # Reset this branch using git reset --hard spec execute_command('git reset --hard ' + spec, cwd=directory) finally: if current_branch: checkout(current_branch, directory=directory) return 0
def _set_trim_sub_dir(sub_dir, force, config, directory): debug("_set_trim_sub_dir(" + str(sub_dir) + ", " + str(force) + ", " + \ str(config) + ", " + str(directory) + ")") if sub_dir is not None: if config['trim'] != '' and config['trim'] != sub_dir: warning("You are trying to set the trim sub directory to " + \ sub_dir + ", but it is already set to " + \ config['trim'] + ".") if not force: warning("Changing the sud directory is not advised. " "If you are sure you want to do this, use " "'--force'") return None else: warning("Forcing the change of the sub directory.") # Make the sub_dir absolute git_root = get_root(directory) sub_dir_abs = os.path.join(git_root, sub_dir) # Make sure it is a directory if not os.path.isdir(sub_dir_abs): error("The given sub directory, (" + sub_dir + ") does not " "exist in the git repository at " + git_root) return None # Set the trim sub directory config['trim'] = sub_dir return config
def run_generator(generator, arguments): try: gen = generator try_execute('generator handle arguments', '', gen.handle_arguments, arguments) try_execute('generator summarize', '', gen.summarize) if arguments.interactive: if not maybe_continue('y'): error("Answered no to continue, aborting.", exit=True) for branch_args in generator.get_branching_arguments(): parsed_branch_args = parse_branch_args(branch_args, arguments.interactive) destination, source, interactive = parsed_branch_args # Summarize branch command msg = summarize_branch_cmd(destination, source, interactive) ### Run pre - branch - post # Pre branch try_execute('generator pre_branch', msg, gen.pre_branch, destination, source) # Branch try_execute('git-bloom-branch', msg, execute_branch, source, destination, interactive) # Post branch try_execute('generator post_branch', msg, gen.post_branch, destination, source) ### Run pre - export patches - post # Pre patch try_execute('generator pre_export_patches', msg, gen.pre_export_patches, destination) # Export patches try_execute('git-bloom-patch export', msg, export_patches) # Post branch try_execute('generator post_export_patches', msg, gen.post_export_patches, destination) ### Run pre - rebase - post # Pre rebase try_execute('generator pre_rebase', msg, gen.pre_rebase, destination) # Rebase ret = try_execute('git-bloom-patch rebase', msg, rebase_patches) # Post rebase try_execute('generator post_rebase', msg, gen.post_rebase, destination) ### Run pre - import patches - post # Pre patch try_execute('generator pre_patch', msg, gen.pre_patch, destination) if ret == 0: # Import patches try_execute('git-bloom-patch import', msg, import_patches) elif ret < 0: debug("Skipping patching because rebase did not run.") # Post branch try_execute('generator post_patch', msg, gen.post_patch, destination) except CommandFailed as err: sys.exit(err.returncode or 1)
def execute_command(cmd, shell=True, autofail=True, silent=True, silent_error=False, cwd=None, return_io=False): """ Executes a given command using vcstools' run_shell_command function. """ io_type = None result = 0 if silent: io_type = PIPE debug(((cwd) if cwd else os.getcwd()) + ":$ " + str(cmd)) p = Popen(cmd, shell=True, cwd=cwd, stdout=io_type, stderr=io_type) out, err = p.communicate() result = p.returncode if result != 0: if not silent_error: error("'execute_command' failed to call '{0}'".format(cmd) + \ " which had a return code ({0}):".format(result)) if out: error(" stdout:\n" + ansi('reset') + str(out)) error("end stdout") if err: error(" stderr:\n" + ansi('reset') + str(err)) error("end stderr") if autofail: raise CalledProcessError(cmd=cmd, output=out, returncode=result) if return_io: return result, out, err else: return result
def execute_command(cmd, shell=True, autofail=True, silent=True, silent_error=False, cwd=None, return_io=False): """ Executes a given command using vcstools' run_shell_command function. """ out_io = None err_io = None result = 0 if silent: out_io = PIPE err_io = STDOUT debug(((cwd) if cwd else os.getcwd()) + ":$ " + str(cmd)) p = Popen(cmd, shell=True, cwd=cwd, stdout=out_io, stderr=err_io) out, err = p.communicate() if out is not None and not isinstance(out, str): out = out.decode('utf-8') if err is not None and not isinstance(err, str): err = err.decode('utf-8') result = p.returncode if result != 0: if not silent_error: error("'execute_command' failed to call '{0}'".format(cmd) + " which had a return code ({0}):".format(result)) error("```") info(out, use_prefix=False) error("```") if autofail: raise CalledProcessError(result, cmd) if return_io: return result, out, err else: return result
def execute_command(cmd, shell=True, autofail=True, silent=True, silent_error=False, cwd=None, return_io=False): """ Executes a given command using Popen. """ out_io = None err_io = None result = 0 if silent: out_io = PIPE err_io = STDOUT debug(((cwd) if cwd else os.getcwd()) + ":$ " + str(cmd)) env = __get_env_for_cmd(cmd) p = Popen(cmd, shell=shell, cwd=cwd, stdout=out_io, stderr=err_io, env=env) out, err = p.communicate() if out is not None and not isinstance(out, str): out = out.decode('utf-8') if err is not None and not isinstance(err, str): err = err.decode('utf-8') result = p.returncode if result != 0: if not silent_error: error("'execute_command' failed to call '{0}'".format(cmd) + " which had a return code ({0}):".format(result)) error("```") info(out, use_prefix=False) error("```") if autofail: raise CalledProcessError(result, cmd) if return_io: return result, out, err else: return result
def resolve_rosdep_key(key, os_name, os_version, ros_distro=None, ignored=None, retry=True): ignored = ignored or [] ctx = create_default_installer_context() try: installer_key = ctx.get_default_os_installer_key(os_name) except KeyError: BloomGenerator.exit("Could not determine the installer for '{0}'".format(os_name)) installer = ctx.get_installer(installer_key) ros_distro = ros_distro or DEFAULT_ROS_DISTRO view = get_view(os_name, os_version, ros_distro) try: return resolve_more_for_os(key, view, installer, os_name, os_version) except (KeyError, ResolutionError) as exc: debug(traceback.format_exc()) if key in ignored: return None, None, None if isinstance(exc, KeyError): error("Could not resolve rosdep key '{0}'".format(key)) returncode = code.GENERATOR_NO_SUCH_ROSDEP_KEY else: error("Could not resolve rosdep key '{0}' for distro '{1}':".format(key, os_version)) info(str(exc), use_prefix=False) returncode = code.GENERATOR_NO_ROSDEP_KEY_FOR_DISTRO if retry: error("Try to resolve the problem with rosdep and then continue.") if maybe_continue(): update_rosdep() invalidate_view_cache() return resolve_rosdep_key(key, os_name, os_version, ros_distro, ignored, retry=True) BloomGenerator.exit("Failed to resolve rosdep key '{0}', aborting.".format(key), returncode=returncode)
def execute_command(cmd, shell=True, autofail=True, silent=True, silent_error=False, cwd=None, return_io=False): """ Executes a given command using vcstools' run_shell_command function. """ out_io = None err_io = None result = 0 if silent: out_io = PIPE err_io = STDOUT debug(((cwd) if cwd else os.getcwd()) + ":$ " + str(cmd)) p = Popen(cmd, shell=True, cwd=cwd, stdout=out_io, stderr=err_io) out, err = p.communicate() result = p.returncode if result != 0: if not silent_error: error("'execute_command' failed to call '{0}'".format(cmd) + " which had a return code ({0}):".format(result)) error("```") info(out, use_prefix=False) error("```") if autofail: raise CalledProcessError(result, cmd) if return_io: return result, out, err else: return result
def generate_ros_distro_diff(track, repository, distro, distro_file_url, distro_file, distro_file_raw): with inbranch('upstream'): # Check for package.xml(s) try: from catkin_pkg.packages import find_packages except ImportError: debug(traceback.format_exc()) error("catkin_pkg was not detected, please install it.", file=sys.stderr, exit=True) packages = find_packages(os.getcwd()) if len(packages) == 0: warning("No packages found, will not generate 'package: path' entries for rosdistro.") track_dict = get_tracks_dict_raw()['tracks'][track] last_version = track_dict['last_version'] release_inc = track_dict['release_inc'] if repository not in distro_file['repositories']: global _user_provided_release_url distro_file['repositories'][repository] = {'url': _user_provided_release_url or ''} distro_file['repositories'][repository]['version'] = '{0}-{1}'.format(last_version, release_inc) if packages and (len(packages) > 1 or packages.keys()[0] != '.'): distro_file['repositories'][repository]['packages'] = {} for path, package in packages.iteritems(): if os.path.basename(path) == package.name: distro_file['repositories'][repository]['packages'][package.name] = None else: distro_file['repositories'][repository]['packages'][package.name] = path distro_file_name = os.path.join('release', distro_file_url.split('/')[-1]) distro_dump = yaml.dump(distro_file, indent=2, default_flow_style=False) if distro_file_raw != distro_dump: udiff = difflib.unified_diff(distro_file_raw.splitlines(), distro_dump.splitlines(), fromfile=distro_file_name, tofile=distro_file_name) temp_dir = tempfile.mkdtemp() version = distro_file['repositories'][repository]['version'] udiff_file = os.path.join(temp_dir, repository + '-' + version + '.patch') udiff_raw = '' info("Unified diff for the ROS distro file located at '{0}':".format(udiff_file)) for line in udiff: if line.startswith('@@'): udiff_raw += line line = fmt('@{cf}' + line) if line.startswith('+'): if not line.startswith('+++'): line += '\n' udiff_raw += line line = fmt('@{gf}' + line) if line.startswith('-'): if not line.startswith('---'): line += '\n' udiff_raw += line line = fmt('@{rf}' + line) if line.startswith(' '): line += '\n' udiff_raw += line info(line, use_prefix=False, end='') with open(udiff_file, 'w+') as f: f.write(udiff_raw) return udiff_file, distro_dump else: warning("This release resulted in no changes to the ROS distro file...") return None, None
def _set_trim_sub_dir(sub_dir, force, config, directory): debug("_set_trim_sub_dir(" + str(sub_dir) + ", " + str(force) + ", " + str(config) + ", " + str(directory) + ")") if sub_dir is not None: if config['trim'] != '' and config['trim'] != sub_dir: warning("You are trying to set the trim sub directory to " + sub_dir + ", but it is already set to " + config['trim'] + ".") if not force: warning("Changing the sud directory is not advised. " "If you are sure you want to do this, use " "'--force'") return None else: warning("Forcing the change of the sub directory.") # Make the sub_dir absolute git_root = get_root(directory) sub_dir_abs = os.path.join(git_root, sub_dir) # Make sure it is a directory if not os.path.isdir(sub_dir_abs): error("The given sub directory, (" + sub_dir + ") does not " "exist in the git repository at " + git_root) return None # Set the trim sub directory config['trim'] = sub_dir return config
def main(args=None, get_subs_fn=None): get_subs_fn = get_subs_fn or get_subs _place_template_files = True _process_template_files = True package_path = os.getcwd() if args is not None: package_path = args.package_path or os.getcwd() _place_template_files = args.place_template_files _process_template_files = args.process_template_files pkgs_dict = find_packages(package_path) if len(pkgs_dict) == 0: sys.exit("No packages found in path: '{0}'".format(package_path)) if len(pkgs_dict) > 1: sys.exit("Multiple packages found, " "this tool only supports one package at a time.") os_data = create_default_installer_context().get_os_name_and_version() os_name, os_version = os_data ros_distro = os.environ.get("ROS_DISTRO", "indigo") # Allow args overrides os_name = args.os_name or os_name os_version = args.os_version or os_version ros_distro = args.ros_distro or ros_distro # Summarize info( fmt("@!@{gf}==> @|") + fmt( "Generating debs for @{cf}%s:%s@| for package(s) %s" % (os_name, os_version, [p.name for p in pkgs_dict.values()]) ) ) for path, pkg in pkgs_dict.items(): template_files = None try: subs = get_subs_fn(pkg, os_name, os_version, ros_distro) if _place_template_files: # Place template files place_template_files(path) if _process_template_files: # Just process existing template files template_files = process_template_files(path, subs) if not _place_template_files and not _process_template_files: # If neither, do both place_template_files(path) template_files = process_template_files(path, subs) if template_files is not None: for template_file in template_files: os.remove(os.path.normpath(template_file)) except Exception as exc: debug(traceback.format_exc()) error(type(exc).__name__ + ": " + str(exc), exit=True) except (KeyboardInterrupt, EOFError): sys.exit(1)
def checkout(reference, raise_exc=False, directory=None, show_git_status=True): """ Returns True if the checkout to a the reference was successful, else False :param reference: branch, tag, or commit hash to checkout to :param directory: directory in which to run this command :returns: True if the checkout was successful, else False """ def checkout_summarize(fail_msg, branch, directory): branch = "(no branch)" if branch is None else branch directory = os.getcwd() if directory is None else directory error( "Failed to checkout to '{0}'".format(str(reference)) + " because the working directory {0}".format(str(fail_msg)) ) debug(" Working directory: '{0}'".format(str(directory))) debug(" Working branch: '{0}'".format(str(branch))) debug(" Has local changes: '{0}'".format(str(changes))) debug(" Has untrakced files: '{0}'".format(str(untracked))) pdb_hook() if not bloom.util._quiet and show_git_status: print("\n++ git status:\n") os.system("git status") return 1 debug("Checking out to " + str(reference)) if reference == get_current_branch(directory): debug("Requested checkout reference is the same as the current branch") return 0 fail_msg = "" git_root = get_root(directory) if git_root is not None: changes = has_changes(directory) untracked = has_untracked_files(directory) branch = get_current_branch(directory) else: fail_msg = "is not a git repository" if fail_msg == "" and changes: fail_msg = "has local changes" if fail_msg == "" and untracked: fail_msg = "has untracked files" try: if not changes and not untracked: execute_command('git checkout "{0}"'.format(str(reference)), cwd=directory) except CalledProcessError as err: fail_msg = "CalledProcessError: " + str(err) if raise_exc: checkout_summarize(fail_msg, branch, directory) raise if fail_msg != "": return checkout_summarize(fail_msg, branch, directory) else: return 0
def set_releaser_history(self, history): # Assumes that this is called in the target branch patches_branch = 'patches/' + get_current_branch() debug("Writing release history to '{0}' branch".format(patches_branch)) with inbranch(patches_branch): with open('releaser_history.json', 'w') as f: f.write(json.dumps(history)) execute_command('git add releaser_history.json') if has_changes(): execute_command('git commit -m "Store releaser history"')
def build_debian_pkg(args=None, get_subs_fn=None): get_subs_fn = get_subs_fn or get_subs _place_template_files = True _process_template_files = True package_path = os.getcwd() if args is not None: package_path = args.package_path or os.getcwd() _place_template_files = args.place_template_files _process_template_files = args.process_template_files pkgs_dict = find_packages(package_path) if len(pkgs_dict) == 0: sys.exit("No packages found in path: '{0}'".format(package_path)) # if len(pkgs_dict) > 1: # sys.exit("Multiple packages found, " # "this tool only supports one package at a time.") os_data = create_default_installer_context().get_os_name_and_version() os_name, os_version = os_data ros_distro = os.environ.get('ROS_DISTRO', 'indigo') # Allow args overrides os_name = args.os_name or os_name os_version = args.os_version or os_version ros_distro = args.ros_distro or ros_distro install_prefix = args.install_prefix or "/opt" # Summarize info(fmt("@!@{gf}==> @|") + fmt("Generating debs for @{cf}%s:%s@| for package(s) %s" % (os_name, os_version, [p.name for p in pkgs_dict.values()]))) # Test Creating single all_subs = merge_packages(pkgs_dict, get_subs_fn, os_name, os_version, ros_distro, install_prefix, args.native) path = '' build_type = 'cmake' try: if _place_template_files: # Place template files place_template_files(path, build_type) if _process_template_files: # Just process existing template files template_files = process_template_files(path, all_subs) if not _place_template_files and not _process_template_files: # If neither, do both place_template_files(path, build_type) template_files = process_template_files(path, all_subs) if template_files is not None: for template_file in template_files: os.remove(os.path.normpath(template_file)) except Exception as exc: debug(traceback.format_exc()) error(type(exc).__name__ + ": " + str(exc), exit=True) except (KeyboardInterrupt, EOFError): sys.exit(1)
def checkout(reference, raise_exc=False, directory=None, show_git_status=True): """ Returns True if the checkout to a the reference was successful, else False :param reference: branch, tag, or commit hash to checkout to :param directory: directory in which to run this command :returns: True if the checkout was successful, else False """ def checkout_summarize(fail_msg, branch, directory): branch = '(no branch)' if branch is None else branch directory = os.getcwd() if directory is None else directory error("Failed to checkout to '{0}'".format(str(reference)) + " because the working directory {0}".format(str(fail_msg))) debug(" Working directory: '{0}'".format(str(directory))) debug(" Working branch: '{0}'".format(str(branch))) debug(" Has local changes: '{0}'".format(str(changes))) debug(" Has untrakced files: '{0}'".format(str(untracked))) pdb_hook() if not bloom.util._quiet and show_git_status: info('\n++ git status:\n', use_prefix=False) os.system('git status') return False debug("Checking out to " + str(reference)) if reference == get_current_branch(directory): debug("Requested checkout reference is the same as the current branch") return True fail_msg = '' git_root = get_root(directory) if git_root is not None: changes = has_changes(directory) untracked = has_untracked_files(directory) branch = get_current_branch(directory) or 'could not determine branch' else: fail_msg = "is not a git repository" if fail_msg == '' and changes: fail_msg = "has local changes" if fail_msg == '' and untracked: fail_msg = "has untracked files" try: if not changes and not untracked: execute_command('git checkout "{0}"'.format(str(reference)), cwd=directory) except CalledProcessError as err: fail_msg = "CalledProcessError: " + str(err) if raise_exc: checkout_summarize(fail_msg, branch, directory) raise if fail_msg != '': return checkout_summarize(fail_msg, branch, directory) else: return True
def match_branches_with_prefix(prefix, get_branches): debug("match_branches_with_prefix(" + str(prefix) + ", " + str(get_branches()) + ")") branches = [] # Match branches existing_branches = get_branches() for branch in existing_branches: if branch.startswith('remotes/origin/'): branch = branch.split('/', 2)[-1] if branch.startswith(prefix): branches.append(branch) return list(set(branches))
def _undo(config, directory): debug("_undo(" + str(config) + ", " + str(directory) + ")") # TODO: handle repo with changes # TODO: handle repo with patches applied if config['trimbase'] == '': debug("Branch has not been trimmed previously, undo not required.") return None # Reset with git-reset execute_command('git reset --hard ' + config['trimbase'], cwd=directory) # Unset the trimbase config['trimbase'] = '' return config
def try_execute(msg, err_msg, func, *args, **kwargs): try: retcode = func(*args, **kwargs) retcode = retcode if retcode is not None else 0 except CalledProcessError as err: print_exc(traceback.format_exc()) error("Error calling {0}: ".format(msg) + str(err)) retcode = err.returncode ret_msg = msg + " returned exit code ({0})".format(str(retcode)) if retcode > 0: error(ret_msg) raise CommandFailed(retcode) elif retcode < 0: debug(ret_msg)
def __place_template_folder(group, src, dst, gbp=False): template_files = pkg_resources.resource_listdir(group, src) # For each template, place for template_file in template_files: if not gbp and os.path.basename(template_file) == 'gbp.conf.em': debug("Skipping template '{0}'".format(template_file)) continue template_path = os.path.join(src, template_file) template_dst = os.path.join(dst, template_file) if pkg_resources.resource_isdir(group, template_path): debug("Recursing on folder '{0}'".format(template_path)) __place_template_folder(group, template_path, template_dst, gbp) else: try: debug("Placing template '{0}'".format(template_path)) template = pkg_resources.resource_string(group, template_path) template_abs_path = pkg_resources.resource_filename(group, template_path) except IOError as err: error("Failed to load template " "'{0}': {1}".format(template_file, str(err)), exit=True) if not os.path.exists(dst): os.makedirs(dst) if os.path.exists(template_dst): debug("Removing existing file '{0}'".format(template_dst)) os.remove(template_dst) with open(template_dst, 'w') as f: if not isinstance(template, str): template = template.decode('utf-8') f.write(template) shutil.copystat(template_abs_path, template_dst)
def __place_template_folder(group, src, dst, gbp=False): template_files = pkg_resources.resource_listdir(group, src) # For each template, place for template_file in template_files: if not gbp and os.path.basename(template_file) == 'gbp.conf.em': debug("Skipping template '{0}'".format(template_file)) continue template_path = os.path.join(src, template_file) template_dst = os.path.join(dst, template_file) if pkg_resources.resource_isdir(group, template_path): debug("Recursing on folder '{0}'".format(template_path)) __place_template_folder(group, template_path, template_dst, gbp) else: try: debug("Placing template '{0}'".format(template_path)) template = pkg_resources.resource_string(group, template_path) template_abs_path = pkg_resources.resource_filename( group, template_path) except IOError as err: error("Failed to load template " "'{0}': {1}".format(template_file, str(err)), exit=True) if not os.path.exists(dst): os.makedirs(dst) if os.path.exists(template_dst): debug("Removing existing file '{0}'".format(template_dst)) os.remove(template_dst) with open(template_dst, 'w') as f: if not isinstance(template, str): template = template.decode('utf-8') f.write(template) shutil.copystat(template_abs_path, template_dst)
def _undo(config, directory): debug("_undo(" + str(config) + ", " + str(directory) + ")") # TODO: handle repo with changes # TODO: handle repo with patches applied if config['trimbase'] == '': debug("Branch has not been trimmed previously, undo not required.") return None # Reset with git-revert cmt = get_commit_hash(get_current_branch(directory), directory) cmd = 'git revert --no-edit -Xtheirs ' + config['trimbase'] + '..' + cmt execute_command(cmd, cwd=directory) # Unset the trimbase config['trimbase'] = '' return config
def import_patches(directory=None): # Get current branch current_branch = get_current_branch(directory) # Construct the patches branch name patches_branch = 'patches/' + current_branch # Ensure the patches branch exists and is tracked if branch_exists(patches_branch, False, directory=directory): if not branch_exists(patches_branch, True, directory=directory): track_branches(patches_branch, directory) else: error("The patches branch ({0}) does not ".format(patches_branch) + \ "exist, did you use git-bloom-branch?") return code.BRANCH_DOES_NOT_EXIST # Create a swap space tmp_dir = tempfile.mkdtemp() try: # Get parent branch and base commit from patches branch config = get_patch_config(patches_branch, directory) parent_branch, commit = config['parent'], config['base'] if commit != get_commit_hash(current_branch, directory): warning("The current commit is not the same as the most recent " "rebase commit. This might mean that you have committed " "since the last time you did 'git-bloom-patch export'.") return code.PATCHES_NOT_EXPORTED # Checkout to the patches branch checkout(patches_branch, directory=directory) # Copy the patches to a temp location patches = list_patches(directory) if len(patches) == 0: debug("No patches in the patches branch, nothing to do") return code.NOTHING_TO_DO tmp_dir_patches = [] for patch in patches: tmp_dir_patches.append(os.path.join(tmp_dir, patch)) if directory is not None: patch = os.path.join(directory, patch) shutil.copy(patch, tmp_dir) # Now checkout back to the original branch and import them checkout(current_branch, directory=directory) cmd = 'git am {0}*.patch'.format(tmp_dir + os.sep) execute_command(cmd, cwd=directory) # Notify the user info("Applied {0} patches".format(len(patches))) finally: if current_branch: checkout(current_branch, directory=directory) if os.path.exists(tmp_dir): shutil.rmtree(tmp_dir) return 0
def main(args=None, get_subs_fn=None): get_subs_fn = get_subs_fn or get_subs _place_template_files = True _process_template_files = True package_path = os.getcwd() if args is not None: package_path = args.package_path or os.getcwd() _place_template_files = args.place_template_files _process_template_files = args.process_template_files pkgs_dict = find_packages(package_path) if len(pkgs_dict) == 0: sys.exit("No packages found in path: '{0}'".format(package_path)) if len(pkgs_dict) > 1: sys.exit("Multiple packages found, this tool only supports one package at a time.") ros_distro = os.environ.get('ROS_DISTRO', 'groovy') # Allow args overrides ros_distro = args.ros_distro or ros_distro # Summarize info(fmt("@!@{gf}==> @|") + fmt("Generating Homebrew formula for package(s) %s" % ([p.name for p in pkgs_dict.values()]))) for path, pkg in pkgs_dict.items(): template_files = None try: subs = get_subs_fn(pkg, ros_distro) if _place_template_files: # Place template files place_template_files(path) if _process_template_files: # Just process existing template files template_files = process_template_files(path, subs) if not _place_template_files and not _process_template_files: # If neither, do both place_template_files(path) template_files = process_template_files(path, subs) if template_files is not None: for template_file in template_files: os.remove(os.path.normpath(template_file)) except Exception as exc: debug(traceback.format_exc()) error(type(exc).__name__ + ": " + str(exc), exit=True) except (KeyboardInterrupt, EOFError): sys.exit(1)
def try_execute(msg, err_msg, func, *args, **kwargs): retcode = 0 try: retcode = func(*args, **kwargs) retcode = retcode if retcode is not None else 0 except CalledProcessError as err: print_exc(traceback.format_exc()) error("Error calling {0}: {1}".format(msg, str(err))) retcode = err.returncode ret_msg = msg + " returned exit code ({0})".format(str(retcode)) if retcode > 0: error(ret_msg) raise CommandFailed(retcode) elif retcode < 0: debug(ret_msg) return retcode
def _undo(config, directory): debug("_undo(" + str(config) + ", " + str(directory) + ")") # TODO: handle repo with changes # TODO: handle repo with patches applied if config['trimbase'] == '': debug("Branch has not been trimmed previously, undo not required.") return None # Reset with git-revert current_branch = get_current_branch(directory) if current_branch is None: error("Could not determine current branch.", exit=True) cmt = get_commit_hash(current_branch, directory) cmd = 'git revert --no-edit -Xtheirs ' + config['trimbase'] + '..' + cmt execute_command(cmd, cwd=directory) # Unset the trimbase config['trimbase'] = '' return config
def _undo(config, directory): debug("_undo(" + str(config) + ", " + str(directory) + ")") # TODO: handle repo with changes # TODO: handle repo with patches applied if config["trimbase"] == "": debug("Branch has not been trimmed previously, undo not required.") return None # Reset with git-revert current_branch = get_current_branch(directory) if current_branch is None: error("Could not determine current branch.", exit=True) cmt = get_commit_hash(current_branch, directory) cmd = "git revert --no-edit -Xtheirs " + config["trimbase"] + ".." + cmt execute_command(cmd, cwd=directory) # Unset the trimbase config["trimbase"] = "" return config
def track_branches(branches=None, directory=None): """ Tracks all specified branches. :param branches: a list of branches that are to be tracked if not already tracked. If this is set to None then all remote branches will be tracked. :param directory: directory in which to run all commands :raises: subprocess.CalledProcessError if git command fails """ if type(branches) == str: branches = [branches] debug("track_branches(" + str(branches) + ", " + str(directory) + ")") if branches == []: return # Save the current branch current_branch = get_current_branch(directory) try: # Get the local branches local_branches = get_branches(local_only=True, directory=directory) # Get the remote and local branches all_branches = get_branches(local_only=False, directory=directory) # Calculate the untracked branches untracked_branches = [] for branch in all_branches: if branch.startswith('remotes/'): if branch.count('/') >= 2: branch = '/'.join(branch.split('/')[2:]) if branch not in local_branches: untracked_branches.append(branch) # Prune any untracked branches by specified branches if branches is not None: branches_to_track = [] for untracked in untracked_branches: if untracked in branches: branches_to_track.append(untracked) else: branches_to_track = untracked_branches # Track branches debug("Tracking branches: " + str(branches_to_track)) for branch in branches_to_track: checkout(branch, directory=directory) finally: if current_branch: checkout(current_branch, directory=directory)
def rebase_patches(without_git_rebase=True, directory=None): ### Ensure a clean/valid working environment ret = ensure_clean_working_env(git_status=True, directory=directory) if ret != 0: return ret ### Make sure we need to actually call this # Get the current branch current_branch = get_current_branch(directory) # Get the patches branch patches_branch = "patches/" + current_branch # Get the current patches.conf config = get_patch_config(patches_branch, directory=directory) # Get the current upstream commit hash upstream_commit_hash = get_commit_hash(config["parent"], directory) # If the current upstream commit hash is the same as the stored one, noop if upstream_commit_hash == config["previous"]: debug( "Nothing to do: Current branch (" + current_branch + ")'s " "base commit hash is the same as the source branch (" + config["parent"] + ")'s commit hash." ) debug(" Did you forget to update the parent branch first?") debug( " Updating the parent branch can be done by calling " "'git-bloom-patch rebase' on it, or 'git-bloom-import-upsteam'" " if the parent branch is the upstream branch." ) return 0 else: debug( "rebase_patches: " + upstream_commit_hash + " == " + config["previous"] + ": " + str(upstream_commit_hash == config["previous"]) ) ### Execute the rebase if without_git_rebase: non_git_rebase(config["parent"], directory=directory) else: git_rebase(config["parent"], directory=directory) ### Update the patches information # Get the latest configs config = get_patch_config(patches_branch, directory) # Set the base to the current hash (before patches) config["base"] = get_commit_hash(current_branch, directory) # Set the new upstream hash to the previous upstream hash config["previous"] = get_commit_hash(config["parent"], directory) # Clear the trimbase (it needs to be reapplied) config["trimbase"] = "" # Write the new configs set_patch_config(patches_branch, config, directory) return 0
def resolve_rosdep_key(key, os_name, os_version, ros_distro=None, ignored=None, retry=True): ignored = ignored or [] ctx = create_default_installer_context() try: installer_key = ctx.get_default_os_installer_key(os_name) except KeyError: BloomGenerator.exit( "Could not determine the installer for '{0}'".format(os_name)) installer = ctx.get_installer(installer_key) ros_distro = ros_distro or DEFAULT_ROS_DISTRO view = get_view(os_name, os_version, ros_distro) try: return resolve_more_for_os(key, view, installer, os_name, os_version) except (KeyError, ResolutionError) as exc: debug(traceback.format_exc()) if key in ignored: return None, None, None if isinstance(exc, KeyError): error("Could not resolve rosdep key '{0}'".format(key)) returncode = code.GENERATOR_NO_SUCH_ROSDEP_KEY else: error( "Could not resolve rosdep key '{0}' for distro '{1}':".format( key, os_version)) info(str(exc), use_prefix=False) returncode = code.GENERATOR_NO_ROSDEP_KEY_FOR_DISTRO if retry: error("Try to resolve the problem with rosdep and then continue.") if maybe_continue(): update_rosdep() invalidate_view_cache() return resolve_rosdep_key(key, os_name, os_version, ros_distro, ignored, retry=True) BloomGenerator.exit( "Failed to resolve rosdep key '{0}', aborting.".format(key), returncode=returncode)
def export_patches(directory=None): ### Ensure a clean/valid working environment ensure_clean_working_env(git_status=True, directory=directory) # Get current branch current_branch = get_current_branch(directory) if current_branch is None: error("Could not determine current branch.", exit=True) # Construct the patches branch name patches_branch = 'patches/' + current_branch # Ensure the patches branch exists if not branch_exists(patches_branch, False, directory=directory): error("The patches branch ({0}) does not ".format(patches_branch) + "exist, did you use git-bloom-branch?", exit=True) try: # Get parent branch and base commit from patches branch config = get_patch_config(patches_branch, directory) if config is None: error("Failed to get patches information.", exit=True) # Checkout to the patches branch checkout(patches_branch, directory=directory) # Notify the user debug("Exporting patches from " "{0}...{1}".format(config['base'], current_branch)) # Remove all the old patches if len(list_patches(directory)) > 0: cmd = 'git rm ./*.patch' execute_command(cmd, cwd=directory) # Create the patches using git format-patch cmd = "git format-patch -M -B " \ "{0}...{1}".format(config['base'], current_branch) execute_command(cmd, cwd=directory) # Report of the number of patches created patches_list = list_patches(directory) debug("Created {0} patches".format(len(patches_list))) # Clean up and commit if len(patches_list) > 0: cmd = 'git add ./*.patch' execute_command(cmd, cwd=directory) if has_changes(directory): cmd = 'git commit -m "Updating patches."' execute_command(cmd, cwd=directory) finally: if current_branch: checkout(current_branch, directory=directory)
def get_index_url(): global _rosdistro_index_commit, _rosdistro_index_original_branch index_url = rosdistro.get_index_url() pr = urlparse(index_url) if pr.netloc in ['raw.github.com', 'raw.githubusercontent.com']: # Try to determine what the commit hash was tokens = [x for x in pr.path.split('/') if x] if len(tokens) <= 3: debug("Failed to get commit for rosdistro index file: index url") debug(tokens) return index_url owner = tokens[0] repo = tokens[1] branch = tokens[2] gh = get_github_interface(quiet=True) if gh is None: # Failed to get it with auth, try without auth (may fail) gh = Github(username=None, auth=None) try: data = gh.get_branch(owner, repo, branch) except GithubException: debug(traceback.format_exc()) debug("Failed to get commit for rosdistro index file: api") return index_url _rosdistro_index_commit = data.get('commit', {}).get('sha', None) if _rosdistro_index_commit is not None: info("ROS Distro index file associate with commit '{0}'".format( _rosdistro_index_commit)) # Also mutate the index_url to use the commit (rather than the moving branch name) base_info = get_gh_info(index_url) base_branch = base_info['branch'] rosdistro_index_commit = _rosdistro_index_commit # Copy global into local for substitution middle = "{org}/{repo}".format(**base_info) index_url = index_url.replace( "{pr.netloc}/{middle}/{base_branch}/".format(**locals()), "{pr.netloc}/{middle}/{rosdistro_index_commit}/".format( **locals())) info("New ROS Distro index url: '{0}'".format(index_url)) _rosdistro_index_original_branch = base_branch else: debug("Failed to get commit for rosdistro index file: json") return index_url
def match_branches_with_prefix(prefix, get_branches, prune=False): debug("match_branches_with_prefix(" + str(prefix) + ", " + str(get_branches()) + ")") branches = [] # Match branches existing_branches = get_branches() for branch in existing_branches: if branch.startswith('remotes/origin/'): branch = branch.split('/', 2)[-1] if branch.startswith(prefix): branches.append(branch) branches = list(set(branches)) if prune: # Prune listed branches by packages in latest upstream with inbranch('upstream'): pkg_names, version, pkgs_dict = get_package_data('upstream') for branch in branches: if branch.split(prefix)[-1].strip('/') not in pkg_names: branches.remove(branch) return branches
def _trim(config, force, directory): debug("_trim(" + str(config) + ", " + str(force) + ", " + \ str(directory) + ")") if config['trimbase'] != '': warning("It looks like the trim operation has already been done, " "nested trimming is not supported.") if force: warning("Proceeding anyways because of '--force'") else: warning("If you would like to continue anyways use '--force'") return None config['trimbase'] = get_commit_hash(get_current_branch(directory)) tmp_dir = tempfile.mkdtemp() try: # Buckup trim sub directory git_root = get_root() sub_dir = os.path.join(git_root, config['trim']) storage = os.path.join(tmp_dir, config['trim']) shutil.copytree(sub_dir, storage) # Clear out the git repo execute_command('git rm -rf ./*', cwd=directory) # Copy the sub directory back for item in os.listdir(storage): src = os.path.join(storage, item) dst = os.path.join(git_root, item) if os.path.isdir(src): shutil.copytree(src, dst) else: shutil.copy(src, dst) # Stage execute_command('git add ./*', cwd=directory) # Commit cmd = 'git commit -m "Trimmed the branch to only the ' + \ config['trim'] + ' sub directory"' execute_command(cmd, cwd=directory) # Update the patch base to be this commit config['base'] = get_commit_hash(get_current_branch(directory)) finally: if os.path.exists(tmp_dir): shutil.rmtree(tmp_dir) return config
def checkout_summarize(fail_msg, branch, directory): branch = '(no branch)' if branch is None else branch directory = os.getcwd() if directory is None else directory error("Failed to checkout to '{0}'".format(str(reference)) + " because the working directory {0}".format(str(fail_msg))) debug(" Working directory: '{0}'".format(str(directory))) debug(" Working branch: '{0}'".format(str(branch))) debug(" Has local changes: '{0}'".format(str(changes))) debug(" Has untrakced files: '{0}'".format(str(untracked))) pdb_hook() if not bloom.util._quiet and show_git_status: info('\n++ git status:\n', use_prefix=False) os.system('git status') return False
def rebase_patches(without_git_rebase=True, directory=None): ### Ensure a clean/valid working environment ensure_clean_working_env(git_status=True, directory=directory) ### Make sure we need to actually call this # Get the current branch current_branch = get_current_branch(directory) if current_branch is None: error("Could not determine current branch.", exit=True) # Get the patches branch patches_branch = 'patches/' + current_branch # Get the current patches.conf config = get_patch_config(patches_branch, directory=directory) ### Execute the rebase if without_git_rebase: non_git_rebase(config['parent'], directory=directory) else: git_rebase(config['parent'], directory=directory) ### Update the patches information # Get the latest configs config = get_patch_config(patches_branch, directory) # Set the base to the current hash (before patches) current_branch_ = get_current_branch(directory) debug('Current branch: ' + current_branch_ or 'could not determine branch') config['base'] = get_commit_hash(current_branch_, directory) debug('New current commit hash after rebase: ' + config['base']) # Set the new upstream hash to the previous upstream hash config['previous'] = get_commit_hash(config['parent'], directory) debug('New parent commit hash after rebase: ' + config['previous']) # Clear the trimbase (it needs to be reapplied) config['trimbase'] = '' # Write the new configs set_patch_config(patches_branch, config, directory)
def checkout_summarize(fail_msg, branch, directory): branch = '(no branch)' if branch is None else branch directory = os.getcwd() if directory is None else directory error("Failed to checkout to '{0}'".format(str(reference)) + " because the working directory {0}".format(str(fail_msg))) debug(" Working directory: '{0}'".format(str(directory))) debug(" Working branch: '{0}'".format(str(branch))) debug(" Has local changes: '{0}'".format(str(changes))) debug(" Has untrakced files: '{0}'".format(str(untracked))) pdb_hook() if not bloom.util._quiet and show_git_status: print('\n++ git status:\n') os.system('git status') return False
def check_for_bloom(): """ Checks for the bloom branch, else looks for and converts the catkin branch. Then it checks for the bloom branch and that it contains a bloom.conf file. """ if branch_exists('catkin') and not branch_exists('bloom'): # Found catkin branch, migrate it to bloom info('catkin branch detected, up converting to the bloom branch') convert_catkin_to_bloom() return # Check for bloom.conf if os.path.exists('bloom'): error("File or directory bloom prevents checking out to the bloom " "branch, remove it.") sys.exit(1) if not branch_exists('bloom'): debug('no bloom branch') not_a_bloom_release_repo() with inbranch('bloom'): if not os.path.exists(os.path.join(os.getcwd(), 'bloom.conf')): debug('no bloom.conf file') not_a_bloom_release_repo()
def resolve_rosdep_key(key, os_name, os_version, ros_distro=None, ignored=None, retry=True): ignored = ignored or [] if os_name not in default_installers: BloomGenerator.exit( "Could not determine the installer for '{0}'".format(os_name)) installer = get_installer(default_installers[os_name][0]) ros_distro = ros_distro or DEFAULT_ROS_DISTRO view = get_view(os_name, os_version, ros_distro) try: return resolve_for_os(key, view, installer, os_name, os_version) except (KeyError, ResolutionError) as exc: debug(traceback.format_exc()) if key in ignored: return None if isinstance(exc, KeyError): error("Could not resolve rosdep key '{0}'".format(key)) else: error( "Could not resolve rosdep key '{0}' for distro '{1}':".format( key, os_version)) info(str(exc), use_prefix=False) if retry: error("Try to resolve the problem with rosdep and then continue.") if maybe_continue(): update_rosdep() invalidate_view_cache() return resolve_rosdep_key(key, os_name, os_version, ros_distro, ignored, retry=True) BloomGenerator.exit( "Failed to resolve rosdep key '{0}', aborting.".format(key))
def remove_patches(directory=None): # Get the current branch current_branch = get_current_branch(directory) if current_branch is None: error("Could not determine current branch.", exit=True) # Ensure the current branch is valid if current_branch is None: error("Could not determine current branch, are you in a git repo?", exit=True) # Construct the patches branch patches_branch = 'patches/' + current_branch try: # See if the patches branch exists if branch_exists(patches_branch, False, directory=directory): if not branch_exists(patches_branch, True, directory=directory): track_branches(patches_branch, directory) else: error("No patches branch (" + patches_branch + ") found, cannot " "remove patches.", exit=True) # Get the parent branch from the patches branch config = get_patch_config(patches_branch, directory=directory) parent, spec = config['parent'], config['base'] if None in [parent, spec]: error("Could not retrieve patches info.", exit=True) debug("Removing patches from " + current_branch + " back to base " "commit " + spec) # Reset this branch using git revert --no-edit spec current_commit = get_commit_hash(current_branch, directory) command_spec = spec + '..' + current_commit execute_command('git revert --no-edit -Xtheirs ' + command_spec, cwd=directory) # Update the base config['base'] = get_commit_hash(current_branch, directory) set_patch_config(patches_branch, config, directory=directory) finally: if current_branch: checkout(current_branch, directory=directory)
def remove_patches(directory=None): # Get the current branch current_branch = get_current_branch(directory) if current_branch is None: error("Could not determine current branch.", exit=True) # Ensure the current branch is valid if current_branch is None: error("Could not determine current branch, are you in a git repo?", exit=True) # Construct the patches branch patches_branch = 'patches/' + current_branch try: # See if the patches branch exists if branch_exists(patches_branch, False, directory=directory): if not branch_exists(patches_branch, True, directory=directory): track_branches(patches_branch, directory) else: error("No patches branch (" + patches_branch + ") found, cannot " "remove patches.", exit=True) # Get the parent branch from the patches branch config = get_patch_config(patches_branch, directory=directory) parent, spec = config['parent'], config['base'] if None in [parent, spec]: error("Could not retrieve patches info.", exit=True) debug("Removing patches from " + current_branch + " back to base " "commit " + spec) # Reset this branch using git revert --no-edit spec current_commit = get_commit_hash(current_branch, directory) command_spec = spec + '..' + current_commit execute_command( 'git revert --no-edit -Xtheirs ' + command_spec, cwd=directory ) # Update the base config['base'] = get_commit_hash(current_branch, directory) set_patch_config(patches_branch, config, directory=directory) finally: if current_branch: checkout(current_branch, directory=directory)
def resolve_rosdep_key( key, os_name, os_version, ros_distro=None, ignored=None, retry=True ): ignored = ignored or [] if os_name not in default_installers: BloomGenerator.exit("Could not determine the installer for '{0}'" .format(os_name)) installer = get_installer(default_installers[os_name][0]) ros_distro = ros_distro or DEFAULT_ROS_DISTRO view = get_view(os_name, os_version, ros_distro) try: return resolve_for_os(key, view, installer, os_name, os_version) except (KeyError, ResolutionError) as exc: debug(traceback.format_exc()) if key in ignored: return None if isinstance(exc, KeyError): error("Could not resolve rosdep key '{0}'".format(key)) else: error("Could not resolve rosdep key '{0}' for distro '{1}':" .format(key, os_version)) info(str(exc), use_prefix=False) if retry: error("Try to resolve the problem with rosdep and then continue.") if maybe_continue(): update_rosdep() invalidate_view_cache() return resolve_rosdep_key(key, os_name, os_version, ros_distro, ignored, retry=True) BloomGenerator.exit("Failed to resolve rosdep key '{0}', aborting." .format(key))
def convert_old_bloom_conf(prefix=None): prefix = prefix if prefix is not None else 'convert' tracks_dict = get_tracks_dict_raw() track = prefix track_count = 0 while track in tracks_dict['tracks']: track_count += 1 track = prefix + str(track_count) track_dict = copy.copy(DEFAULT_TEMPLATE) cmd = 'git config -f bloom.conf bloom.upstream' upstream_repo = check_output(cmd, shell=True).strip() cmd = 'git config -f bloom.conf bloom.upstreamtype' upstream_type = check_output(cmd, shell=True).strip() try: cmd = 'git config -f bloom.conf bloom.upstreambranch' upstream_branch = check_output(cmd, shell=True).strip() except subprocess.CalledProcessError: upstream_branch = '' for key in template_entry_order: if key == 'vcs_uri': track_dict[key] = upstream_repo continue if key == 'vcs_type': track_dict[key] = upstream_type continue if key == 'vcs_uri': track_dict[key] = upstream_branch or None continue track_dict[key] = track_dict[key].default debug('Converted bloom.conf:') with open('bloom.conf', 'r') as f: debug(f.read()) debug('To this track:') debug(str({track: track_dict})) tracks_dict['tracks'][track] = track_dict write_tracks_dict_raw(tracks_dict) execute_command('git rm bloom.conf', shell=True) execute_command('git commit -m "Removed bloom.conf"', shell=True) # Now move the old bloom branch into master upconvert_bloom_to_config_branch()
from bloom.logging import debug from bloom.logging import error from bloom.logging import fmt from bloom.logging import info from debian_generator import generate_substitutions_from_package from debian_generator import merge_packages from debian_generator import place_template_files from debian_generator import process_template_files from bloom.util import get_distro_list_prompt try: from rosdep2 import create_default_installer_context except ImportError: debug(traceback.format_exc()) error("rosdep was not detected, please install it.", exit=True) try: from catkin_pkg.packages import find_packages except ImportError: debug(traceback.format_exc()) error("catkin_pkg was not detected, please install it.", exit=True) def prepare_arguments(parser): add = parser.add_argument add('package_path', nargs='?', help="path to or containing the package.xml of a package") action = parser.add_mutually_exclusive_group(required=False) add = action.add_argument
def perform_release(repository, track, distro, new_track, interactive, pretend): release_repo = get_release_repo(repository, distro) with change_directory(release_repo.get_path()): # Check to see if the old bloom.conf exists if check_for_bloom_conf(repository): # Convert to a track info("Old bloom.conf file detected.") info(fmt("@{gf}@!==> @|Converting to bloom.conf to track")) convert_old_bloom_conf(None if new_track else distro) upconvert_bloom_to_config_branch() # Check that the track is valid tracks_dict = get_tracks_dict_raw() # If new_track, create the new track first if new_track: if not track: error("You must specify a track when creating a new one.", exit=True) if track in tracks_dict['tracks']: warning("Track '{0}' exists, editing...".format(track)) edit_track_cmd(track) tracks_dict = get_tracks_dict_raw() else: # Create a new track called <track>, # copying an existing track if possible, # and overriding the ros_distro warning("Creating track '{0}'...".format(track)) overrides = {'ros_distro': distro} new_track_cmd(track, copy_track='', overrides=overrides) tracks_dict = get_tracks_dict_raw() if track and track not in tracks_dict['tracks']: error("Given track '{0}' does not exist in release repository.". format(track)) error("Available tracks: " + str(tracks_dict['tracks'].keys()), exit=True) elif not track: tracks = tracks_dict['tracks'].keys() # Error out if there are no tracks if len(tracks) == 0: error("Release repository has no tracks.") info("Manually clone the repository:") info(" git clone {0}".format(release_repo.get_url())) info("And then create a new track:") info(" git-bloom-config new <track name>") error("Run again after creating a track.", exit=True) # Error out if there is more than one track if len(tracks) != 1: error("No track specified and there is not just one track.") error("Please specify one of the available tracks: " + str(tracks), exit=True) # Get the only track track = tracks[0] start_summary(track) # Ensure the track is complete track_dict = tracks_dict['tracks'][track] track_dict = update_track(track_dict) tracks_dict['tracks'][track] = track_dict # Set the release repositories' remote if given release_repo_url = track_dict.get('release_repo_url', None) if release_repo_url is not None: info( fmt("@{gf}@!==> @|") + "Setting release repository remote url to '{0}'".format( release_repo_url)) cmd = 'git remote set-url origin ' + release_repo_url info(fmt("@{bf}@!==> @|@!") + str(cmd)) try: subprocess.check_call(cmd, shell=True) except subprocess.CalledProcessError: error("Setting the remote url failed, exiting.", exit=True) # Check for push permissions try: info( fmt("@{gf}@!==> @|Testing for push permission on release repository" )) cmd = 'git remote -v' info(fmt("@{bf}@!==> @|@!") + str(cmd)) subprocess.check_call(cmd, shell=True) # Dry run will authenticate, but not push cmd = 'git push --dry-run' info(fmt("@{bf}@!==> @|@!") + str(cmd)) subprocess.check_call(cmd, shell=True) except subprocess.CalledProcessError: error("Cannot push to remote release repository.", exit=True) # Write the track config before releasing write_tracks_dict_raw(tracks_dict) # Run the release info( fmt("@{gf}@!==> @|") + "Releasing '{0}' using release track '{1}'".format( repository, track)) cmd = 'git-bloom-release ' + str(track) if pretend: cmd += ' --pretend' info(fmt("@{bf}@!==> @|@!" + str(cmd))) try: subprocess.check_call(cmd, shell=True) except subprocess.CalledProcessError: error("Release failed, exiting.", exit=True) info( fmt(_success) + "Released '{0}' using release track '{1}' successfully".format( repository, track)) # Commit the summary update_summary(track, repository, distro) commit_summary() # Check for pushing if interactive: info("Releasing complete, push?") if not maybe_continue(): error("User answered no to continue prompt, aborting.", exit=True) # Push changes to the repository info( fmt("@{gf}@!==> @|") + "Pushing changes to release repository for '{0}'".format( repository)) cmd = 'git push --all' if pretend: cmd += ' --dry-run' info(fmt("@{bf}@!==> @|@!" + str(cmd))) try: subprocess.check_call(cmd, shell=True) except subprocess.CalledProcessError: error( "Pushing changes failed, would you like to add '--force' to 'git push --all'?" ) if not maybe_continue(): error("Pushing changes failed, exiting.", exit=True) cmd += ' --force' info(fmt("@{bf}@!==> @|@!" + str(cmd))) try: subprocess.check_call(cmd, shell=True) except subprocess.CalledProcessError: error("Pushing changes failed, exiting.", exit=True) info(fmt(_success) + "Pushed changes successfully") # Push tags to the repository info( fmt("@{gf}@!==> @|") + "Pushing tags to release repository for '{0}'".format(repository)) cmd = 'git push --tags' if pretend: cmd += ' --dry-run' info(fmt("@{bf}@!==> @|@!" + str(cmd))) try: subprocess.check_call(cmd, shell=True) except subprocess.CalledProcessError: error( "Pushing changes failed, would you like to add '--force' to 'git push --tags'?" ) if not maybe_continue(): error("Pushing tags failed, exiting.", exit=True) cmd += ' --force' info(fmt("@{bf}@!==> @|@!" + str(cmd))) try: subprocess.check_call(cmd, shell=True) except subprocess.CalledProcessError: error("Pushing tags failed, exiting.", exit=True) info(fmt(_success) + "Pushed tags successfully") # Propose github pull request info( fmt("@{gf}@!==> @|") + "Generating pull request to distro file located at '{0}'".format( get_release_file_url(distro))) try: pull_request_url = open_pull_request(track, repository, distro) if pull_request_url: info( fmt(_success) + "Pull request opened at: {0}".format(pull_request_url)) if 'BLOOM_NO_WEBBROWSER' in os.environ and platform.system( ) not in ['Darwin']: webbrowser.open(pull_request_url) else: info( "The release of your packages was successful, but the pull request failed." ) info( "Please manually open a pull request by editing the file here: '{0}'" .format(get_release_file_url(distro))) info(fmt(_error) + "No pull request opened.") except Exception as e: debug(traceback.format_exc()) error("Failed to open pull request: {0} - {1}".format( type(e).__name__, e), exit=True)
def import_patches(directory=None): # Get current branch current_branch = get_current_branch(directory) if current_branch is None: error("Could not determine current branch.", exit=True) # Construct the patches branch name patches_branch = 'patches/' + current_branch # Ensure the patches branch exists and is tracked if branch_exists(patches_branch, False, directory=directory): if not branch_exists(patches_branch, True, directory=directory): track_branches(patches_branch, directory) else: error("The patches branch ({0}) does not ".format(patches_branch) + "exist, did you use git-bloom-branch?", exit=True) # Create a swap space tmp_dir = tempfile.mkdtemp() try: # Get parent branch and base commit from patches branch config = get_patch_config(patches_branch, directory) parent_branch, commit = config['parent'], config['base'] if commit != get_commit_hash(current_branch, directory): debug( "commit != get_commit_hash(current_branch, directory)" ) debug( "{0} != get_commit_hash({1}, {2}) != {3}".format( commit, current_branch, directory, get_commit_hash(current_branch, directory) ) ) os.system('git log') warning( "The current commit is not the same as the most recent " "rebase commit." ) warning( "This might mean that you have committed since the last " "time you did:" ) warning( " 'git-bloom-patch rebase' or 'git-bloom-patch remove'" ) warning( "Make sure you export any commits you want to save first:" ) warning(" 'git-bloom-patch export'") error("Patches not exported", exit=True) # Checkout to the patches branch checkout(patches_branch, directory=directory) # Copy the patches to a temp location patches = list_patches(directory) if len(patches) == 0: debug("No patches in the patches branch, nothing to do") return -1 # Indicates that nothing was done tmp_dir_patches = [] for patch in patches: tmp_dir_patches.append(os.path.join(tmp_dir, patch)) if directory is not None: patch = os.path.join(directory, patch) shutil.copy(patch, tmp_dir) # Now checkout back to the original branch and import them checkout(current_branch, directory=directory) try: cmd = 'git am {0}*.patch'.format(tmp_dir + os.sep) execute_command(cmd, cwd=directory) except subprocess.CalledProcessError as e: warning("Failed to apply one or more patches for the " "'{0}' branch.".format(str(e))) info('', use_prefix=False) info('', use_prefix=False) info(">>> Resolve any conflicts and when you have resolved this " "problem run 'git am --resolved' and then exit the " "shell using 'exit 0'. <<<", use_prefix=False) info(" To abort use 'exit 1'", use_prefix=False) if 'bash' in os.environ['SHELL']: ret = subprocess.call([ "/bin/bash", "-l", "-c", """\ /bin/bash --rcfile <(echo "if [ -f /etc/bashrc ]; then source /etc/bashrc; fi; \ if [ -f ~/.bashrc ]; then source ~/.bashrc; fi;PS1='(bloom)$PS1'") -i""" ]) else: ret = subprocess.call("$SHELL", shell=True) if ret != 0: error("User failed to resolve patch conflicts, exiting.") sys.exit("'git-bloom-patch import' aborted.") info("User reports that conflicts have been resolved, continuing.") # Notify the user info("Applied {0} patches".format(len(patches))) finally: if current_branch: checkout(current_branch, directory=directory) if os.path.exists(tmp_dir): shutil.rmtree(tmp_dir)
def execute_branch(src, dst, interactive, directory=None): """ Changes to the destination branch, creates branch and patches/branch if they do not exist. If the dst branch does not exist yet, then it is created by branching the current working branch or the specified SRC_BRANCH. If the patches/dst branch branch does not exist yet then it is created. If the branches are created successful, then the working branch will be set to the dst branch, otherwise the working branch will remain unchanged. :param src: source branch from which to copy :param dst: destination branch :param interactive: if True actions are summarized before committing :param directory: directory in which to preform this action :raises: subprocess.CalledProcessError if any git calls fail """ # Determine if the srouce branch exists if src is None: error("No source specified and/or not a branch currently", exit=True) if branch_exists(src, local_only=False, directory=directory): if not branch_exists(src, local_only=True, directory=directory): debug("Tracking source branch: {0}".format(src)) track_branches(src, directory) elif tag_exists(src): pass else: error("Specified source branch does not exist: {0}".format(src), exit=True) # Determine if the destination branch needs to be created create_dst_branch = False if branch_exists(dst, local_only=False, directory=directory): if not branch_exists(dst, local_only=True, directory=directory): debug("Tracking destination branch: {0}".format(dst)) track_branches(dst, directory) else: create_dst_branch = True # Determine if the destination patches branch needs to be created create_dst_patches_branch = False dst_patches = 'patches/' + dst if branch_exists(dst_patches, False, directory=directory): if not branch_exists(dst_patches, True, directory=directory): track_branches(dst_patches, directory) else: create_dst_patches_branch = True # Summarize if interactive: info("Summary of changes:") if create_dst_branch: info(" " * 22 + "- The specified destination branch, " + ansi('boldon') + dst + ansi('reset') + ", does not exist; it will be created from the source " "branch " + ansi('boldon') + src + ansi('reset')) if create_dst_patches_branch: info(" " * 22 + "- The destination patches branch, " + ansi('boldon') + dst_patches + ansi('reset') + ", does not exist; it will be created") info(" " * 22 + "- The working branch will be set to " + ansi('boldon') + dst + ansi('reset')) if not maybe_continue(): error("Answered no to continue, aborting.", exit=True) # Make changes to the layout current_branch = get_current_branch(directory) try: # Change to the src branch checkout(src, directory=directory) # Create the dst branch if needed if create_dst_branch: create_branch(dst, changeto=True, directory=directory) else: checkout(dst, directory=directory) # Create the dst patches branch if needed if create_dst_patches_branch: create_branch(dst_patches, orphaned=True, directory=directory) # Create the starting config data if it does not exist patches_ls = ls_tree(dst_patches, directory=directory) if 'patches.conf' not in patches_ls: # Patches config not setup, set it up config = { 'parent': src, 'previous': '', 'base': get_commit_hash(dst, directory=directory), 'trim': '', 'trimbase': '' } set_patch_config(dst_patches, config, directory=directory) else: config = get_patch_config(dst_patches, directory=directory) if config['parent'] != src: warning("Updated parent to '{0}' from '{1}'".format( src, config['parent'])) config['parent'] = src config['base'] = get_commit_hash(dst, directory=directory) set_patch_config(dst_patches, config, directory=directory) # Command successful, do not switch back to previous branch current_branch = None finally: if current_branch is not None: checkout(current_branch, directory=directory)
def import_tarball(tarball_path, target_branch, version, name): if tarball_path.endswith('.zip'): error("Zip archives are not yet supported.", exit=True) # Create the tarfile handle targz = tarfile.open(tarball_path, 'r:gz') with inbranch(target_branch): # Prepare list of members to extract, ignoring some ignores = ('.git', '.gitignore', '.svn', '.hgignore', '.hg', 'CVS') members = targz.getmembers() members = [m for m in members if m.name.split('/')[-1] not in ignores] # Clear out the local branch items = [] for item in os.listdir(os.getcwd()): if item in ['.git', '..', '.']: continue items.append(item) if len(items) > 0: execute_command('git rm -rf ' + ' '.join(items)) # Clear out any untracked files execute_command('git clean -fdx') # Extract the tarball into the clean branch targz.extractall(os.getcwd(), members) # Check for folder nesting (mostly hg) items = [] for item in os.listdir(os.getcwd()): if not item.startswith('.'): items.append(item) tarball_prefix = os.path.basename(tarball_path)[:-len('.tag.gz')] if tarball_prefix in items: debug('Removing nested tarball folder: ' + str(tarball_prefix)) tarball_prefix_path = os.path.join(os.getcwd(), tarball_prefix) for item in os.listdir(tarball_prefix_path): if item in ['.', '..']: continue item_path = os.path.join(os.getcwd(), tarball_prefix, item) debug( 'moving ' + str(item_path) + ' to ' + str(os.path.join(os.getcwd(), item)) ) shutil.move(item_path, os.path.join(os.getcwd(), item)) shutil.rmtree(tarball_prefix_path) else: debug('No nested tarball folder found.') # Commit changes to the repository items = [] for item in os.listdir(os.getcwd()): if item in ['.git', '..', '.']: continue items.append(item) if len(items) > 0: execute_command('git add ' + ' '.join(items)) # Remove any straggling untracked files execute_command('git clean -dXf') # Only if we have local changes commit # (not true if the upstream didn't change any files) if has_changes(): msg = "Imported upstream version '{0}' of '{1}'" msg = msg.format(version, name or 'upstream') cmd = 'git commit -m "{0}"'.format(msg) execute_command(cmd)
def _trim(config, force, directory): debug("_trim(" + str(config) + ", " + str(force) + ", " + str(directory) + ")") if config['trimbase'] != '': warning("It looks like the trim operation has already been done, " "nested trimming is not supported.") if force: warning("Proceeding anyways because of '--force'") else: warning("If you would like to continue anyways use '--force'") return None current_branch = get_current_branch(directory) if current_branch is None: error("Could not determine current branch.", exit=True) config['trimbase'] = get_commit_hash(current_branch) tmp_dir = tempfile.mkdtemp() try: # Buckup trim sub directory git_root = get_root() sub_dir = os.path.join(git_root, config['trim']) storage = os.path.join(tmp_dir, config['trim']) shutil.copytree(sub_dir, storage) # Clear out any untracked files execute_command('git clean -fdx', cwd=directory) # Collect al files (excluding .git) items = [] for item in os.listdir(git_root): if item in ['.git', '..', '.']: continue items.append(item) # Remove and .* files missed by 'git rm -rf *' if len(items) > 0: execute_command('git rm -rf ' + ' '.join(items), cwd=directory) # Copy the sub directory back for item in os.listdir(storage): src = os.path.join(storage, item) dst = os.path.join(git_root, item) if os.path.isdir(src): shutil.copytree(src, dst) else: shutil.copy(src, dst) # Stage execute_command('git add ./*', cwd=directory) # Collect .* files dot_items = [] for item in os.listdir(git_root): if item in ['.git', '..', '.']: continue if item.startswith('.'): dot_items.append(item) # Add any .* files missed by 'git add ./*' if len(dot_items) > 0: execute_command('git add ' + ' '.join(dot_items), cwd=directory) # Remove any straggling untracked files execute_command('git clean -dXf', cwd=directory) # Commit cmd = 'git commit -m "Trimmed the branch to only the ' + \ config['trim'] + ' sub directory"' execute_command(cmd, cwd=directory) # Update the patch base to be this commit current_branch = get_current_branch(directory) if current_branch is None: error("Could not determine current branch.", exit=True) config['base'] = get_commit_hash(current_branch) finally: if os.path.exists(tmp_dir): shutil.rmtree(tmp_dir) return config
def main(args=None, get_subs_fn=None): get_subs_fn = get_subs_fn or get_subs _place_template_files = True _process_template_files = True package_path = os.getcwd() skip_package_names = None if args is not None: package_path = args.package_path or os.getcwd() _place_template_files = args.place_template_files _process_template_files = args.process_template_files # create dependency-skip package name lists skip_name_path = args.skip_package_names or '' if os.path.isfile(os.path.abspath(skip_name_path)): with open(os.path.abspath(skip_name_path), 'r') as f: skip_package_names = [] for line in f: # comment string if line[:1] == '#': continue # add lists skip_package_names.append(line.strip()) pkgs_dict = find_packages(package_path) if len(pkgs_dict) == 0: sys.exit("No packages found in path: '{0}'".format(package_path)) if len(pkgs_dict) > 1: sys.exit("Multiple packages found, " "this tool only supports one package at a time.") os_data = create_default_installer_context().get_os_name_and_version() os_name, os_version = os_data ros_distro = os.environ.get('ROS_DISTRO', 'indigo') # Allow args overrides os_name = args.os_name or os_name os_version = args.os_version or os_version ros_distro = args.ros_distro or ros_distro # Summarize info( fmt("@!@{gf}==> @|") + fmt("Generating debs for @{cf}%s:%s@| for package(s) %s" % (os_name, os_version, [p.name for p in pkgs_dict.values()]))) for path, pkg in pkgs_dict.items(): template_files = None try: subs = get_subs_fn(pkg, os_name, os_version, ros_distro, args.native, skip_package_names) if _place_template_files: # Place template files place_template_files(path, pkg.get_build_type()) if _process_template_files: # Just process existing template files template_files = process_template_files(path, subs) if not _place_template_files and not _process_template_files: # If neither, do both place_template_files(path, pkg.get_build_type()) template_files = process_template_files(path, subs) if template_files is not None: for template_file in template_files: os.remove(os.path.normpath(template_file)) except Exception as exc: debug(traceback.format_exc()) error(type(exc).__name__ + ": " + str(exc), exit=True) except (KeyboardInterrupt, EOFError): sys.exit(1)
def handle_arguments(self, args): """ Hook to handle parsed arguments from argparse """ debug("BloomGenerator.handle_arguments: got args -> " + str(args))