def run_generator(generator, arguments): try: gen = generator try_execute('generator handle arguments', '', gen.handle_arguments, arguments) try_execute('generator summarize', '', gen.summarize) if arguments.interactive: if not maybe_continue('y'): error("Answered no to continue, aborting.", exit=True) for branch_args in generator.get_branching_arguments(): parsed_branch_args = parse_branch_args(branch_args, arguments.interactive) destination, source, interactive = parsed_branch_args # Summarize branch command msg = summarize_branch_cmd(destination, source, interactive) ### Run pre - branch - post # Pre branch try_execute('generator pre_branch', msg, gen.pre_branch, destination, source) # Branch try_execute('git-bloom-branch', msg, execute_branch, source, destination, interactive) # Post branch try_execute('generator post_branch', msg, gen.post_branch, destination, source) ### Run pre - export patches - post # Pre patch try_execute('generator pre_export_patches', msg, gen.pre_export_patches, destination) # Export patches try_execute('git-bloom-patch export', msg, export_patches) # Post branch try_execute('generator post_export_patches', msg, gen.post_export_patches, destination) ### Run pre - rebase - post # Pre rebase try_execute('generator pre_rebase', msg, gen.pre_rebase, destination) # Rebase ret = try_execute('git-bloom-patch rebase', msg, rebase_patches) # Post rebase try_execute('generator post_rebase', msg, gen.post_rebase, destination) ### Run pre - import patches - post # Pre patch try_execute('generator pre_patch', msg, gen.pre_patch, destination) if ret == 0: # Import patches try_execute('git-bloom-patch import', msg, import_patches) elif ret < 0: debug("Skipping patching because rebase did not run.") # Post branch try_execute('generator post_patch', msg, gen.post_patch, destination) except CommandFailed as err: sys.exit(err.returncode or 1)
def maybe_continue(default='y', msg='Continue'): """Prompts the user for continuation""" default = default.lower() msg = "@!{msg} ".format(msg=sanitize(msg)) if default == 'y': msg += "@{yf}[Y/n]? @|" else: msg += "@{yf}[y/N]? @|" msg = fmt(msg) while True: response = safe_input(msg) if not response: response = default response = response.lower() if response not in ['y', 'n', 'q']: error_msg = 'Reponse `' + response + '` was not recognized, ' \ 'please use one of y, Y, n, N.' error(error_msg) else: break if response in ['n', 'q']: return False return True
def main(sysargs=None): if len(sysargs if sysargs is not None else sys.argv[1:]) == 0: # This means show me the current config, first check we have an env ensure_clean_working_env() if not branch_exists(BLOOM_CONFIG_BRANCH): sys.exit("No {0} branch found".format(BLOOM_CONFIG_BRANCH)) show_current() info("See: 'git-bloom-config -h' on how to change the configs") return 0 parser = get_argument_parser() add_global_arguments(parser) args = parser.parse_args(sysargs) handle_global_arguments(args) # Also check to see if git has been init'ed check_git_init() # Check that the current directory is a serviceable git/bloom repo try: ensure_clean_working_env() ensure_git_root() except SystemExit: parser.print_usage() raise # Then call the verb try: args.func(args) except (KeyboardInterrupt, EOFError): error("\nUser sent a Keyboard Interrupt, aborting.", exit=True)
def generate_ros_distro_diff(track, repository, distro, distro_file_url, distro_file, distro_file_raw): with inbranch('upstream'): # Check for package.xml(s) try: from catkin_pkg.packages import find_packages except ImportError: debug(traceback.format_exc()) error("catkin_pkg was not detected, please install it.", file=sys.stderr, exit=True) packages = find_packages(os.getcwd()) if len(packages) == 0: warning("No packages found, will not generate 'package: path' entries for rosdistro.") track_dict = get_tracks_dict_raw()['tracks'][track] last_version = track_dict['last_version'] release_inc = track_dict['release_inc'] if repository not in distro_file['repositories']: global _user_provided_release_url distro_file['repositories'][repository] = {'url': _user_provided_release_url or ''} distro_file['repositories'][repository]['version'] = '{0}-{1}'.format(last_version, release_inc) if packages and (len(packages) > 1 or packages.keys()[0] != '.'): distro_file['repositories'][repository]['packages'] = {} for path, package in packages.iteritems(): if os.path.basename(path) == package.name: distro_file['repositories'][repository]['packages'][package.name] = None else: distro_file['repositories'][repository]['packages'][package.name] = path distro_file_name = os.path.join('release', distro_file_url.split('/')[-1]) distro_dump = yaml.dump(distro_file, indent=2, default_flow_style=False) if distro_file_raw != distro_dump: udiff = difflib.unified_diff(distro_file_raw.splitlines(), distro_dump.splitlines(), fromfile=distro_file_name, tofile=distro_file_name) temp_dir = tempfile.mkdtemp() version = distro_file['repositories'][repository]['version'] udiff_file = os.path.join(temp_dir, repository + '-' + version + '.patch') udiff_raw = '' info("Unified diff for the ROS distro file located at '{0}':".format(udiff_file)) for line in udiff: if line.startswith('@@'): udiff_raw += line line = fmt('@{cf}' + line) if line.startswith('+'): if not line.startswith('+++'): line += '\n' udiff_raw += line line = fmt('@{gf}' + line) if line.startswith('-'): if not line.startswith('---'): line += '\n' udiff_raw += line line = fmt('@{rf}' + line) if line.startswith(' '): line += '\n' udiff_raw += line info(line, use_prefix=False, end='') with open(udiff_file, 'w+') as f: f.write(udiff_raw) return udiff_file, distro_dump else: warning("This release resulted in no changes to the ROS distro file...") return None, None
def rebase_patches(without_git_rebase=True, directory=None): ### Ensure a clean/valid working environment ensure_clean_working_env(git_status=True, directory=directory) ### Make sure we need to actually call this # Get the current branch current_branch = get_current_branch(directory) if current_branch is None: error("Could not determine current branch.", exit=True) # Get the patches branch patches_branch = 'patches/' + current_branch # Get the current patches.conf config = get_patch_config(patches_branch, directory=directory) ### Execute the rebase if without_git_rebase: non_git_rebase(config['parent'], directory=directory) else: git_rebase(config['parent'], directory=directory) ### Update the patches information # Get the latest configs config = get_patch_config(patches_branch, directory) # Set the base to the current hash (before patches) current_branch_ = get_current_branch(directory) debug('Current branch: ' + current_branch_ or 'could not determine branch') config['base'] = get_commit_hash(current_branch_, directory) debug('New current commit hash after rebase: ' + config['base']) # Set the new upstream hash to the previous upstream hash config['previous'] = get_commit_hash(config['parent'], directory) debug('New parent commit hash after rebase: ' + config['previous']) # Clear the trimbase (it needs to be reapplied) config['trimbase'] = '' # Write the new configs set_patch_config(patches_branch, config, directory)
def delete_cmd(track): tracks_dict = get_tracks_dict_raw() if track not in tracks_dict['tracks']: error("Track '{0}' does not exist.".format(track), exit=True) del tracks_dict['tracks'][track] info("Deleted track '{0}'.".format(track)) write_tracks_dict_raw(tracks_dict)
def pre_modify(self): info("\nPre-verifying RPM dependency keys...") # Run rosdep update is needed if not self.has_run_rosdep: self.update_rosdep() peer_packages = [p.name for p in self.packages.values()] while not self._check_all_keys_are_valid(peer_packages, self.rosdistro): error("Some of the dependencies for packages in this repository could not be resolved by rosdep.") error("You can try to address the issues which appear above and try again if you wish, " "or continue without releasing into RPM-based distributions (e.g. Fedora 24).") try: if not maybe_continue(msg="Would you like to try again?"): error("User aborted after rosdep keys were not resolved.") sys.exit(code.GENERATOR_NO_ROSDEP_KEY_FOR_DISTRO) except (KeyboardInterrupt, EOFError): error("\nUser quit.", exit=True) update_rosdep() invalidate_view_cache() info("All keys are " + ansi('greenf') + "OK" + ansi('reset') + "\n") for package in self.packages.values(): if not package.licenses or not package.licenses[0]: error("No license set for package '{0}', aborting.".format(package.name), exit=True)
def commit(self): if self.disabled: return info(fmt("@{bf}<==@| Command successful, committing changes to working copy")) current_branch = get_current_branch() if current_branch is None: error("Could not determine current branch.", exit=True) with inbranch(get_commit_hash(get_current_branch())): with change_directory(self.clone_dir): new_branches = get_branches() for branch in self.current_branches: if branch in new_branches: new_branches.remove(branch) for branch in get_branches(local_only=True): if branch not in new_branches: with inbranch(branch): cmd = 'git pull --rebase origin ' + branch execute_command(cmd) execute_command('git push --all', silent=False) try: execute_command('git push --tags', silent=False) except subprocess.CalledProcessError: warning("Force pushing tags from clone to working repository, " "you will have to force push back to origin...") execute_command('git push --force --tags', silent=False) self.clean_up()
def _set_trim_sub_dir(sub_dir, force, config, directory): debug("_set_trim_sub_dir(" + str(sub_dir) + ", " + str(force) + ", " + str(config) + ", " + str(directory) + ")") if sub_dir is not None: if config['trim'] != '' and config['trim'] != sub_dir: warning("You are trying to set the trim sub directory to " + sub_dir + ", but it is already set to " + config['trim'] + ".") if not force: warning("Changing the sud directory is not advised. " "If you are sure you want to do this, use " "'--force'") return None else: warning("Forcing the change of the sub directory.") # Make the sub_dir absolute git_root = get_root(directory) sub_dir_abs = os.path.join(git_root, sub_dir) # Make sure it is a directory if not os.path.isdir(sub_dir_abs): error("The given sub directory, (" + sub_dir + ") does not " "exist in the git repository at " + git_root) return None # Set the trim sub directory config['trim'] = sub_dir return config
def fn(config): global _patch_config_keys conf_path = 'patches.conf' if directory is not None: conf_path = os.path.join(directory, conf_path) config_keys = config.keys() config_keys.sort() if _patch_config_keys != config_keys: raise RuntimeError("Invalid config passed to set_patch_config") cmd = 'git config -f {0} patches.'.format(conf_path) try: for key in config: _cmd = cmd + key + ' "' + config[key] + '"' execute_command(_cmd, cwd=directory) # Stage the patches.conf file cmd = 'git add ' + conf_path execute_command(cmd, cwd=directory) if has_changes(directory): # Commit the changed config file cmd = 'git commit -m "Updated patches.conf"' execute_command(cmd, cwd=directory) except subprocess.CalledProcessError as err: print_exc(traceback.format_exc()) error("Failed to set patches info: " + str(err)) raise
def fn(config): global _patch_config_keys conf_path = 'patches.conf' if directory is not None: conf_path = os.path.join(directory, conf_path) config_keys = list(config.keys()) config_keys.sort() if _patch_config_keys != config_keys: raise RuntimeError("Invalid config passed to set_patch_config") cmd = 'git config -f {0} patches.'.format(conf_path) try: for key in config: _cmd = cmd + key + ' "' + config[key] + '"' execute_command(_cmd, cwd=directory) # Stage the patches.conf file cmd = 'git add ' + conf_path execute_command(cmd, cwd=directory) if has_changes(directory): # Commit the changed config file cmd = 'git commit -m "Updated patches.conf"' execute_command(cmd, cwd=directory) except subprocess.CalledProcessError as err: print_exc(traceback.format_exc()) error("Failed to set patches info: " + str(err)) raise
def main(sysargs=None): # Check that the current directory is a serviceable git/bloom repo ensure_clean_working_env() ensure_git_root() # Get tracks tracks_dict = get_tracks_dict_raw() if not tracks_dict['tracks']: error("No tracks configured, first create a track with " "'git-bloom-config new <track_name>'", exit=True) # Do argparse stuff parser = get_argument_parser([str(t) for t in tracks_dict['tracks']]) parser = add_global_arguments(parser) args = parser.parse_args(sysargs) handle_global_arguments(args) verify_track(args.track, tracks_dict['tracks'][args.track]) execute_track(args.track, tracks_dict['tracks'][args.track], args.release_increment, args.pretend, args.debug, args.unsafe) # Notify the user of success and next action suggestions print('\n\n') warning("Tip: Check to ensure that the debian tags created have the same " "version as the upstream version you are releasing.") info(fmt("@{gf}@!Everything went as expected, " "you should check that the new tags match your expectations, and " "then push to the release repo with:@|")) info(fmt(" git push --all && git push --tags " "@{kf}@!# You might have to add --force to the second command if you " "are over-writing existing flags"))
def verify_track(track_name, track): upconvert_bloom_to_config_branch() for entry in DEFAULT_TEMPLATE: if entry not in track: error("Track '{0}' is missing configuration ".format(track_name) + "'{0}', it may be out of date, please run 'git-bloom-config edit {1}'." .format(entry, track_name), exit=True)
def __setattr__(self, key, value): if key == 'default' and self.values: if value not in self.values: error("Invalid input '{0}' for '{1}', acceptable values: {2}.". format(value, self.name, self.values), exit=True) object.__setattr__(self, key, value)
def post_rebase(self, destination): name = destination.split('/')[-1] # Retrieve the package package = self.packages[name] # Handle differently if this is an arch vs distro branch if destination in self.arch_branches: info("Placing Arch template files into '{0}' branch." .format(destination)) # Then this is an arch branch # Place the raw template files self.place_template_files() else: # This is a distro specific arch branch # Determine the current package being generated distro = destination.split('/')[-2] # Create Arch packages for each distro with inbranch(destination): # To fit Arch Linux philosophy a bit better, we move all the source files into a subdirectory. # Arch Linux doesn't support source distribution through a subdirectory; therefore we should ideally compress the sources or provide a URL. # At this point in the generator, it is tricky to get the release URL. Furthermore it wouldn't fit bloom's patch mechanism very well. # To work around, we copy the sources to the $srcdir at the beginning inside the prepare() function. temp_dir = mkdtemp(dir='.') for item in os.listdir("."): itemsrc = os.path.abspath(item) if os.path.basename(itemsrc) in ['.', '..', '.git', '.svn', 'arch',os.path.basename(temp_dir)]: continue itemdst = os.path.abspath(os.path.join(temp_dir,item)) execute_command('git mv ' + itemsrc + ' ' + itemdst) execute_command('git mv ' + temp_dir + ' ' + name) execute_command('git commit --amend --no-edit') # Then generate the PKGBUILD data = self.generate_arch(package, distro) # And finally move the PKGBUILD to the root directory of the package. for item in os.listdir("arch"): itemsrc = os.path.abspath(os.path.join("arch",item)) if os.path.basename(itemsrc) in ['.', '..', '.git', '.svn']: continue itemdst = os.path.abspath(item) execute_command('git mv ' + itemsrc + ' ' + itemdst) execute_command('git commit --amend --no-edit') # Create the tag name for later self.tag_names[destination] = self.generate_tag_name(data) # Update the patch configs patches_branch = 'patches/' + destination config = get_patch_config(patches_branch) # Store it self.store_original_config(config, patches_branch) # Modify the base so import/export patch works current_branch = get_current_branch() if current_branch is None: error("Could not determine current branch.", exit=True) config['base'] = get_commit_hash(current_branch) # Set it set_patch_config(patches_branch, config)
def main(sysargs=None): if len(sysargs if sysargs is not None else sys.argv[1:]) == 0: # This means show me the current config, first check we have an env ensure_clean_working_env() if not branch_exists('bloom'): sys.exit("No bloom branch found") show_current() info("See: 'git-bloom-config -h' on how to change the configs") return 0 parser = get_argument_parser() add_global_arguments(parser) args = parser.parse_args(sysargs) handle_global_arguments(args) # Also check to see if git has been init'ed check_git_init() # Check that the current directory is a serviceable git/bloom repo try: ensure_clean_working_env() ensure_git_root() except SystemExit: parser.print_usage() raise # Then call the verb try: args.func(args) except (KeyboardInterrupt, EOFError): error("\nUser sent a Keyboard Interrupt, aborting.", exit=True)
def main(sysargs): parser = get_argument_parser() args = parser.parse_args(sys.argv[1:]) repository = args.repository verbose = args.verbose # checkout target rpository info("Manually clone the repository") info(" git clone {0}".format(repository)) git = get_vcs_client('git', tempfile.mktemp()) info(fmt("@{gf}@!==> @|") + "Fetching repository from '{0}'".format(repository)) ret = git.checkout(repository, verbose=verbose) if not ret: error("Could not checkout {}".format(repository)) return 1 # get the github repository info base_org, base_repo = get_gh_info(git.get_url()) # get correct repo info (case sensitive) gh = get_github_interface() base_org, base_repo = get_gh_info(gh.get_repo(base_org, base_repo)['html_url']) base_branch = git.get_branches()[0] # is this ok? with change_directory(git.get_path()): # write travis yaml write_travis_yaml() # write readme write_readme_md(**locals()) # create pull request open_pull_request(base_org=base_org, base_repo=base_repo, base_branch=base_branch, new_branch="add_travis")
def validate_args(upstream_repo_type): # Ensure that the upstream-repo-type is valid if upstream_repo_type not in ['git', 'svn', 'hg', 'bzr']: error("Invalid upstream repository type: " "{0}\n".format(upstream_repo_type)) return False return True
def maybe_continue(default='y'): """Prompts the user for continuation""" default = default.lower() msg = "{0}Continue ".format(ansi('boldon')) if default == 'y': msg += "{0}[Y/n]? {1}".format(ansi('yellowf'), ansi('reset')) else: msg += "{0}[y/N]? {1}".format(ansi('yellowf'), ansi('reset')) while True: response = raw_input(msg) if not response: response = default response = response.lower() if response not in ['y', 'n', 'q']: error_msg = 'Reponse `' + response + '` was not recognized, ' \ 'please use one of y, Y, n, N.' error(error_msg) else: break if response in ['n', 'q']: return False return True
def place_template_files(self, build_type, debian_dir='debian'): # Create/Clean the debian folder if os.path.exists(debian_dir): if self.interactive: warning("debian directory exists: " + debian_dir) warning("Do you wish to overwrite it?") if not maybe_continue('y'): error("Answered no to continue, aborting.", exit=True) elif 'BLOOM_CLEAR_DEBIAN_ON_GENERATION' in os.environ: warning("Overwriting debian directory: " + debian_dir) execute_command('git rm -rf ' + debian_dir) execute_command( 'git commit -m "Clearing previous debian folder"') if os.path.exists(debian_dir): shutil.rmtree(debian_dir) else: warning("Not overwriting debian directory.") # Use generic place template files command place_template_files('.', build_type, gbp=True) # Commit results execute_command('git add ' + debian_dir) _, has_files, _ = execute_command('git diff --cached --name-only', return_io=True) if has_files: execute_command('git commit -m "Placing debian template files"')
def __place_template_folder(group, src, dst, gbp=False): template_files = pkg_resources.resource_listdir(group, src) # For each template, place for template_file in template_files: template_path = os.path.join(src, template_file) template_dst = os.path.join(dst, template_file) if pkg_resources.resource_isdir(group, template_path): debug("Recursing on folder '{0}'".format(template_path)) __place_template_folder(group, template_path, template_dst, gbp) else: try: debug("Placing template '{0}'".format(template_path)) template = pkg_resources.resource_string(group, template_path) template_abs_path = pkg_resources.resource_filename(group, template_path) except IOError as err: error("Failed to load template " "'{0}': {1}".format(template_file, str(err)), exit=True) if not os.path.exists(dst): os.makedirs(dst) if os.path.exists(template_dst): debug("Removing existing file '{0}'".format(template_dst)) os.remove(template_dst) with open(template_dst, 'w') as f: if not isinstance(template, str): template = template.decode('utf-8') f.write(template) shutil.copystat(template_abs_path, template_dst)
def post_patch(self, destination, color='bluef'): if destination in self.rpm_branches: return # Tag after patches have been applied with inbranch(destination): # Tag tag_name = self.tag_names[destination] if tag_exists(tag_name): if self.interactive: warning("Tag exists: " + tag_name) warning("Do you wish to overwrite it?") if not maybe_continue('y'): error("Answered no to continue, aborting.", exit=True) else: warning("Overwriting tag: " + tag_name) else: info("Creating tag: " + tag_name) execute_command('git tag -f ' + tag_name) # Report of success name = destination.split('/')[-1] package = self.packages[name] distro = destination.split('/')[-2] info(ansi(color) + "####" + ansi('reset'), use_prefix=False) info( ansi(color) + "#### " + ansi('greenf') + "Successfully" + ansi(color) + " generated '" + ansi('boldon') + distro + ansi('boldoff') + "' RPM for package" " '" + ansi('boldon') + package.name + ansi('boldoff') + "'" + " at version '" + ansi('boldon') + package.version + "-" + str(self.rpm_inc) + ansi('boldoff') + "'" + ansi('reset'), use_prefix=False ) info(ansi(color) + "####\n" + ansi('reset'), use_prefix=False)
def handle_arguments(self, args): self.interactive = args.interactive self.debian_inc = args.debian_inc self.os_name = args.os_name self.distros = args.distros if self.distros in [None, []]: self.distros = get_ubuntu_targets(self.rosdistro) self.install_prefix = args.install_prefix if args.install_prefix is None: self.install_prefix = self.default_install_prefix self.prefix = args.prefix self.branches = match_branches_with_prefix(self.prefix, get_branches) if len(self.branches) == 0: error("No packages found, check your --prefix or --src arguments.") return code.NO_PACKAGE_XML_FOUND self.packages = {} self.tag_names = {} self.names = [] self.branch_args = [] for branch in self.branches: stackage, kind = get_stackage_from_branch(branch) self.packages[stackage.name] = (stackage, kind) self.names.append(stackage.name) args = self.generate_branching_arguments(stackage, branch) self.branch_args.extend(args)
def post_patch(self, destination, color='bluef'): # Tag after patches have been applied with inbranch(destination): # Tag tag_name = self.tag_names[destination] if tag_exists(tag_name): if self.interactive: warning("Tag exists: " + tag_name) warning("Do you wish to overwrite it?") if not maybe_continue('y'): error("Answered no to continue, aborting.") return code.ANSWERED_NO_TO_CONTINUE else: warning("Overwriting tag: " + tag_name) else: info("Creating tag: " + tag_name) execute_command('git tag -f ' + tag_name) # Report of success name = destination.split('/')[-1] stackage, kind = self.packages[name] distro = destination.split('/')[-2] info(ansi(color) + "####" + ansi('reset'), use_prefix=False) info( ansi(color) + "#### " + ansi('greenf') + "Successfully" + \ ansi(color) + " generated '" + ansi('boldon') + distro + \ ansi('boldoff') + "' debian for " + kind + \ " '" + ansi('boldon') + stackage.name + ansi('boldoff') + "'" + \ " at version '" + ansi('boldon') + stackage.version + \ "-" + str(self.debian_inc) + ansi('boldoff') + "'" + \ ansi('reset'), use_prefix=False ) info(ansi(color) + "####\n" + ansi('reset'), use_prefix=False)
def __place_template_folder(group, src, dst, gbp=False): template_files = pkg_resources.resource_listdir(group, src) # For each template, place for template_file in template_files: if not gbp and os.path.basename(template_file) == 'gbp.conf.em': debug("Skipping template '{0}'".format(template_file)) continue template_path = os.path.join(src, template_file) template_dst = os.path.join(dst, template_file) if pkg_resources.resource_isdir(group, template_path): debug("Recursing on folder '{0}'".format(template_path)) __place_template_folder(group, template_path, template_dst, gbp) else: try: debug("Placing template '{0}'".format(template_path)) template = pkg_resources.resource_string(group, template_path) template_abs_path = pkg_resources.resource_filename( group, template_path) except IOError as err: error("Failed to load template " "'{0}': {1}".format(template_file, str(err)), exit=True) if not os.path.exists(dst): os.makedirs(dst) if os.path.exists(template_dst): debug("Removing existing file '{0}'".format(template_dst)) os.remove(template_dst) with open(template_dst, 'w') as f: if not isinstance(template, str): template = template.decode('utf-8') f.write(template) shutil.copystat(template_abs_path, template_dst)
def update_rosdep(): info("Running 'rosdep update'...") try: rosdep2.catkin_support.update_rosdep() except: print_exc(traceback.format_exc()) error("Failed to update rosdep, did you run 'rosdep init' first?", exit=True)
def post_rebase(self, destination): name = destination.split('/')[-1] # Retrieve the package package = self.packages[name] # Handle differently if this is a debian vs distro branch if destination in self.debian_branches: info("Placing debian template files into '{0}' branch.".format( destination)) # Then this is a debian branch # Place the raw template files self.place_template_files() else: # This is a distro specific debian branch # Determine the current package being generated distro = destination.split('/')[-2] # Create debians for each distro with inbranch(destination): data = self.generate_debian(package, distro) # Create the tag name for later self.tag_names[destination] = self.generate_tag_name(data) # Update the patch configs patches_branch = 'patches/' + destination config = get_patch_config(patches_branch) # Store it self.store_original_config(config, patches_branch) # Modify the base so import/export patch works current_branch = get_current_branch() if current_branch is None: error("Could not determine current branch.", exit=True) config['base'] = get_commit_hash(current_branch) # Set it set_patch_config(patches_branch, config)
def _set_trim_sub_dir(sub_dir, force, config, directory): debug("_set_trim_sub_dir(" + str(sub_dir) + ", " + str(force) + ", " + \ str(config) + ", " + str(directory) + ")") if sub_dir is not None: if config['trim'] != '' and config['trim'] != sub_dir: warning("You are trying to set the trim sub directory to " + \ sub_dir + ", but it is already set to " + \ config['trim'] + ".") if not force: warning("Changing the sud directory is not advised. " "If you are sure you want to do this, use " "'--force'") return None else: warning("Forcing the change of the sub directory.") # Make the sub_dir absolute git_root = get_root(directory) sub_dir_abs = os.path.join(git_root, sub_dir) # Make sure it is a directory if not os.path.isdir(sub_dir_abs): error("The given sub directory, (" + sub_dir + ") does not " "exist in the git repository at " + git_root) return None # Set the trim sub directory config['trim'] = sub_dir return config
def get_repo_uri(repository, distro): url = None # Fetch the distro file distribution_file = get_distribution_file(distro) if repository in distribution_file.repositories and \ distribution_file.repositories[repository].release_repository is not None: url = distribution_file.repositories[repository].release_repository.url else: error("Specified repository '{0}' is not in the distribution file located at '{1}'" .format(repository, get_disitrbution_file_url(distro))) matches = difflib.get_close_matches(repository, distribution_file.repositories) if matches: info(fmt("@{yf}Did you mean one of these: '" + "', '".join([m for m in matches]) + "'?")) if not url: info("Could not determine release repository url for repository '{0}' of distro '{1}'" .format(repository, distro)) info("You can continue the release process by manually specifying the location of the RELEASE repository.") info("To be clear this is the url of the RELEASE repository not the upstream repository.") try: url = safe_input('Release repository url [press enter to abort]: ') except (KeyboardInterrupt, EOFError): url = None info('', use_prefix=False) if not url: error("No release repository url given, aborting.", exit=True) global _user_provided_release_url _user_provided_release_url = url return url
def post_patch(self, destination, color='bluef'): if destination in self.debian_branches: return # Tag after patches have been applied with inbranch(destination): # Tag tag_name = self.tag_names[destination] if tag_exists(tag_name): if self.interactive: warning("Tag exists: " + tag_name) warning("Do you wish to overwrite it?") if not maybe_continue('y'): error("Answered no to continue, aborting.", exit=True) else: warning("Overwriting tag: " + tag_name) else: info("Creating tag: " + tag_name) execute_command('git tag -f ' + tag_name) # Report of success name = destination.split('/')[-1] package = self.packages[name] distro = destination.split('/')[-2] info(ansi(color) + "####" + ansi('reset'), use_prefix=False) info(ansi(color) + "#### " + ansi('greenf') + "Successfully" + ansi(color) + " generated '" + ansi('boldon') + distro + ansi('boldoff') + "' debian for package" " '" + ansi('boldon') + package.name + ansi('boldoff') + "'" + " at version '" + ansi('boldon') + package.version + "-" + str(self.debian_inc) + ansi('boldoff') + "'" + ansi('reset'), use_prefix=False) info(ansi(color) + "####\n" + ansi('reset'), use_prefix=False)
def segment_version(full_version): version_list = full_version.split('.') if len(version_list) != 3: error('Invalid version element in the stack.xml, expected: ' \ '<major>.<minor>.<patch>') sys.exit(code.INVALID_VERSION) return version_list
def merge_packages(pkgs_dict, get_subs_fn, os_name, os_version, ros_distro, install_prefix, native=False): all_subs = {} for path, pkg in pkgs_dict.items(): try: subs = get_subs_fn(pkg, os_name, os_version, ros_distro, install_prefix, native) all_subs[subs['Name']] = subs except Exception as exc: debug(traceback.format_exc()) error(type(exc).__name__ + ": " + str(exc), exit=True) except (KeyboardInterrupt, EOFError): sys.exit(1) repo_header = {} cnt = 0 for pkg, sub in all_subs.items(): try: if (0 == cnt): repo_header['Package'] = convertToUnicode( sanitize_package_name('tesseract_core')) repo_header['DebianInc'] = sub['DebianInc'] repo_header['format'] = sub['format'] repo_header['InstallationPrefix'] = sub['InstallationPrefix'] repo_header['Maintainer'] = sub['Maintainers'][0] repo_header['Maintainers'] = sub['Maintainers'] repo_header['BuildDepends'] = sub['BuildDepends'] repo_header['Homepage'] = sub['Homepage'] repo_header['Copyright'] = sub['Copyright'] repo_header['debhelper_version'] = sub['debhelper_version'] repo_header['changelogs'] = sub['changelogs'] repo_header['Distribution'] = sub['Distribution'] else: repo_header['Maintainers'].join(', '.join(sub['Maintainers'])) repo_header['BuildDepends'].extend(sub['BuildDepends']) repo_header['Copyright'].join(sub['Copyright']) cnt = cnt + 1 except Exception as exc: debug(traceback.format_exc()) error(type(exc).__name__ + ": " + str(exc), exit=True) except (KeyboardInterrupt, EOFError): sys.exit(1) # Remove build depends in this repository repo_header['BuildDepends'] = [ x for x in repo_header['BuildDepends'] if x not in all_subs.keys() ] # Remove duplicates repo_header['BuildDepends'] = list( dict.fromkeys(repo_header['BuildDepends'])) # TODO Remove Duplicates from repo_header['Maintainers'] all_subs[repo_header['Package']] = repo_header return all_subs
def fetch_distro_file(distro_file_url): try: raw_distro_file = urllib2.urlopen(distro_file_url) except urllib2.HTTPError as e: error("Failed to fetch ROS distro file at '{0}': {1}" .format(distro_file_url, e), exit=True) return raw_distro_file.read()
def create_pull_request(org, repo, user, password, base_branch, head_branch, title, body=""): headers = {} headers["Authorization"] = "Basic {0}".format( base64.b64encode('{0}:{1}'.format(user, password))) headers['User-Agent'] = 'bloom-{0}'.format(bloom.__version__) conn = httplib.HTTPSConnection('api.github.com') data = { 'title': title, 'body': body, 'head': "{0}:{1}".format(user, head_branch), 'base': base_branch } conn.request('POST', '/repos/{0}/{1}/pulls'.format(org, repo), json.dumps(data), headers) resp = conn.getresponse() if str(resp.status) != '201': error("Failed to create pull request: {0} {1}".format( resp.status, resp.reason), exit=True) api_location = resp.msg.dict['location'] api_dict = fetch_github_api(api_location) return api_dict['html_url']
def _check_all_keys_are_valid(self, peer_packages, rosdistro): keys_to_resolve = set() key_to_packages_which_depends_on = collections.defaultdict(list) keys_to_ignore = set() for package in self.packages.values(): evaluate_package_conditions(package, rosdistro) depends = [ dep for dep in (package.run_depends + package.buildtool_export_depends) if dep.evaluated_condition is not False] build_depends = [ dep for dep in (package.build_depends + package.buildtool_depends + package.test_depends) if dep.evaluated_condition is not False] unresolved_keys = [ dep for dep in (depends + build_depends + package.replaces + package.conflicts) if dep.evaluated_condition is not False] keys_to_ignore = { dep for dep in keys_to_ignore.union(package.replaces + package.conflicts) if dep.evaluated_condition is not False} keys = [d.name for d in unresolved_keys] keys_to_resolve.update(keys) for key in keys: key_to_packages_which_depends_on[key].append(package.name) for skip_key in self.skip_keys: try: keys_to_resolve.remove(skip_key) except KeyError: warning("Key '{0}' specified by --skip-keys was not found".format(skip_key)) else: warning("Skipping dependency key '{0}' per --skip-keys".format(skip_key)) os_name = self.os_name rosdistro = self.rosdistro all_keys_valid = True for key in sorted(keys_to_resolve): for os_version in self.distros: try: extended_peer_packages = peer_packages + [d.name for d in keys_to_ignore] rule, installer_key, default_installer_key = \ resolve_rosdep_key(key, os_name, os_version, rosdistro, extended_peer_packages, retry=False) if rule is None: continue if installer_key != default_installer_key: error("Key '{0}' resolved to '{1}' with installer '{2}', " "which does not match the default installer '{3}'." .format(key, rule, installer_key, default_installer_key)) BloomGenerator.exit( "The RPM generator does not support dependencies " "which are installed with the '{0}' installer." .format(installer_key), returncode=code.GENERATOR_INVALID_INSTALLER_KEY) except (GeneratorError, RuntimeError) as e: print(fmt("Failed to resolve @{cf}@!{key}@| on @{bf}{os_name}@|:@{cf}@!{os_version}@| with: {e}") .format(**locals())) print(fmt("@{cf}@!{0}@| is depended on by these packages: ").format(key) + str(list(set(key_to_packages_which_depends_on[key])))) print(fmt("@{kf}@!<== @{rf}@!Failed@|")) all_keys_valid = False return all_keys_valid
def find_version_from_upstream(vcs_uri, vcs_type, devel_branch=None, ros_distro='indigo'): # Check for github.com # if vcs_uri.startswith('http') and 'github.com' in vcs_uri: # info("Detected github.com repository, checking for package.xml " # "in root of devel branch using raw.github.com...") # version = find_version_from_upstream_github(vcs_uri, devel_branch) # if version: # return version, None # warning(" Failed to find the version using raw.github.com.") # Try to clone the upstream repository info("Checking upstream devel branch '{0}' for package.xml(s)".format( devel_branch or '<default>')) upstream_repo = get_upstream_repo(vcs_uri, vcs_type) if not upstream_repo.checkout(vcs_uri, devel_branch or ''): error("Failed to checkout to the upstream branch " "'{0}' in the repository from '{1}'".format( devel_branch or '<default>', vcs_uri), exit=True) meta = get_upstream_meta(upstream_repo.get_path(), ros_distro) if not meta: error( "Failed to find any package.xml(s) in the " "upstream devel branch '{0}' in the repository from '{1}'".format( devel_branch or '<default>', vcs_uri)) info("Detected version '{0}' from package(s): {1}".format( meta['version'], meta['name'])) return meta['version'], upstream_repo
def post_rebase(self, destination): # Determine the current package being generated name = destination.split('/')[-1] distro = destination.split('/')[-2] # Retrieve the stackage stackage, kind = self.packages[name] # Ask to continue if interactive if self.interactive: if not maybe_continue('y'): error("Answered no to continue, aborting.") return code.ANSWERED_NO_TO_CONTINUE ### Start debian generation # Get time of day from dateutil import tz stamp = datetime.datetime.now(tz.tzlocal()) # Convert stackage to debian data data = self.convert_stackage_to_debian_data(stackage, kind) # Get apt_installer from rosdep from rosdep2.catkin_support import get_installer self.apt_installer = get_installer(APT_INSTALLER) # Create debians for each distro with inbranch(destination): self.generate_debian(data, stamp, distro) # Create the tag name for later self.tag_names[destination] = self.generate_tag_name(data) # Update the patch configs patches_branch = 'patches/' + destination config = get_patch_config(patches_branch) # Store it self.store_original_config(config, patches_branch) # Modify the base so import/export patch works config['base'] = get_commit_hash(get_current_branch()) # Set it set_patch_config(patches_branch, config)
def commit(self): if self.disabled: return info( fmt("@{bf}<==@| Command successful, committing changes to working copy" )) current_branch = get_current_branch() if current_branch is None: error("Could not determine current branch.", exit=True) with inbranch(get_commit_hash(get_current_branch())): with change_directory(self.clone_dir): new_branches = get_branches() for branch in self.current_branches: if branch in new_branches: new_branches.remove(branch) for branch in get_branches(local_only=True): if branch not in new_branches: with inbranch(branch): cmd = 'git pull --rebase origin ' + branch execute_command(cmd) execute_command('git push --all', silent=False) try: execute_command('git push --tags', silent=False) except subprocess.CalledProcessError: warning( "Force pushing tags from clone to working repository, " "you will have to force push back to origin...") execute_command('git push --force --tags', silent=False) self.clean_up()
def create_from_template(self, template_name, data, directory, chmod=None, outfile=None): # Configure template name extention = '.em' if not template_name.endswith(extention): template_file = template_name + extention else: template_file = template_name template_name = template_name[:len(extention)] template_path = os.path.join('templates', template_file) # Get the template contents using pkg_resources group = 'bloom.generators.debian' # info("Looking for template: " + group + ':' + template_path) try: template = pkg_resources.resource_string(group, template_path) except IOError as err: error("Failed to load template " "'{0}': {1}".format(template_name, str(err))) self.exit(code.DEBIAN_FAILED_TO_LOAD_TEMPLATE) # Expand template outfile = outfile if outfile is not None else template_name info("Expanding template: '" + template_file + "' to '" + \ outfile + "'") result = em.expand(template, **data) # Write the template out with change_directory(directory): with open(outfile, 'w+') as f: f.write(result) # Set permissions if needed if chmod is not None: os.chmod(outfile, chmod)
def pre_modify(self): info("\nPre-verifying RPM dependency keys...") # Run rosdep update is needed if not self.has_run_rosdep: self.update_rosdep() peer_packages = [p.name for p in self.packages.values()] while not self._check_all_keys_are_valid(peer_packages, self.rosdistro): error("Some of the dependencies for packages in this repository could not be resolved by rosdep.") if not self.interactive: sys.exit(code.GENERATOR_NO_ROSDEP_KEY_FOR_DISTRO) error("You can try to address the issues which appear above and try again if you wish, " "or continue without releasing into RPM-based distributions (e.g. Fedora 24).") try: if not maybe_continue(msg="Would you like to try again?"): error("User aborted after rosdep keys were not resolved.") sys.exit(code.GENERATOR_NO_ROSDEP_KEY_FOR_DISTRO) except (KeyboardInterrupt, EOFError): error("\nUser quit.", exit=True) update_rosdep() invalidate_view_cache() info("All keys are " + ansi('greenf') + "OK" + ansi('reset') + "\n") for package in self.packages.values(): if not package.licenses or not package.licenses[0]: error("No license set for package '{0}', aborting.".format(package.name), exit=True)
def post_rebase(self, destination): name = destination.split('/')[-1] # Retrieve the package package = self.packages[name] # Handle differently if this is an rpm vs distro branch if destination in self.rpm_branches: info("Placing RPM template files into '{0}' branch." .format(destination)) # Then this is an rpm branch # Place the raw template files self.place_template_files() else: # This is a distro specific rpm branch # Determine the current package being generated distro = destination.split('/')[-2] # Create RPMs for each distro with inbranch(destination): data = self.generate_rpm(package, distro) # Create the tag name for later self.tag_names[destination] = self.generate_tag_name(data) # Update the patch configs patches_branch = 'patches/' + destination config = get_patch_config(patches_branch) # Store it self.store_original_config(config, patches_branch) # Modify the base so import/export patch works current_branch = get_current_branch() if current_branch is None: error("Could not determine current branch.", exit=True) config['base'] = get_commit_hash(current_branch) # Set it set_patch_config(patches_branch, config)
def execute_command(cmd, shell=True, autofail=True, silent=True, silent_error=False, cwd=None, return_io=False): """ Executes a given command using vcstools' run_shell_command function. """ io_type = None result = 0 if silent: io_type = PIPE debug(((cwd) if cwd else os.getcwd()) + ":$ " + str(cmd)) p = Popen(cmd, shell=True, cwd=cwd, stdout=io_type, stderr=io_type) out, err = p.communicate() result = p.returncode if result != 0: if not silent_error: error("'execute_command' failed to call '{0}'".format(cmd) + \ " which had a return code ({0}):".format(result)) if out: error(" stdout:\n" + ansi('reset') + str(out)) error("end stdout") if err: error(" stderr:\n" + ansi('reset') + str(err)) error("end stderr") if autofail: raise CalledProcessError(cmd=cmd, output=out, returncode=result) if return_io: return result, out, err else: return result
def handle_tree(tree, directory, root_path, version): for path, kind in tree.items(): if kind == 'directory': # Path relative to start path rel_path = os.path.join(directory, path) # If it is a file, error if os.path.isfile(rel_path): error("In patches path '{0}' is a directory".format(rel_path) + ", but it exists in the upstream branch as a file.", exit=True) # If it is not already a directory, create it if not os.path.isdir(rel_path): info(" Createing directory... '{0}'".format(rel_path)) os.mkdir(rel_path) # Recurse on the directory handle_tree( ls_tree(BLOOM_CONFIG_BRANCH, os.path.join(root_path, rel_path)), rel_path, root_path, version) if kind == 'file': # Path relative to start path rel_path = os.path.join(directory, path) # If the local version is a directory, error if os.path.isdir(rel_path): error("In patches path '{0}' is a file, ".format(rel_path) + "but it exists in the upstream branch as a directory.", exit=True) # If the file already exists, warn if os.path.isfile(rel_path): warning(" File '{0}' already exists, overwriting...".format( rel_path)) execute_command('git rm {0}'.format(rel_path), shell=True) # If package.xml tempalte in version, else grab data if path in ['stack.xml']: warning( " Skipping '{0}' templating, fuerte not supported".format( rel_path)) if path in ['package.xml']: info(" Templating '{0}' into upstream branch...".format( rel_path)) file_data = show(BLOOM_CONFIG_BRANCH, os.path.join(root_path, rel_path)) file_data = file_data.replace(':{version}', version) else: info(" Overlaying '{0}' into upstream branch...".format( rel_path)) file_data = show(BLOOM_CONFIG_BRANCH, os.path.join(root_path, rel_path)) # Write file with open(rel_path, 'wb') as f: # Python 2 will treat this as an ascii string but # Python 3 will not re-decode a utf-8 string. if sys.version_info.major == 2: file_data = file_data.decode('utf-8').encode('utf-8') else: file_data = file_data.encode('utf-8') f.write(file_data) # Add it with git execute_command('git add {0}'.format(rel_path), shell=True)
def show(args): tracks_dict = get_tracks_dict_raw() if args.track not in tracks_dict['tracks']: error("Track '{0}' does not exist.".format(args.track), exit=True) info( yaml.dump({args.track: tracks_dict['tracks'][args.track]}, indent=2, default_flow_style=False))
def check_for_bloom_conf(repository): bloom_ls = ls_tree('bloom') if bloom_ls is None: error("Release repository '{0}' not initialized,".format(repository) + " please initialize the bloom repository before releasing from it.", exit=True) bloom_files = [f for f, t in bloom_ls.iteritems() if t == 'file'] return 'bloom.conf' in bloom_files
def get_repo_uri(repository, distro, distro_file_url=ROS_DISTRO_FILE): # Fetch the distro file distro_file_url = distro_file_url.format(distro) distro_file = yaml.load(fetch_distro_file(distro_file_url)) if repository not in distro_file['repositories']: error("Specified repository '{0}' is not in the distro file located at '{1}'" .format(repository, distro_file_url), exit=True) return distro_file['repositories'][repository]['url']
def __setattr__(self, key, value): if key == 'default' and self.values: if value not in self.values: error( "Invalid input '{0}' for '{1}', acceptable values: {2}." .format(value, self.name, self.values), exit=True ) object.__setattr__(self, key, value)
def metapackage_check(self, path, pkg): if pkg.is_metapackage(): try: metapackage.validate_metapackage(path, pkg) except metapackage.InvalidMetapackage as e: warning("Invalid metapackage:") warning(" %s\n" % str(e)) error(fmt("Refusing to release invalid metapackage '@|%s@{rf}@!', metapackage requirements:\n @|%s" % (pkg.name, metapackage.DEFINITION_URL)), exit=True)
def copy_track(src, dst): tracks_dict = get_tracks_dict_raw() if src not in tracks_dict['tracks']: error("Track '{0}' does not exist.".format(src), exit=True) if dst in tracks_dict['tracks']: error("Track '{0}' already exists.".format(dst), exit=True) tracks_dict['tracks'][dst] = copy.deepcopy(tracks_dict['tracks'][src]) info("Saving '{0}' track.".format(dst)) write_tracks_dict_raw(tracks_dict)
def get_versions_from_upstream_tag(tag): """ Returns the [major, minor, patch] version list given an upstream tag. """ tag_version = tag.split('/') if len(tag_version) != 2: error("Malformed tag {0}".format(tag)) sys.exit(code.INVALID_UPSTREAM_TAG) tag_version = tag_version[1] return segment_version(tag_version)
def build_debian_pkg(args=None, get_subs_fn=None): get_subs_fn = get_subs_fn or get_subs _place_template_files = True _process_template_files = True package_path = os.getcwd() if args is not None: package_path = args.package_path or os.getcwd() _place_template_files = args.place_template_files _process_template_files = args.process_template_files pkgs_dict = find_packages(package_path) if len(pkgs_dict) == 0: sys.exit("No packages found in path: '{0}'".format(package_path)) # if len(pkgs_dict) > 1: # sys.exit("Multiple packages found, " # "this tool only supports one package at a time.") os_data = create_default_installer_context().get_os_name_and_version() os_name, os_version = os_data ros_distro = os.environ.get('ROS_DISTRO', 'indigo') # Allow args overrides os_name = args.os_name or os_name os_version = args.os_version or os_version ros_distro = args.ros_distro or ros_distro install_prefix = args.install_prefix or "/opt" # Summarize info(fmt("@!@{gf}==> @|") + fmt("Generating debs for @{cf}%s:%s@| for package(s) %s" % (os_name, os_version, [p.name for p in pkgs_dict.values()]))) # Test Creating single all_subs = merge_packages(pkgs_dict, get_subs_fn, os_name, os_version, ros_distro, install_prefix, args.native) path = '' build_type = 'cmake' try: if _place_template_files: # Place template files place_template_files(path, build_type) if _process_template_files: # Just process existing template files template_files = process_template_files(path, all_subs) if not _place_template_files and not _process_template_files: # If neither, do both place_template_files(path, build_type) template_files = process_template_files(path, all_subs) if template_files is not None: for template_file in template_files: os.remove(os.path.normpath(template_file)) except Exception as exc: debug(traceback.format_exc()) error(type(exc).__name__ + ": " + str(exc), exit=True) except (KeyboardInterrupt, EOFError): sys.exit(1)
def __fetch_github_api(url, data): try: if data is not None: req = urllib2.Request(url=url, data=data) raw_gh_api = urllib2.urlopen(req) else: raw_gh_api = urllib2.urlopen(url) except urllib2.HTTPError as e: error("Failed to fetch github API '{0}': {1}".format(url, e)) return None return json.load(raw_gh_api)
def ensure_git_root(): """ Checks that you are in the root of the git repository, else exit. :raises: SystemExit if this is not a valid git repository. :raises: SystemExit if not in the root of a git repository. """ root = get_root() if root is None: error("Not in a git repository.", exit=True) if os.getcwd() != root: error("Must call from the top folder of the git repository", exit=True)