def _my_run(cmd, msg=None): if msg: info(fmt("@{bf}@!==> @|@!" + sanitize(msg))) else: info(fmt("@{bf}@!==> @|@!" + sanitize(str(cmd)))) from subprocess import check_call check_call(cmd, shell=True)
def execute_command(cmd, shell=True, autofail=True, silent=True, silent_error=False, cwd=None, return_io=False): """ Executes a given command using Popen. """ out_io = None err_io = None result = 0 if silent: out_io = PIPE err_io = STDOUT debug(((cwd) if cwd else os.getcwd()) + ":$ " + str(cmd)) env = __get_env_for_cmd(cmd) p = Popen(cmd, shell=shell, cwd=cwd, stdout=out_io, stderr=err_io, env=env) out, err = p.communicate() if out is not None and not isinstance(out, str): out = out.decode('utf-8') if err is not None and not isinstance(err, str): err = err.decode('utf-8') result = p.returncode if result != 0: if not silent_error: error("'execute_command' failed to call '{0}'".format(cmd) + " which had a return code ({0}):".format(result)) error("```") info(out, use_prefix=False) error("```") if autofail: raise CalledProcessError(result, cmd) if return_io: return result, out, err else: return result
def execute_command(cmd, shell=True, autofail=True, silent=True, silent_error=False, cwd=None, return_io=False): """ Executes a given command using vcstools' run_shell_command function. """ out_io = None err_io = None result = 0 if silent: out_io = PIPE err_io = STDOUT debug(((cwd) if cwd else os.getcwd()) + ":$ " + str(cmd)) p = Popen(cmd, shell=True, cwd=cwd, stdout=out_io, stderr=err_io) out, err = p.communicate() if out is not None and not isinstance(out, str): out = out.decode('utf-8') if err is not None and not isinstance(err, str): err = err.decode('utf-8') result = p.returncode if result != 0: if not silent_error: error("'execute_command' failed to call '{0}'".format(cmd) + " which had a return code ({0}):".format(result)) error("```") info(out, use_prefix=False) error("```") if autofail: raise CalledProcessError(result, cmd) if return_io: return result, out, err else: return result
def __process_template_folder(path, subs): items = os.listdir(path) processed_items = [] for item in list(items): item = os.path.abspath(os.path.join(path, item)) if os.path.basename(item) in ['.', '..', '.git', '.svn']: continue if os.path.isdir(item): sub_items = __process_template_folder(item, subs) processed_items.extend([os.path.join(item, s) for s in sub_items]) if not item.endswith(TEMPLATE_EXTENSION): continue with open(item, 'r') as f: template = f.read() # Remove extension template_path = item[:-len(TEMPLATE_EXTENSION)] # Expand template info("Expanding '{0}' -> '{1}'".format(os.path.relpath(item), os.path.relpath(template_path))) result = em.expand(template, **subs) # Write the result with open(template_path, 'w') as f: f.write(result) # Copy the permissions shutil.copymode(item, template_path) processed_items.append(item) return processed_items
def get_index_url(): global _rosdistro_index_commit index_url = rosdistro.get_index_url() pr = urlparse(index_url) if pr.netloc == 'raw.github.com': # Try to determine what the commit hash was tokens = [x for x in pr.path.split('/') if x] if len(tokens) <= 3: debug("Failed to get commit for rosdistro index file: index url") debug(tokens) return index_url owner = tokens[0] repo = tokens[1] branch = tokens[2] gh = get_github_interface(quiet=True) if gh is None: # Failed to get it with auth, try without auth (may fail) gh = Github(username=None, auth=None) try: data = gh.get_branch(owner, repo, branch) except GithubException: debug(traceback.format_exc()) debug("Failed to get commit for rosdistro index file: api") return index_url _rosdistro_index_commit = data.get('commit', {}).get('sha', None) if _rosdistro_index_commit is not None: info("ROS Distro index file associate with commit '{0}'" .format(_rosdistro_index_commit)) else: debug("Failed to get commit for rosdistro index file: json") return index_url
def delete_cmd(track): tracks_dict = get_tracks_dict_raw() if track not in tracks_dict['tracks']: error("Track '{0}' does not exist.".format(track), exit=True) del tracks_dict['tracks'][track] info("Deleted track '{0}'.".format(track)) write_tracks_dict_raw(tracks_dict)
def generate_ros_distro_diff(track, repository, distro, distro_file_url, distro_file, distro_file_raw): with inbranch('upstream'): # Check for package.xml(s) try: from catkin_pkg.packages import find_packages except ImportError: debug(traceback.format_exc()) error("catkin_pkg was not detected, please install it.", file=sys.stderr, exit=True) packages = find_packages(os.getcwd()) if len(packages) == 0: warning("No packages found, will not generate 'package: path' entries for rosdistro.") track_dict = get_tracks_dict_raw()['tracks'][track] last_version = track_dict['last_version'] release_inc = track_dict['release_inc'] if repository not in distro_file['repositories']: global _user_provided_release_url distro_file['repositories'][repository] = {'url': _user_provided_release_url or ''} distro_file['repositories'][repository]['version'] = '{0}-{1}'.format(last_version, release_inc) if packages and (len(packages) > 1 or packages.keys()[0] != '.'): distro_file['repositories'][repository]['packages'] = {} for path, package in packages.iteritems(): if os.path.basename(path) == package.name: distro_file['repositories'][repository]['packages'][package.name] = None else: distro_file['repositories'][repository]['packages'][package.name] = path distro_file_name = os.path.join('release', distro_file_url.split('/')[-1]) distro_dump = yaml.dump(distro_file, indent=2, default_flow_style=False) if distro_file_raw != distro_dump: udiff = difflib.unified_diff(distro_file_raw.splitlines(), distro_dump.splitlines(), fromfile=distro_file_name, tofile=distro_file_name) temp_dir = tempfile.mkdtemp() version = distro_file['repositories'][repository]['version'] udiff_file = os.path.join(temp_dir, repository + '-' + version + '.patch') udiff_raw = '' info("Unified diff for the ROS distro file located at '{0}':".format(udiff_file)) for line in udiff: if line.startswith('@@'): udiff_raw += line line = fmt('@{cf}' + line) if line.startswith('+'): if not line.startswith('+++'): line += '\n' udiff_raw += line line = fmt('@{gf}' + line) if line.startswith('-'): if not line.startswith('---'): line += '\n' udiff_raw += line line = fmt('@{rf}' + line) if line.startswith(' '): line += '\n' udiff_raw += line info(line, use_prefix=False, end='') with open(udiff_file, 'w+') as f: f.write(udiff_raw) return udiff_file, distro_dump else: warning("This release resulted in no changes to the ROS distro file...") return None, None
def resolve_rosdep_key(rosdep_key, view, try_again=True): from rosdep2.catkin_support import resolve_for_os from rosdep2.lookup import ResolutionError try: return resolve_for_os(rosdep_key, view, self.apt_installer, os_name, debian_distro) except (KeyError, ResolutionError) as err: if rosdep_key in self.packages: return [sanitize_package_name( 'ros-{0}-{1}'.format(self.rosdistro, rosdep_key) )] if type(err) == KeyError: error( "Could not resolve rosdep key '" + rosdep_key + "'" ) else: error( "Could not resolve the rosdep key '" + rosdep_key + "' for distro '" + debian_distro + "': \n" ) info(str(err), use_prefix=False) if try_again: error("Resolve problem with rosdep and then continue to try again.") if maybe_continue(): self.update_rosdep() new_view = self.get_rosdep_view(debian_distro, os_name) return resolve_rosdep_key(rosdep_key, new_view) self.exit("Failed to resolve rosdep key '{0}', aborting." .format(rosdep_key))
def main(sysargs=None): if len(sysargs if sysargs is not None else sys.argv[1:]) == 0: # This means show me the current config, first check we have an env ensure_clean_working_env() if not branch_exists(BLOOM_CONFIG_BRANCH): sys.exit("No {0} branch found".format(BLOOM_CONFIG_BRANCH)) show_current() info("See: 'git-bloom-config -h' on how to change the configs") return 0 parser = get_argument_parser() add_global_arguments(parser) args = parser.parse_args(sysargs) handle_global_arguments(args) # Also check to see if git has been init'ed check_git_init() # Check that the current directory is a serviceable git/bloom repo try: ensure_clean_working_env() ensure_git_root() except SystemExit: parser.print_usage() raise # Then call the verb try: args.func(args) except (KeyboardInterrupt, EOFError): error("\nUser sent a Keyboard Interrupt, aborting.", exit=True)
def find_version_from_upstream(vcs_uri, vcs_type, devel_branch=None, ros_distro='indigo'): # Check for github.com # if vcs_uri.startswith('http') and 'github.com' in vcs_uri: # info("Detected github.com repository, checking for package.xml " # "in root of devel branch using raw.github.com...") # version = find_version_from_upstream_github(vcs_uri, devel_branch) # if version: # return version, None # warning(" Failed to find the version using raw.github.com.") # Try to clone the upstream repository info("Checking upstream devel branch '{0}' for package.xml(s)".format( devel_branch or '<default>')) upstream_repo = get_upstream_repo(vcs_uri, vcs_type) if not upstream_repo.checkout(vcs_uri, devel_branch or ''): error("Failed to checkout to the upstream branch " "'{0}' in the repository from '{1}'".format( devel_branch or '<default>', vcs_uri), exit=True) meta = get_upstream_meta(upstream_repo.get_path(), ros_distro) if not meta: error( "Failed to find any package.xml(s) in the " "upstream devel branch '{0}' in the repository from '{1}'".format( devel_branch or '<default>', vcs_uri)) info("Detected version '{0}' from package(s): {1}".format( meta['version'], meta['name'])) return meta['version'], upstream_repo
def commit(self): if self.disabled: return info( fmt("@{bf}<==@| Command successful, committing changes to working copy" )) current_branch = get_current_branch() if current_branch is None: error("Could not determine current branch.", exit=True) with inbranch(get_commit_hash(get_current_branch())): with change_directory(self.clone_dir): new_branches = get_branches() for branch in self.current_branches: if branch in new_branches: new_branches.remove(branch) for branch in get_branches(local_only=True): if branch not in new_branches: with inbranch(branch): cmd = 'git pull --rebase origin ' + branch execute_command(cmd) execute_command('git push --all', silent=False) try: execute_command('git push --tags', silent=False) except subprocess.CalledProcessError: warning( "Force pushing tags from clone to working repository, " "you will have to force push back to origin...") execute_command('git push --force --tags', silent=False) self.clean_up()
def main(sysargs=None): # Check that the current directory is a serviceable git/bloom repo ensure_clean_working_env() ensure_git_root() # Get tracks tracks_dict = get_tracks_dict_raw() if not tracks_dict['tracks']: error("No tracks configured, first create a track with " "'git-bloom-config new <track_name>'", exit=True) # Do argparse stuff parser = get_argument_parser([str(t) for t in tracks_dict['tracks']]) parser = add_global_arguments(parser) args = parser.parse_args(sysargs) handle_global_arguments(args) verify_track(args.track, tracks_dict['tracks'][args.track]) execute_track(args.track, tracks_dict['tracks'][args.track], args.release_increment, args.pretend, args.debug, args.unsafe) # Notify the user of success and next action suggestions print('\n\n') warning("Tip: Check to ensure that the debian tags created have the same " "version as the upstream version you are releasing.") info(fmt("@{gf}@!Everything went as expected, " "you should check that the new tags match your expectations, and " "then push to the release repo with:@|")) info(fmt(" git push --all && git push --tags " "@{kf}@!# You might have to add --force to the second command if you " "are over-writing existing flags"))
def create_from_template(self, template_name, data, directory, chmod=None, outfile=None): # Configure template name extention = '.em' if not template_name.endswith(extention): template_file = template_name + extention else: template_file = template_name template_name = template_name[:len(extention)] # Open the template with change_directory(directory): with open(template_file, 'r') as f: template = f.read() execute_command('git rm ' + template_file) # Expand template outfile = outfile if outfile is not None else template_name info("Expanding template: '" + template_file + "' to '" + outfile + "'") result = em.expand(template, **data) # Write the template out with change_directory(directory): with open(outfile, 'w+') as f: f.write(result) # Set permissions if needed if chmod is not None: os.chmod(outfile, chmod)
def main(sysargs=None): if len(sysargs if sysargs is not None else sys.argv[1:]) == 0: # This means show me the current config, first check we have an env ensure_clean_working_env() if not branch_exists('bloom'): sys.exit("No bloom branch found") show_current() info("See: 'git-bloom-config -h' on how to change the configs") return 0 parser = get_argument_parser() add_global_arguments(parser) args = parser.parse_args(sysargs) handle_global_arguments(args) # Also check to see if git has been init'ed check_git_init() # Check that the current directory is a serviceable git/bloom repo try: ensure_clean_working_env() ensure_git_root() except SystemExit: parser.print_usage() raise # Then call the verb try: args.func(args) except (KeyboardInterrupt, EOFError): error("\nUser sent a Keyboard Interrupt, aborting.", exit=True)
def generate_rpm(self, package, rpm_distro, rpm_dir='rpm'): info("Generating RPM for {0} {1}...".format(self.os_name, rpm_distro)) # Try to retrieve the releaser_history releaser_history = self.get_releaser_history() # Generate substitution values subs = self.get_subs(package, rpm_distro, releaser_history) # Use subs to create and store releaser history self.set_releaser_history(dict(subs['changelogs'])) # Template files template_files = process_template_files('.', subs) # Remove any residual template files execute_command('git rm -rf ' + ' '.join("'{}'".format(t) for t in template_files)) # Add marker file to tell mock to archive the sources open('.write_tar', 'a').close() # Add marker file changes to the rpm folder execute_command('git add .write_tar ' + rpm_dir) # Commit changes execute_command('git commit -m "Generated RPM files for ' + rpm_distro + '"') # Rename the template spec file execute_command('git mv ' + rpm_dir + '/template.spec ' + rpm_dir + '/' + subs['Package'] + '.spec') # Commit changes execute_command('git commit -m "Renamed RPM spec file for ' + rpm_distro + '"') # Return the subs for other use return subs
def process_template_files(path, subs): info(fmt("@!@{bf}==>@| In place processing templates in 'rpm' folder.")) rpm_dir = os.path.join(path, 'rpm') if not os.path.exists(rpm_dir): sys.exit("No rpm directory found at '{0}', cannot process templates." .format(rpm_dir)) return __process_template_folder(rpm_dir, subs)
def execute_command(cmd, shell=True, autofail=True, silent=True, silent_error=False, cwd=None, return_io=False): """ Executes a given command using vcstools' run_shell_command function. """ out_io = None err_io = None result = 0 if silent: out_io = PIPE err_io = STDOUT debug(((cwd) if cwd else os.getcwd()) + ":$ " + str(cmd)) p = Popen(cmd, shell=True, cwd=cwd, stdout=out_io, stderr=err_io) out, err = p.communicate() result = p.returncode if result != 0: if not silent_error: error("'execute_command' failed to call '{0}'".format(cmd) + " which had a return code ({0}):".format(result)) error("```") info(out, use_prefix=False) error("```") if autofail: raise CalledProcessError(result, cmd) if return_io: return result, out, err else: return result
def __process_template_folder(path, subs): items = os.listdir(path) processed_items = [] for item in list(items): item = os.path.abspath(os.path.join(path, item)) if os.path.basename(item) in ['.', '..', '.git', '.svn']: continue if os.path.isdir(item): sub_items = __process_template_folder(item, subs) processed_items.extend([os.path.join(item, s) for s in sub_items]) if not item.endswith(TEMPLATE_EXTENSION): continue with open(item, 'r') as f: template = f.read() # Remove extension template_path = item[:-len(TEMPLATE_EXTENSION)] # Expand template info("Expanding '{0}' -> '{1}'".format(os.path.relpath(item), os.path.relpath(template_path))) result = em.expand(template, **subs) # Don't write an empty file if len(result) == 0 and \ os.path.basename(template_path) in ['copyright']: processed_items.append(item) continue # Write the result with io.open(template_path, 'w', encoding='utf-8') as f: if sys.version_info.major == 2: result = result.decode('utf-8') f.write(result) # Copy the permissions shutil.copymode(item, template_path) processed_items.append(item) return processed_items
def resolve_rosdep_key(key, os_name, os_version, ros_distro=None, ignored=None, retry=True): ignored = ignored or [] ctx = create_default_installer_context() try: installer_key = ctx.get_default_os_installer_key(os_name) except KeyError: BloomGenerator.exit("Could not determine the installer for '{0}'".format(os_name)) installer = ctx.get_installer(installer_key) ros_distro = ros_distro or DEFAULT_ROS_DISTRO view = get_view(os_name, os_version, ros_distro) try: return resolve_more_for_os(key, view, installer, os_name, os_version) except (KeyError, ResolutionError) as exc: debug(traceback.format_exc()) if key in ignored: return None, None, None if isinstance(exc, KeyError): error("Could not resolve rosdep key '{0}'".format(key)) returncode = code.GENERATOR_NO_SUCH_ROSDEP_KEY else: error("Could not resolve rosdep key '{0}' for distro '{1}':".format(key, os_version)) info(str(exc), use_prefix=False) returncode = code.GENERATOR_NO_ROSDEP_KEY_FOR_DISTRO if retry: error("Try to resolve the problem with rosdep and then continue.") if maybe_continue(): update_rosdep() invalidate_view_cache() return resolve_rosdep_key(key, os_name, os_version, ros_distro, ignored, retry=True) BloomGenerator.exit("Failed to resolve rosdep key '{0}', aborting.".format(key), returncode=returncode)
def post_rebase(self, destination): name = destination.split('/')[-1] # Retrieve the package package = self.packages[name] # Handle differently if this is a debian vs distro branch if destination in self.debian_branches: info("Placing debian template files into '{0}' branch.".format( destination)) # Then this is a debian branch # Place the raw template files self.place_template_files() else: # This is a distro specific debian branch # Determine the current package being generated distro = destination.split('/')[-2] # Create debians for each distro with inbranch(destination): data = self.generate_debian(package, distro) # Create the tag name for later self.tag_names[destination] = self.generate_tag_name(data) # Update the patch configs patches_branch = 'patches/' + destination config = get_patch_config(patches_branch) # Store it self.store_original_config(config, patches_branch) # Modify the base so import/export patch works current_branch = get_current_branch() if current_branch is None: error("Could not determine current branch.", exit=True) config['base'] = get_commit_hash(current_branch) # Set it set_patch_config(patches_branch, config)
def post_rebase(self, destination): name = destination.split('/')[-1] # Retrieve the package package = self.packages[name] # Handle differently if this is an arch vs distro branch if destination in self.arch_branches: info("Placing Arch template files into '{0}' branch." .format(destination)) # Then this is an arch branch # Place the raw template files self.place_template_files() else: # This is a distro specific arch branch # Determine the current package being generated distro = destination.split('/')[-2] # Create Arch packages for each distro with inbranch(destination): # To fit Arch Linux philosophy a bit better, we move all the source files into a subdirectory. # Arch Linux doesn't support source distribution through a subdirectory; therefore we should ideally compress the sources or provide a URL. # At this point in the generator, it is tricky to get the release URL. Furthermore it wouldn't fit bloom's patch mechanism very well. # To work around, we copy the sources to the $srcdir at the beginning inside the prepare() function. temp_dir = mkdtemp(dir='.') for item in os.listdir("."): itemsrc = os.path.abspath(item) if os.path.basename(itemsrc) in ['.', '..', '.git', '.svn', 'arch',os.path.basename(temp_dir)]: continue itemdst = os.path.abspath(os.path.join(temp_dir,item)) execute_command('git mv ' + itemsrc + ' ' + itemdst) execute_command('git mv ' + temp_dir + ' ' + name) execute_command('git commit --amend --no-edit') # Then generate the PKGBUILD data = self.generate_arch(package, distro) # And finally move the PKGBUILD to the root directory of the package. for item in os.listdir("arch"): itemsrc = os.path.abspath(os.path.join("arch",item)) if os.path.basename(itemsrc) in ['.', '..', '.git', '.svn']: continue itemdst = os.path.abspath(item) execute_command('git mv ' + itemsrc + ' ' + itemdst) execute_command('git commit --amend --no-edit') # Create the tag name for later self.tag_names[destination] = self.generate_tag_name(data) # Update the patch configs patches_branch = 'patches/' + destination config = get_patch_config(patches_branch) # Store it self.store_original_config(config, patches_branch) # Modify the base so import/export patch works current_branch = get_current_branch() if current_branch is None: error("Could not determine current branch.", exit=True) config['base'] = get_commit_hash(current_branch) # Set it set_patch_config(patches_branch, config)
def __process_template_folder(path, subs): items = os.listdir(path) processed_items = [] for item in list(items): item = os.path.abspath(os.path.join(path, item)) if os.path.basename(item) in ['.', '..', '.git', '.svn']: continue if os.path.isdir(item): sub_items = __process_template_folder(item, subs) processed_items.extend([os.path.join(item, s) for s in sub_items]) if not item.endswith(TEMPLATE_EXTENSION): continue with open(item, 'r') as f: template = f.read() # Remove extension template_path = item[:-len(TEMPLATE_EXTENSION)] # Expand template info("Expanding '{0}' -> '{1}'".format( os.path.relpath(item), os.path.relpath(template_path))) result = em.expand(template, **subs) # Write the result with open(template_path, 'w') as f: f.write(result.encode('utf8')) # Copy the permissions shutil.copymode(item, template_path) processed_items.append(item) return processed_items
def pre_modify(self): info("\nPre-verifying RPM dependency keys...") # Run rosdep update is needed if not self.has_run_rosdep: self.update_rosdep() peer_packages = [p.name for p in self.packages.values()] while not self._check_all_keys_are_valid(peer_packages, self.rosdistro): error("Some of the dependencies for packages in this repository could not be resolved by rosdep.") error("You can try to address the issues which appear above and try again if you wish, " "or continue without releasing into RPM-based distributions (e.g. Fedora 24).") try: if not maybe_continue(msg="Would you like to try again?"): error("User aborted after rosdep keys were not resolved.") sys.exit(code.GENERATOR_NO_ROSDEP_KEY_FOR_DISTRO) except (KeyboardInterrupt, EOFError): error("\nUser quit.", exit=True) update_rosdep() invalidate_view_cache() info("All keys are " + ansi('greenf') + "OK" + ansi('reset') + "\n") for package in self.packages.values(): if not package.licenses or not package.licenses[0]: error("No license set for package '{0}', aborting.".format(package.name), exit=True)
def pre_modify(self): info("\nPre-verifying Debian dependency keys...") # Run rosdep update is needed if not self.has_run_rosdep: self.update_rosdep() peer_packages = [p.name for p in self.packages.values()] while not self._check_all_keys_are_valid(peer_packages): error( "Some of the dependencies for packages in this repository could not be resolved by rosdep." ) error( "You can try to address the issues which appear above and try again if you wish." ) try: if not maybe_continue(msg="Would you like to try again?"): error("User aborted after rosdep keys were not resolved.") sys.exit(code.GENERATOR_NO_ROSDEP_KEY_FOR_DISTRO) except (KeyboardInterrupt, EOFError): error("\nUser quit.", exit=True) update_rosdep() invalidate_view_cache() info("All keys are " + ansi('greenf') + "OK" + ansi('reset') + "\n")
def main(sysargs=None): parser = get_argument_parser() parser = add_global_arguments(parser) args = parser.parse_args(sysargs) handle_global_arguments(args) # Check that the current directory is a serviceable git/bloom repo try: ensure_clean_working_env() ensure_git_root() except SystemExit: parser.print_usage() raise git_clone = GitClone() with git_clone: import_upstream( args.archive_path, args.patches_path, args.release_version, args.name, args.replace) git_clone.commit() info("I'm happy. You should be too.")
def get_repo_uri(repository, distro): url = None # Fetch the distro file distribution_file = get_distribution_file(distro) if repository in distribution_file.repositories and \ distribution_file.repositories[repository].release_repository is not None: url = distribution_file.repositories[repository].release_repository.url else: error("Specified repository '{0}' is not in the distribution file located at '{1}'" .format(repository, get_disitrbution_file_url(distro))) matches = difflib.get_close_matches(repository, distribution_file.repositories) if matches: info(fmt("@{yf}Did you mean one of these: '" + "', '".join([m for m in matches]) + "'?")) if not url: info("Could not determine release repository url for repository '{0}' of distro '{1}'" .format(repository, distro)) info("You can continue the release process by manually specifying the location of the RELEASE repository.") info("To be clear this is the url of the RELEASE repository not the upstream repository.") try: url = safe_input('Release repository url [press enter to abort]: ') except (KeyboardInterrupt, EOFError): url = None info('', use_prefix=False) if not url: error("No release repository url given, aborting.", exit=True) global _user_provided_release_url _user_provided_release_url = url return url
def pre_modify(self): info("\nPre-verifying RPM dependency keys...") # Run rosdep update is needed if not self.has_run_rosdep: self.update_rosdep() peer_packages = [p.name for p in self.packages.values()] while not self._check_all_keys_are_valid(peer_packages, self.rosdistro): error("Some of the dependencies for packages in this repository could not be resolved by rosdep.") if not self.interactive: sys.exit(code.GENERATOR_NO_ROSDEP_KEY_FOR_DISTRO) error("You can try to address the issues which appear above and try again if you wish, " "or continue without releasing into RPM-based distributions (e.g. Fedora 24).") try: if not maybe_continue(msg="Would you like to try again?"): error("User aborted after rosdep keys were not resolved.") sys.exit(code.GENERATOR_NO_ROSDEP_KEY_FOR_DISTRO) except (KeyboardInterrupt, EOFError): error("\nUser quit.", exit=True) update_rosdep() invalidate_view_cache() info("All keys are " + ansi('greenf') + "OK" + ansi('reset') + "\n") for package in self.packages.values(): if not package.licenses or not package.licenses[0]: error("No license set for package '{0}', aborting.".format(package.name), exit=True)
def summarize_arguments(upstream_repo, upstream_repo_type, upstream_repo_branch): # Summarize the requested operation summary_msg = "Upstream " + ansi('boldon') + upstream_repo summary_msg += ansi('boldoff') + " type: " + ansi('boldon') summary_msg += upstream_repo_type + ansi('boldoff') info(summary_msg)
def commit(self): if self.disabled: return info(fmt("@{bf}<==@| Command successful, committing changes to working copy")) current_branch = get_current_branch() if current_branch is None: error("Could not determine current branch.", exit=True) with inbranch(get_commit_hash(get_current_branch())): with change_directory(self.clone_dir): new_branches = get_branches() for branch in self.current_branches: if branch in new_branches: new_branches.remove(branch) for branch in get_branches(local_only=True): if branch not in new_branches: with inbranch(branch): cmd = 'git pull --rebase origin ' + branch execute_command(cmd) execute_command('git push --all', silent=False) try: execute_command('git push --tags', silent=False) except subprocess.CalledProcessError: warning("Force pushing tags from clone to working repository, " "you will have to force push back to origin...") execute_command('git push --force --tags', silent=False) self.clean_up()
def update_rosdep(): info("Running 'rosdep update'...") try: rosdep2.catkin_support.update_rosdep() except: print_exc(traceback.format_exc()) error("Failed to update rosdep, did you run 'rosdep init' first?", exit=True)
def post_rebase(self, destination): name = destination.split('/')[-1] # Retrieve the package package = self.packages[name] # Handle differently if this is an rpm vs distro branch if destination in self.rpm_branches: info("Placing RPM template files into '{0}' branch." .format(destination)) # Then this is an rpm branch # Place the raw template files self.place_template_files() else: # This is a distro specific rpm branch # Determine the current package being generated distro = destination.split('/')[-2] # Create RPMs for each distro with inbranch(destination): data = self.generate_rpm(package, distro) # Create the tag name for later self.tag_names[destination] = self.generate_tag_name(data) # Update the patch configs patches_branch = 'patches/' + destination config = get_patch_config(patches_branch) # Store it self.store_original_config(config, patches_branch) # Modify the base so import/export patch works current_branch = get_current_branch() if current_branch is None: error("Could not determine current branch.", exit=True) config['base'] = get_commit_hash(current_branch) # Set it set_patch_config(patches_branch, config)
def create_from_template(self, template_name, data, directory, chmod=None, outfile=None): # Configure template name extention = '.em' if not template_name.endswith(extention): template_file = template_name + extention else: template_file = template_name template_name = template_name[:len(extention)] template_path = os.path.join('templates', template_file) # Get the template contents using pkg_resources group = 'bloom.generators.debian' # info("Looking for template: " + group + ':' + template_path) try: template = pkg_resources.resource_string(group, template_path) except IOError as err: error("Failed to load template " "'{0}': {1}".format(template_name, str(err))) self.exit(code.DEBIAN_FAILED_TO_LOAD_TEMPLATE) # Expand template outfile = outfile if outfile is not None else template_name info("Expanding template: '" + template_file + "' to '" + \ outfile + "'") result = em.expand(template, **data) # Write the template out with change_directory(directory): with open(outfile, 'w+') as f: f.write(result) # Set permissions if needed if chmod is not None: os.chmod(outfile, chmod)
def _my_run(cmd): info(fmt("@{bf}@!==> @|@!" + str(cmd))) # out = check_output(cmd, stderr=subprocess.STDOUT, shell=True) out = None from subprocess import call call(cmd, shell=True) if out: info(out, use_prefix=False)
def pre_rebase(self, destination): name = destination.split('/')[-1] info("Releasing package '" + name + "' to: '" + destination + "'") ret = trim(undo=True) if ret == code.NOTHING_TO_DO: return 0 else: return ret
def import_patches(patches_path, patches_path_dict, target_branch, version): info("Overlaying files from patched folder '{0}' on the '{2}' branch into the '{1}' branch..." .format(patches_path, target_branch, BLOOM_CONFIG_BRANCH)) with inbranch(target_branch): handle_tree(patches_path_dict, '', patches_path, version) cmd = ('git commit --allow-empty -m "Overlaid patches from \'{0}\'"' .format(patches_path)) execute_command(cmd, shell=True)
def show(args): tracks_dict = get_tracks_dict_raw() if args.track not in tracks_dict['tracks']: error("Track '{0}' does not exist.".format(args.track), exit=True) info( yaml.dump({args.track: tracks_dict['tracks'][args.track]}, indent=2, default_flow_style=False))
def pre_rebase(self, destination, msg=None): name = destination.split('/')[-1] msg = msg if msg is not None else ( "Releasing package '" + name + "' to: '" + destination + "'" ) info(msg) ret = trim(undo=True) return 0 if ret < 0 else ret # Ret < 0 indicates nothing was done
def place_template_files(path): info(fmt("@!@{bf}==>@| Placing templates files in the 'arch' folder.")) arch_path = os.path.join(path, 'arch') # Create/Clean the arch folder if not os.path.exists(arch_path): os.makedirs(arch_path) # Place template files group = 'bloom.generators.arch' __place_template_folder(group, 'templates', arch_path)
def summarize(self): self.branch_list = self.detect_branches() if type(self.branch_list) not in [list, tuple]: self.exit(self.branch_list if self.branch_list is not None else 1) info( "Releasing package" + \ ('' if len(self.branch_list) == 1 else 's') + ": " + \ str(self.branch_list) )
def show_current(): if os.path.exists('bloom.conf'): info("Current bloom configuration:") f = open('bloom.conf', 'r') print('') map(info, [l.rstrip() for l in f.read().splitlines()]) print('') else: info("No bloom.conf in the bloom branch")
def get_release_repo(repository, distro): global _repositories url = get_repo_uri(repository, distro) if repository not in _repositories.values(): temp_dir = tempfile.mkdtemp() _repositories[repository] = get_vcs_client("git", temp_dir) info(fmt("@{gf}@!==> @|") + "Fetching '{0}' repository from '{1}'".format(repository, url)) _repositories[repository].checkout(url, "master") return _repositories[repository]
def place_template_files(path, gbp=False): info(fmt("@!@{bf}==>@| Placing templates files in the 'debian' folder.")) debian_path = os.path.join(path, 'debian') # Create/Clean the debian folder if not os.path.exists(debian_path): os.makedirs(debian_path) # Place template files group = 'bloom.generators.debian' __place_template_folder(group, 'templates', debian_path, gbp)
def place_template_files(path, gbp=False): info(fmt("@!@{bf}==>@| Placing templates files in the 'rpm' folder.")) rpm_path = os.path.join(path, 'rpm') # Create/Clean the rpm folder if not os.path.exists(rpm_path): os.makedirs(rpm_path) # Place template files group = 'bloom.generators.rpm' __place_template_folder(group, 'templates', rpm_path, gbp)
def copy_track(src, dst): tracks_dict = get_tracks_dict_raw() if src not in tracks_dict['tracks']: error("Track '{0}' does not exist.".format(src), exit=True) if dst in tracks_dict['tracks']: error("Track '{0}' already exists.".format(dst), exit=True) tracks_dict['tracks'][dst] = copy.deepcopy(tracks_dict['tracks'][src]) info("Saving '{0}' track.".format(dst)) write_tracks_dict_raw(tracks_dict)
def main(args=None, get_subs_fn=None): get_subs_fn = get_subs_fn or get_subs _place_template_files = True _process_template_files = True package_path = os.getcwd() if args is not None: package_path = args.package_path or os.getcwd() _place_template_files = args.place_template_files _process_template_files = args.process_template_files pkgs_dict = find_packages(package_path) if len(pkgs_dict) == 0: sys.exit("No packages found in path: '{0}'".format(package_path)) if len(pkgs_dict) > 1: sys.exit("Multiple packages found, " "this tool only supports one package at a time.") os_data = create_default_installer_context().get_os_name_and_version() os_name, os_version = os_data ros_distro = os.environ.get("ROS_DISTRO", "indigo") # Allow args overrides os_name = args.os_name or os_name os_version = args.os_version or os_version ros_distro = args.ros_distro or ros_distro # Summarize info( fmt("@!@{gf}==> @|") + fmt( "Generating debs for @{cf}%s:%s@| for package(s) %s" % (os_name, os_version, [p.name for p in pkgs_dict.values()]) ) ) for path, pkg in pkgs_dict.items(): template_files = None try: subs = get_subs_fn(pkg, os_name, os_version, ros_distro) if _place_template_files: # Place template files place_template_files(path) if _process_template_files: # Just process existing template files template_files = process_template_files(path, subs) if not _place_template_files and not _process_template_files: # If neither, do both place_template_files(path) template_files = process_template_files(path, subs) if template_files is not None: for template_file in template_files: os.remove(os.path.normpath(template_file)) except Exception as exc: debug(traceback.format_exc()) error(type(exc).__name__ + ": " + str(exc), exit=True) except (KeyboardInterrupt, EOFError): sys.exit(1)
def place_template_files(path, build_type, gbp=False): info(fmt("@!@{bf}==>@| Placing templates files in the 'rpm' folder.")) rpm_path = os.path.join(path, 'rpm') # Create/Clean the rpm folder if not os.path.exists(rpm_path): os.makedirs(rpm_path) # Place template files group = 'bloom.generators.rpm' templates = os.path.join('templates', build_type) __place_template_folder(group, templates, rpm_path, gbp)
def build_debian_pkg(args=None, get_subs_fn=None): get_subs_fn = get_subs_fn or get_subs _place_template_files = True _process_template_files = True package_path = os.getcwd() if args is not None: package_path = args.package_path or os.getcwd() _place_template_files = args.place_template_files _process_template_files = args.process_template_files pkgs_dict = find_packages(package_path) if len(pkgs_dict) == 0: sys.exit("No packages found in path: '{0}'".format(package_path)) # if len(pkgs_dict) > 1: # sys.exit("Multiple packages found, " # "this tool only supports one package at a time.") os_data = create_default_installer_context().get_os_name_and_version() os_name, os_version = os_data ros_distro = os.environ.get('ROS_DISTRO', 'indigo') # Allow args overrides os_name = args.os_name or os_name os_version = args.os_version or os_version ros_distro = args.ros_distro or ros_distro install_prefix = args.install_prefix or "/opt" # Summarize info(fmt("@!@{gf}==> @|") + fmt("Generating debs for @{cf}%s:%s@| for package(s) %s" % (os_name, os_version, [p.name for p in pkgs_dict.values()]))) # Test Creating single all_subs = merge_packages(pkgs_dict, get_subs_fn, os_name, os_version, ros_distro, install_prefix, args.native) path = '' build_type = 'cmake' try: if _place_template_files: # Place template files place_template_files(path, build_type) if _process_template_files: # Just process existing template files template_files = process_template_files(path, all_subs) if not _place_template_files and not _process_template_files: # If neither, do both place_template_files(path, build_type) template_files = process_template_files(path, all_subs) if template_files is not None: for template_file in template_files: os.remove(os.path.normpath(template_file)) except Exception as exc: debug(traceback.format_exc()) error(type(exc).__name__ + ": " + str(exc), exit=True) except (KeyboardInterrupt, EOFError): sys.exit(1)
def get_release_repo(repository, distro): global _repositories url = get_repo_uri(repository, distro) if repository not in _repositories.values(): temp_dir = tempfile.mkdtemp() _repositories[repository] = get_vcs_client('git', temp_dir) info( fmt("@{gf}@!==> @|") + "Fetching '{0}' repository from '{1}'".format(repository, url)) _repositories[repository].checkout(url, 'master') return _repositories[repository]
def place_template_files(path, build_type, gbp=False): info(fmt("@!@{bf}==>@| Placing templates files in the 'debian' folder.")) debian_path = os.path.join(path, 'debian') # Remove the debian folder if it exist if os.path.exists(debian_path): shutil.rmtree(debian_path) # Place template files templates = os.path.join(os.curdir, os.path.join('templates', build_type)) shutil.copytree(os.path.join('templates', build_type), debian_path) if not gbp: os.remove(os.path.join(debian_path, 'gbp.conf.em'))
def upconvert_bloom_to_config_branch(): global _has_checked_bloom_branch if _has_checked_bloom_branch: return # Assert that this repository does not have multiple remotes check_for_multiple_remotes() if get_root() is None: # Not a git repository return track_branches(['bloom', BLOOM_CONFIG_BRANCH]) if show('bloom', PLACEHOLDER_FILE) is not None: return if show('bloom', 'bloom.conf') is not None: # Wait for the bloom.conf upconvert... return if not branch_exists('bloom'): return _has_checked_bloom_branch = True info("Moving configurations from deprecated 'bloom' branch " "to the '{0}' branch.".format(BLOOM_CONFIG_BRANCH)) tmp_dir = mkdtemp() git_root = get_root() try: # Copy the new upstream source into the temporary directory with inbranch('bloom'): ignores = ('.git', '.gitignore', '.svn', '.hgignore', '.hg', 'CVS') configs = os.path.join(tmp_dir, 'configs') my_copytree(git_root, configs, ignores) if [x for x in os.listdir(os.getcwd()) if x not in ignores]: execute_command('git rm -rf ./*') with open(PLACEHOLDER_FILE, 'w') as f: f.write("""\ This branch ('bloom') has been deprecated in favor of storing settings and overlay files in the master branch. Please goto the master branch for anything which referenced the bloom branch. You can delete this branch at your convenience. """) execute_command('git add ' + PLACEHOLDER_FILE) if has_changes(): execute_command('git commit -m "DEPRECATING BRANCH"') if not branch_exists(BLOOM_CONFIG_BRANCH): info("Creating '{0}' branch.".format(BLOOM_CONFIG_BRANCH)) create_branch(BLOOM_CONFIG_BRANCH, orphaned=True) with inbranch(BLOOM_CONFIG_BRANCH): my_copytree(configs, git_root) execute_command('git add ./*') if has_changes(): execute_command('git commit -m ' '"Moving configs from bloom branch"') finally: # Clean up if os.path.exists(tmp_dir): shutil.rmtree(tmp_dir)
def print_exc(exc): exc_str = ''.join(exc) try: from pygments import highlight from pygments.lexers import PythonTracebackLexer from pygments.formatters import TerminalFormatter exc_str = highlight(exc_str, PythonTracebackLexer(), TerminalFormatter()) except ImportError: pass info(exc_str, file=sys.stderr, use_prefix=False)