def _my_run(cmd, msg=None):
if msg:
info(fmt("@{bf}@!==> @|@!" + sanitize(msg)))
else:
info(fmt("@{bf}@!==> @|@!" + sanitize(str(cmd))))
from subprocess import check_call
check_call(cmd, shell=True)
def execute_command(cmd, shell=True, autofail=True, silent=True,
silent_error=False, cwd=None, return_io=False):
"""
Executes a given command using Popen.
"""
out_io = None
err_io = None
result = 0
if silent:
out_io = PIPE
err_io = STDOUT
debug(((cwd) if cwd else os.getcwd()) + ":$ " + str(cmd))
env = __get_env_for_cmd(cmd)
p = Popen(cmd, shell=shell, cwd=cwd, stdout=out_io, stderr=err_io, env=env)
out, err = p.communicate()
if out is not None and not isinstance(out, str):
out = out.decode('utf-8')
if err is not None and not isinstance(err, str):
err = err.decode('utf-8')
result = p.returncode
if result != 0:
if not silent_error:
error("'execute_command' failed to call '{0}'".format(cmd) +
" which had a return code ({0}):".format(result))
error("```")
info(out, use_prefix=False)
error("```")
if autofail:
raise CalledProcessError(result, cmd)
if return_io:
return result, out, err
else:
return result
def execute_command(cmd, shell=True, autofail=True, silent=True,
silent_error=False, cwd=None, return_io=False):
"""
Executes a given command using vcstools' run_shell_command function.
"""
out_io = None
err_io = None
result = 0
if silent:
out_io = PIPE
err_io = STDOUT
debug(((cwd) if cwd else os.getcwd()) + ":$ " + str(cmd))
p = Popen(cmd, shell=True, cwd=cwd, stdout=out_io, stderr=err_io)
out, err = p.communicate()
if out is not None and not isinstance(out, str):
out = out.decode('utf-8')
if err is not None and not isinstance(err, str):
err = err.decode('utf-8')
result = p.returncode
if result != 0:
if not silent_error:
error("'execute_command' failed to call '{0}'".format(cmd) +
" which had a return code ({0}):".format(result))
error("```")
info(out, use_prefix=False)
error("```")
if autofail:
raise CalledProcessError(result, cmd)
if return_io:
return result, out, err
else:
return result
def __process_template_folder(path, subs):
items = os.listdir(path)
processed_items = []
for item in list(items):
item = os.path.abspath(os.path.join(path, item))
if os.path.basename(item) in ['.', '..', '.git', '.svn']:
continue
if os.path.isdir(item):
sub_items = __process_template_folder(item, subs)
processed_items.extend([os.path.join(item, s) for s in sub_items])
if not item.endswith(TEMPLATE_EXTENSION):
continue
with open(item, 'r') as f:
template = f.read()
# Remove extension
template_path = item[:-len(TEMPLATE_EXTENSION)]
# Expand template
info("Expanding '{0}' -> '{1}'".format(os.path.relpath(item),
os.path.relpath(template_path)))
result = em.expand(template, **subs)
# Write the result
with open(template_path, 'w') as f:
f.write(result)
# Copy the permissions
shutil.copymode(item, template_path)
processed_items.append(item)
return processed_items
def get_index_url():
global _rosdistro_index_commit
index_url = rosdistro.get_index_url()
pr = urlparse(index_url)
if pr.netloc == 'raw.github.com':
# Try to determine what the commit hash was
tokens = [x for x in pr.path.split('/') if x]
if len(tokens) <= 3:
debug("Failed to get commit for rosdistro index file: index url")
debug(tokens)
return index_url
owner = tokens[0]
repo = tokens[1]
branch = tokens[2]
gh = get_github_interface(quiet=True)
if gh is None:
# Failed to get it with auth, try without auth (may fail)
gh = Github(username=None, auth=None)
try:
data = gh.get_branch(owner, repo, branch)
except GithubException:
debug(traceback.format_exc())
debug("Failed to get commit for rosdistro index file: api")
return index_url
_rosdistro_index_commit = data.get('commit', {}).get('sha', None)
if _rosdistro_index_commit is not None:
info("ROS Distro index file associate with commit '{0}'"
.format(_rosdistro_index_commit))
else:
debug("Failed to get commit for rosdistro index file: json")
return index_url
def delete_cmd(track):
tracks_dict = get_tracks_dict_raw()
if track not in tracks_dict['tracks']:
error("Track '{0}' does not exist.".format(track), exit=True)
del tracks_dict['tracks'][track]
info("Deleted track '{0}'.".format(track))
write_tracks_dict_raw(tracks_dict)
def delete_cmd(track):
tracks_dict = get_tracks_dict_raw()
if track not in tracks_dict['tracks']:
error("Track '{0}' does not exist.".format(track), exit=True)
del tracks_dict['tracks'][track]
info("Deleted track '{0}'.".format(track))
write_tracks_dict_raw(tracks_dict)
def generate_ros_distro_diff(track, repository, distro, distro_file_url, distro_file, distro_file_raw):
with inbranch('upstream'):
# Check for package.xml(s)
try:
from catkin_pkg.packages import find_packages
except ImportError:
debug(traceback.format_exc())
error("catkin_pkg was not detected, please install it.",
file=sys.stderr, exit=True)
packages = find_packages(os.getcwd())
if len(packages) == 0:
warning("No packages found, will not generate 'package: path' entries for rosdistro.")
track_dict = get_tracks_dict_raw()['tracks'][track]
last_version = track_dict['last_version']
release_inc = track_dict['release_inc']
if repository not in distro_file['repositories']:
global _user_provided_release_url
distro_file['repositories'][repository] = {'url': _user_provided_release_url or ''}
distro_file['repositories'][repository]['version'] = '{0}-{1}'.format(last_version, release_inc)
if packages and (len(packages) > 1 or packages.keys()[0] != '.'):
distro_file['repositories'][repository]['packages'] = {}
for path, package in packages.iteritems():
if os.path.basename(path) == package.name:
distro_file['repositories'][repository]['packages'][package.name] = None
else:
distro_file['repositories'][repository]['packages'][package.name] = path
distro_file_name = os.path.join('release', distro_file_url.split('/')[-1])
distro_dump = yaml.dump(distro_file, indent=2, default_flow_style=False)
if distro_file_raw != distro_dump:
udiff = difflib.unified_diff(distro_file_raw.splitlines(), distro_dump.splitlines(),
fromfile=distro_file_name, tofile=distro_file_name)
temp_dir = tempfile.mkdtemp()
version = distro_file['repositories'][repository]['version']
udiff_file = os.path.join(temp_dir, repository + '-' + version + '.patch')
udiff_raw = ''
info("Unified diff for the ROS distro file located at '{0}':".format(udiff_file))
for line in udiff:
if line.startswith('@@'):
udiff_raw += line
line = fmt('@{cf}' + line)
if line.startswith('+'):
if not line.startswith('+++'):
line += '\n'
udiff_raw += line
line = fmt('@{gf}' + line)
if line.startswith('-'):
if not line.startswith('---'):
line += '\n'
udiff_raw += line
line = fmt('@{rf}' + line)
if line.startswith(' '):
line += '\n'
udiff_raw += line
info(line, use_prefix=False, end='')
with open(udiff_file, 'w+') as f:
f.write(udiff_raw)
return udiff_file, distro_dump
else:
warning("This release resulted in no changes to the ROS distro file...")
return None, None
def resolve_rosdep_key(rosdep_key, view, try_again=True):
from rosdep2.catkin_support import resolve_for_os
from rosdep2.lookup import ResolutionError
try:
return resolve_for_os(rosdep_key, view,
self.apt_installer, os_name,
debian_distro)
except (KeyError, ResolutionError) as err:
if rosdep_key in self.packages:
return [sanitize_package_name(
'ros-{0}-{1}'.format(self.rosdistro, rosdep_key)
)]
if type(err) == KeyError:
error(
"Could not resolve rosdep key '" + rosdep_key + "'"
)
else:
error(
"Could not resolve the rosdep key '" + rosdep_key +
"' for distro '" + debian_distro + "': \n"
)
info(str(err), use_prefix=False)
if try_again:
error("Resolve problem with rosdep and then continue to try again.")
if maybe_continue():
self.update_rosdep()
new_view = self.get_rosdep_view(debian_distro, os_name)
return resolve_rosdep_key(rosdep_key, new_view)
self.exit("Failed to resolve rosdep key '{0}', aborting."
.format(rosdep_key))
def main(sysargs=None):
if len(sysargs if sysargs is not None else sys.argv[1:]) == 0:
# This means show me the current config, first check we have an env
ensure_clean_working_env()
if not branch_exists(BLOOM_CONFIG_BRANCH):
sys.exit("No {0} branch found".format(BLOOM_CONFIG_BRANCH))
show_current()
info("See: 'git-bloom-config -h' on how to change the configs")
return 0
parser = get_argument_parser()
add_global_arguments(parser)
args = parser.parse_args(sysargs)
handle_global_arguments(args)
# Also check to see if git has been init'ed
check_git_init()
# Check that the current directory is a serviceable git/bloom repo
try:
ensure_clean_working_env()
ensure_git_root()
except SystemExit:
parser.print_usage()
raise
# Then call the verb
try:
args.func(args)
except (KeyboardInterrupt, EOFError):
error("\nUser sent a Keyboard Interrupt, aborting.", exit=True)
def find_version_from_upstream(vcs_uri,
vcs_type,
devel_branch=None,
ros_distro='indigo'):
# Check for github.com
# if vcs_uri.startswith('http') and 'github.com' in vcs_uri:
# info("Detected github.com repository, checking for package.xml "
# "in root of devel branch using raw.github.com...")
# version = find_version_from_upstream_github(vcs_uri, devel_branch)
# if version:
# return version, None
# warning(" Failed to find the version using raw.github.com.")
# Try to clone the upstream repository
info("Checking upstream devel branch '{0}' for package.xml(s)".format(
devel_branch or '<default>'))
upstream_repo = get_upstream_repo(vcs_uri, vcs_type)
if not upstream_repo.checkout(vcs_uri, devel_branch or ''):
error("Failed to checkout to the upstream branch "
"'{0}' in the repository from '{1}'".format(
devel_branch or '<default>', vcs_uri),
exit=True)
meta = get_upstream_meta(upstream_repo.get_path(), ros_distro)
if not meta:
error(
"Failed to find any package.xml(s) in the "
"upstream devel branch '{0}' in the repository from '{1}'".format(
devel_branch or '<default>', vcs_uri))
info("Detected version '{0}' from package(s): {1}".format(
meta['version'], meta['name']))
return meta['version'], upstream_repo
def commit(self):
if self.disabled:
return
info(
fmt("@{bf}<==@| Command successful, committing changes to working copy"
))
current_branch = get_current_branch()
if current_branch is None:
error("Could not determine current branch.", exit=True)
with inbranch(get_commit_hash(get_current_branch())):
with change_directory(self.clone_dir):
new_branches = get_branches()
for branch in self.current_branches:
if branch in new_branches:
new_branches.remove(branch)
for branch in get_branches(local_only=True):
if branch not in new_branches:
with inbranch(branch):
cmd = 'git pull --rebase origin ' + branch
execute_command(cmd)
execute_command('git push --all', silent=False)
try:
execute_command('git push --tags', silent=False)
except subprocess.CalledProcessError:
warning(
"Force pushing tags from clone to working repository, "
"you will have to force push back to origin...")
execute_command('git push --force --tags', silent=False)
self.clean_up()
def main(sysargs=None):
# Check that the current directory is a serviceable git/bloom repo
ensure_clean_working_env()
ensure_git_root()
# Get tracks
tracks_dict = get_tracks_dict_raw()
if not tracks_dict['tracks']:
error("No tracks configured, first create a track with "
"'git-bloom-config new <track_name>'", exit=True)
# Do argparse stuff
parser = get_argument_parser([str(t) for t in tracks_dict['tracks']])
parser = add_global_arguments(parser)
args = parser.parse_args(sysargs)
handle_global_arguments(args)
verify_track(args.track, tracks_dict['tracks'][args.track])
execute_track(args.track, tracks_dict['tracks'][args.track],
args.release_increment, args.pretend, args.debug, args.unsafe)
# Notify the user of success and next action suggestions
print('\n\n')
warning("Tip: Check to ensure that the debian tags created have the same "
"version as the upstream version you are releasing.")
info(fmt("@{gf}@!Everything went as expected, "
"you should check that the new tags match your expectations, and "
"then push to the release repo with:@|"))
info(fmt(" git push --all && git push --tags "
"@{kf}@!# You might have to add --force to the second command if you "
"are over-writing existing flags"))
def create_from_template(self, template_name, data, directory,
chmod=None, outfile=None):
# Configure template name
extention = '.em'
if not template_name.endswith(extention):
template_file = template_name + extention
else:
template_file = template_name
template_name = template_name[:len(extention)]
# Open the template
with change_directory(directory):
with open(template_file, 'r') as f:
template = f.read()
execute_command('git rm ' + template_file)
# Expand template
outfile = outfile if outfile is not None else template_name
info("Expanding template: '" + template_file + "' to '" +
outfile + "'")
result = em.expand(template, **data)
# Write the template out
with change_directory(directory):
with open(outfile, 'w+') as f:
f.write(result)
# Set permissions if needed
if chmod is not None:
os.chmod(outfile, chmod)
def main(sysargs=None):
if len(sysargs if sysargs is not None else sys.argv[1:]) == 0:
# This means show me the current config, first check we have an env
ensure_clean_working_env()
if not branch_exists('bloom'):
sys.exit("No bloom branch found")
show_current()
info("See: 'git-bloom-config -h' on how to change the configs")
return 0
parser = get_argument_parser()
add_global_arguments(parser)
args = parser.parse_args(sysargs)
handle_global_arguments(args)
# Also check to see if git has been init'ed
check_git_init()
# Check that the current directory is a serviceable git/bloom repo
try:
ensure_clean_working_env()
ensure_git_root()
except SystemExit:
parser.print_usage()
raise
# Then call the verb
try:
args.func(args)
except (KeyboardInterrupt, EOFError):
error("\nUser sent a Keyboard Interrupt, aborting.", exit=True)
def generate_rpm(self, package, rpm_distro, rpm_dir='rpm'):
info("Generating RPM for {0} {1}...".format(self.os_name, rpm_distro))
# Try to retrieve the releaser_history
releaser_history = self.get_releaser_history()
# Generate substitution values
subs = self.get_subs(package, rpm_distro, releaser_history)
# Use subs to create and store releaser history
self.set_releaser_history(dict(subs['changelogs']))
# Template files
template_files = process_template_files('.', subs)
# Remove any residual template files
execute_command('git rm -rf ' + ' '.join("'{}'".format(t) for t in template_files))
# Add marker file to tell mock to archive the sources
open('.write_tar', 'a').close()
# Add marker file changes to the rpm folder
execute_command('git add .write_tar ' + rpm_dir)
# Commit changes
execute_command('git commit -m "Generated RPM files for ' +
rpm_distro + '"')
# Rename the template spec file
execute_command('git mv ' + rpm_dir + '/template.spec ' + rpm_dir + '/' + subs['Package'] + '.spec')
# Commit changes
execute_command('git commit -m "Renamed RPM spec file for ' +
rpm_distro + '"')
# Return the subs for other use
return subs
def process_template_files(path, subs):
info(fmt("@!@{bf}==>@| In place processing templates in 'rpm' folder."))
rpm_dir = os.path.join(path, 'rpm')
if not os.path.exists(rpm_dir):
sys.exit("No rpm directory found at '{0}', cannot process templates."
.format(rpm_dir))
return __process_template_folder(rpm_dir, subs)
def execute_command(cmd, shell=True, autofail=True, silent=True,
silent_error=False, cwd=None, return_io=False):
"""
Executes a given command using vcstools' run_shell_command function.
"""
out_io = None
err_io = None
result = 0
if silent:
out_io = PIPE
err_io = STDOUT
debug(((cwd) if cwd else os.getcwd()) + ":$ " + str(cmd))
p = Popen(cmd, shell=True, cwd=cwd, stdout=out_io, stderr=err_io)
out, err = p.communicate()
result = p.returncode
if result != 0:
if not silent_error:
error("'execute_command' failed to call '{0}'".format(cmd) +
" which had a return code ({0}):".format(result))
error("```")
info(out, use_prefix=False)
error("```")
if autofail:
raise CalledProcessError(result, cmd)
if return_io:
return result, out, err
else:
return result
def __process_template_folder(path, subs):
items = os.listdir(path)
processed_items = []
for item in list(items):
item = os.path.abspath(os.path.join(path, item))
if os.path.basename(item) in ['.', '..', '.git', '.svn']:
continue
if os.path.isdir(item):
sub_items = __process_template_folder(item, subs)
processed_items.extend([os.path.join(item, s) for s in sub_items])
if not item.endswith(TEMPLATE_EXTENSION):
continue
with open(item, 'r') as f:
template = f.read()
# Remove extension
template_path = item[:-len(TEMPLATE_EXTENSION)]
# Expand template
info("Expanding '{0}' -> '{1}'".format(os.path.relpath(item),
os.path.relpath(template_path)))
result = em.expand(template, **subs)
# Don't write an empty file
if len(result) == 0 and \
os.path.basename(template_path) in ['copyright']:
processed_items.append(item)
continue
# Write the result
with io.open(template_path, 'w', encoding='utf-8') as f:
if sys.version_info.major == 2:
result = result.decode('utf-8')
f.write(result)
# Copy the permissions
shutil.copymode(item, template_path)
processed_items.append(item)
return processed_items
def resolve_rosdep_key(key, os_name, os_version, ros_distro=None, ignored=None, retry=True):
ignored = ignored or []
ctx = create_default_installer_context()
try:
installer_key = ctx.get_default_os_installer_key(os_name)
except KeyError:
BloomGenerator.exit("Could not determine the installer for '{0}'".format(os_name))
installer = ctx.get_installer(installer_key)
ros_distro = ros_distro or DEFAULT_ROS_DISTRO
view = get_view(os_name, os_version, ros_distro)
try:
return resolve_more_for_os(key, view, installer, os_name, os_version)
except (KeyError, ResolutionError) as exc:
debug(traceback.format_exc())
if key in ignored:
return None, None, None
if isinstance(exc, KeyError):
error("Could not resolve rosdep key '{0}'".format(key))
returncode = code.GENERATOR_NO_SUCH_ROSDEP_KEY
else:
error("Could not resolve rosdep key '{0}' for distro '{1}':".format(key, os_version))
info(str(exc), use_prefix=False)
returncode = code.GENERATOR_NO_ROSDEP_KEY_FOR_DISTRO
if retry:
error("Try to resolve the problem with rosdep and then continue.")
if maybe_continue():
update_rosdep()
invalidate_view_cache()
return resolve_rosdep_key(key, os_name, os_version, ros_distro, ignored, retry=True)
BloomGenerator.exit("Failed to resolve rosdep key '{0}', aborting.".format(key), returncode=returncode)
def post_rebase(self, destination):
name = destination.split('/')[-1]
# Retrieve the package
package = self.packages[name]
# Handle differently if this is a debian vs distro branch
if destination in self.debian_branches:
info("Placing debian template files into '{0}' branch.".format(
destination))
# Then this is a debian branch
# Place the raw template files
self.place_template_files()
else:
# This is a distro specific debian branch
# Determine the current package being generated
distro = destination.split('/')[-2]
# Create debians for each distro
with inbranch(destination):
data = self.generate_debian(package, distro)
# Create the tag name for later
self.tag_names[destination] = self.generate_tag_name(data)
# Update the patch configs
patches_branch = 'patches/' + destination
config = get_patch_config(patches_branch)
# Store it
self.store_original_config(config, patches_branch)
# Modify the base so import/export patch works
current_branch = get_current_branch()
if current_branch is None:
error("Could not determine current branch.", exit=True)
config['base'] = get_commit_hash(current_branch)
# Set it
set_patch_config(patches_branch, config)
def post_rebase(self, destination):
name = destination.split('/')[-1]
# Retrieve the package
package = self.packages[name]
# Handle differently if this is an arch vs distro branch
if destination in self.arch_branches:
info("Placing Arch template files into '{0}' branch."
.format(destination))
# Then this is an arch branch
# Place the raw template files
self.place_template_files()
else:
# This is a distro specific arch branch
# Determine the current package being generated
distro = destination.split('/')[-2]
# Create Arch packages for each distro
with inbranch(destination):
# To fit Arch Linux philosophy a bit better, we move all the source files into a subdirectory.
# Arch Linux doesn't support source distribution through a subdirectory; therefore we should ideally compress the sources or provide a URL.
# At this point in the generator, it is tricky to get the release URL. Furthermore it wouldn't fit bloom's patch mechanism very well.
# To work around, we copy the sources to the $srcdir at the beginning inside the prepare() function.
temp_dir = mkdtemp(dir='.')
for item in os.listdir("."):
itemsrc = os.path.abspath(item)
if os.path.basename(itemsrc) in ['.', '..', '.git', '.svn', 'arch',os.path.basename(temp_dir)]:
continue
itemdst = os.path.abspath(os.path.join(temp_dir,item))
execute_command('git mv ' + itemsrc + ' ' + itemdst)
execute_command('git mv ' + temp_dir + ' ' + name)
execute_command('git commit --amend --no-edit')
# Then generate the PKGBUILD
data = self.generate_arch(package, distro)
# And finally move the PKGBUILD to the root directory of the package.
for item in os.listdir("arch"):
itemsrc = os.path.abspath(os.path.join("arch",item))
if os.path.basename(itemsrc) in ['.', '..', '.git', '.svn']:
continue
itemdst = os.path.abspath(item)
execute_command('git mv ' + itemsrc + ' ' + itemdst)
execute_command('git commit --amend --no-edit')
# Create the tag name for later
self.tag_names[destination] = self.generate_tag_name(data)
# Update the patch configs
patches_branch = 'patches/' + destination
config = get_patch_config(patches_branch)
# Store it
self.store_original_config(config, patches_branch)
# Modify the base so import/export patch works
current_branch = get_current_branch()
if current_branch is None:
error("Could not determine current branch.", exit=True)
config['base'] = get_commit_hash(current_branch)
# Set it
set_patch_config(patches_branch, config)
def __process_template_folder(path, subs):
items = os.listdir(path)
processed_items = []
for item in list(items):
item = os.path.abspath(os.path.join(path, item))
if os.path.basename(item) in ['.', '..', '.git', '.svn']:
continue
if os.path.isdir(item):
sub_items = __process_template_folder(item, subs)
processed_items.extend([os.path.join(item, s) for s in sub_items])
if not item.endswith(TEMPLATE_EXTENSION):
continue
with open(item, 'r') as f:
template = f.read()
# Remove extension
template_path = item[:-len(TEMPLATE_EXTENSION)]
# Expand template
info("Expanding '{0}' -> '{1}'".format(
os.path.relpath(item),
os.path.relpath(template_path)))
result = em.expand(template, **subs)
# Write the result
with open(template_path, 'w') as f:
f.write(result.encode('utf8'))
# Copy the permissions
shutil.copymode(item, template_path)
processed_items.append(item)
return processed_items
def pre_modify(self):
info("\nPre-verifying RPM dependency keys...")
# Run rosdep update is needed
if not self.has_run_rosdep:
self.update_rosdep()
peer_packages = [p.name for p in self.packages.values()]
while not self._check_all_keys_are_valid(peer_packages, self.rosdistro):
error("Some of the dependencies for packages in this repository could not be resolved by rosdep.")
error("You can try to address the issues which appear above and try again if you wish, "
"or continue without releasing into RPM-based distributions (e.g. Fedora 24).")
try:
if not maybe_continue(msg="Would you like to try again?"):
error("User aborted after rosdep keys were not resolved.")
sys.exit(code.GENERATOR_NO_ROSDEP_KEY_FOR_DISTRO)
except (KeyboardInterrupt, EOFError):
error("\nUser quit.", exit=True)
update_rosdep()
invalidate_view_cache()
info("All keys are " + ansi('greenf') + "OK" + ansi('reset') + "\n")
for package in self.packages.values():
if not package.licenses or not package.licenses[0]:
error("No license set for package '{0}', aborting.".format(package.name), exit=True)
def pre_modify(self):
info("\nPre-verifying Debian dependency keys...")
# Run rosdep update is needed
if not self.has_run_rosdep:
self.update_rosdep()
peer_packages = [p.name for p in self.packages.values()]
while not self._check_all_keys_are_valid(peer_packages):
error(
"Some of the dependencies for packages in this repository could not be resolved by rosdep."
)
error(
"You can try to address the issues which appear above and try again if you wish."
)
try:
if not maybe_continue(msg="Would you like to try again?"):
error("User aborted after rosdep keys were not resolved.")
sys.exit(code.GENERATOR_NO_ROSDEP_KEY_FOR_DISTRO)
except (KeyboardInterrupt, EOFError):
error("\nUser quit.", exit=True)
update_rosdep()
invalidate_view_cache()
info("All keys are " + ansi('greenf') + "OK" + ansi('reset') + "\n")
def main(sysargs=None):
parser = get_argument_parser()
parser = add_global_arguments(parser)
args = parser.parse_args(sysargs)
handle_global_arguments(args)
# Check that the current directory is a serviceable git/bloom repo
try:
ensure_clean_working_env()
ensure_git_root()
except SystemExit:
parser.print_usage()
raise
git_clone = GitClone()
with git_clone:
import_upstream(
args.archive_path,
args.patches_path,
args.release_version,
args.name,
args.replace)
git_clone.commit()
info("I'm happy. You should be too.")
def process_template_files(path, subs):
info(fmt("@!@{bf}==>@| In place processing templates in 'rpm' folder."))
rpm_dir = os.path.join(path, 'rpm')
if not os.path.exists(rpm_dir):
sys.exit("No rpm directory found at '{0}', cannot process templates."
.format(rpm_dir))
return __process_template_folder(rpm_dir, subs)
def get_repo_uri(repository, distro):
url = None
# Fetch the distro file
distribution_file = get_distribution_file(distro)
if repository in distribution_file.repositories and \
distribution_file.repositories[repository].release_repository is not None:
url = distribution_file.repositories[repository].release_repository.url
else:
error("Specified repository '{0}' is not in the distribution file located at '{1}'"
.format(repository, get_disitrbution_file_url(distro)))
matches = difflib.get_close_matches(repository, distribution_file.repositories)
if matches:
info(fmt("@{yf}Did you mean one of these: '" + "', '".join([m for m in matches]) + "'?"))
if not url:
info("Could not determine release repository url for repository '{0}' of distro '{1}'"
.format(repository, distro))
info("You can continue the release process by manually specifying the location of the RELEASE repository.")
info("To be clear this is the url of the RELEASE repository not the upstream repository.")
try:
url = safe_input('Release repository url [press enter to abort]: ')
except (KeyboardInterrupt, EOFError):
url = None
info('', use_prefix=False)
if not url:
error("No release repository url given, aborting.", exit=True)
global _user_provided_release_url
_user_provided_release_url = url
return url
def pre_modify(self):
info("\nPre-verifying RPM dependency keys...")
# Run rosdep update is needed
if not self.has_run_rosdep:
self.update_rosdep()
peer_packages = [p.name for p in self.packages.values()]
while not self._check_all_keys_are_valid(peer_packages, self.rosdistro):
error("Some of the dependencies for packages in this repository could not be resolved by rosdep.")
if not self.interactive:
sys.exit(code.GENERATOR_NO_ROSDEP_KEY_FOR_DISTRO)
error("You can try to address the issues which appear above and try again if you wish, "
"or continue without releasing into RPM-based distributions (e.g. Fedora 24).")
try:
if not maybe_continue(msg="Would you like to try again?"):
error("User aborted after rosdep keys were not resolved.")
sys.exit(code.GENERATOR_NO_ROSDEP_KEY_FOR_DISTRO)
except (KeyboardInterrupt, EOFError):
error("\nUser quit.", exit=True)
update_rosdep()
invalidate_view_cache()
info("All keys are " + ansi('greenf') + "OK" + ansi('reset') + "\n")
for package in self.packages.values():
if not package.licenses or not package.licenses[0]:
error("No license set for package '{0}', aborting.".format(package.name), exit=True)
def summarize_arguments(upstream_repo, upstream_repo_type,
upstream_repo_branch):
# Summarize the requested operation
summary_msg = "Upstream " + ansi('boldon') + upstream_repo
summary_msg += ansi('boldoff') + " type: " + ansi('boldon')
summary_msg += upstream_repo_type + ansi('boldoff')
info(summary_msg)
def commit(self):
if self.disabled:
return
info(fmt("@{bf}<==@| Command successful, committing changes to working copy"))
current_branch = get_current_branch()
if current_branch is None:
error("Could not determine current branch.", exit=True)
with inbranch(get_commit_hash(get_current_branch())):
with change_directory(self.clone_dir):
new_branches = get_branches()
for branch in self.current_branches:
if branch in new_branches:
new_branches.remove(branch)
for branch in get_branches(local_only=True):
if branch not in new_branches:
with inbranch(branch):
cmd = 'git pull --rebase origin ' + branch
execute_command(cmd)
execute_command('git push --all', silent=False)
try:
execute_command('git push --tags', silent=False)
except subprocess.CalledProcessError:
warning("Force pushing tags from clone to working repository, "
"you will have to force push back to origin...")
execute_command('git push --force --tags', silent=False)
self.clean_up()
def update_rosdep():
info("Running 'rosdep update'...")
try:
rosdep2.catkin_support.update_rosdep()
except:
print_exc(traceback.format_exc())
error("Failed to update rosdep, did you run 'rosdep init' first?", exit=True)
def post_rebase(self, destination):
name = destination.split('/')[-1]
# Retrieve the package
package = self.packages[name]
# Handle differently if this is an rpm vs distro branch
if destination in self.rpm_branches:
info("Placing RPM template files into '{0}' branch."
.format(destination))
# Then this is an rpm branch
# Place the raw template files
self.place_template_files()
else:
# This is a distro specific rpm branch
# Determine the current package being generated
distro = destination.split('/')[-2]
# Create RPMs for each distro
with inbranch(destination):
data = self.generate_rpm(package, distro)
# Create the tag name for later
self.tag_names[destination] = self.generate_tag_name(data)
# Update the patch configs
patches_branch = 'patches/' + destination
config = get_patch_config(patches_branch)
# Store it
self.store_original_config(config, patches_branch)
# Modify the base so import/export patch works
current_branch = get_current_branch()
if current_branch is None:
error("Could not determine current branch.", exit=True)
config['base'] = get_commit_hash(current_branch)
# Set it
set_patch_config(patches_branch, config)
def _my_run(cmd, msg=None):
if msg:
info(fmt("@{bf}@!==> @|@!" + sanitize(msg)))
else:
info(fmt("@{bf}@!==> @|@!" + sanitize(str(cmd))))
from subprocess import check_call
check_call(cmd, shell=True)
def create_from_template(self, template_name, data, directory,
chmod=None, outfile=None):
# Configure template name
extention = '.em'
if not template_name.endswith(extention):
template_file = template_name + extention
else:
template_file = template_name
template_name = template_name[:len(extention)]
template_path = os.path.join('templates', template_file)
# Get the template contents using pkg_resources
group = 'bloom.generators.debian'
# info("Looking for template: " + group + ':' + template_path)
try:
template = pkg_resources.resource_string(group, template_path)
except IOError as err:
error("Failed to load template "
"'{0}': {1}".format(template_name, str(err)))
self.exit(code.DEBIAN_FAILED_TO_LOAD_TEMPLATE)
# Expand template
outfile = outfile if outfile is not None else template_name
info("Expanding template: '" + template_file + "' to '" + \
outfile + "'")
result = em.expand(template, **data)
# Write the template out
with change_directory(directory):
with open(outfile, 'w+') as f:
f.write(result)
# Set permissions if needed
if chmod is not None:
os.chmod(outfile, chmod)
def _my_run(cmd):
info(fmt("@{bf}@!==> @|@!" + str(cmd)))
# out = check_output(cmd, stderr=subprocess.STDOUT, shell=True)
out = None
from subprocess import call
call(cmd, shell=True)
if out:
info(out, use_prefix=False)
def update_rosdep():
info("Running 'rosdep update'...")
try:
rosdep2.catkin_support.update_rosdep()
except:
print_exc(traceback.format_exc())
error("Failed to update rosdep, did you run 'rosdep init' first?",
exit=True)
def _my_run(cmd):
info(fmt("@{bf}@!==> @|@!" + str(cmd)))
# out = check_output(cmd, stderr=subprocess.STDOUT, shell=True)
out = None
from subprocess import call
call(cmd, shell=True)
if out:
info(out, use_prefix=False)
def pre_rebase(self, destination):
name = destination.split('/')[-1]
info("Releasing package '" + name + "' to: '" + destination + "'")
ret = trim(undo=True)
if ret == code.NOTHING_TO_DO:
return 0
else:
return ret
def import_patches(patches_path, patches_path_dict, target_branch, version):
info("Overlaying files from patched folder '{0}' on the '{2}' branch into the '{1}' branch..."
.format(patches_path, target_branch, BLOOM_CONFIG_BRANCH))
with inbranch(target_branch):
handle_tree(patches_path_dict, '', patches_path, version)
cmd = ('git commit --allow-empty -m "Overlaid patches from \'{0}\'"'
.format(patches_path))
execute_command(cmd, shell=True)
def show(args):
tracks_dict = get_tracks_dict_raw()
if args.track not in tracks_dict['tracks']:
error("Track '{0}' does not exist.".format(args.track), exit=True)
info(
yaml.dump({args.track: tracks_dict['tracks'][args.track]},
indent=2,
default_flow_style=False))
def pre_rebase(self, destination, msg=None):
name = destination.split('/')[-1]
msg = msg if msg is not None else (
"Releasing package '" + name + "' to: '" + destination + "'"
)
info(msg)
ret = trim(undo=True)
return 0 if ret < 0 else ret # Ret < 0 indicates nothing was done
def place_template_files(path):
info(fmt("@!@{bf}==>@| Placing templates files in the 'arch' folder."))
arch_path = os.path.join(path, 'arch')
# Create/Clean the arch folder
if not os.path.exists(arch_path):
os.makedirs(arch_path)
# Place template files
group = 'bloom.generators.arch'
__place_template_folder(group, 'templates', arch_path)
def summarize(self):
self.branch_list = self.detect_branches()
if type(self.branch_list) not in [list, tuple]:
self.exit(self.branch_list if self.branch_list is not None else 1)
info(
"Releasing package" + \
('' if len(self.branch_list) == 1 else 's') + ": " + \
str(self.branch_list)
)
def show_current():
if os.path.exists('bloom.conf'):
info("Current bloom configuration:")
f = open('bloom.conf', 'r')
print('')
map(info, [l.rstrip() for l in f.read().splitlines()])
print('')
else:
info("No bloom.conf in the bloom branch")
def get_release_repo(repository, distro):
global _repositories
url = get_repo_uri(repository, distro)
if repository not in _repositories.values():
temp_dir = tempfile.mkdtemp()
_repositories[repository] = get_vcs_client("git", temp_dir)
info(fmt("@{gf}@!==> @|") + "Fetching '{0}' repository from '{1}'".format(repository, url))
_repositories[repository].checkout(url, "master")
return _repositories[repository]
def place_template_files(path, gbp=False):
info(fmt("@!@{bf}==>@| Placing templates files in the 'debian' folder."))
debian_path = os.path.join(path, 'debian')
# Create/Clean the debian folder
if not os.path.exists(debian_path):
os.makedirs(debian_path)
# Place template files
group = 'bloom.generators.debian'
__place_template_folder(group, 'templates', debian_path, gbp)
def place_template_files(path, gbp=False):
info(fmt("@!@{bf}==>@| Placing templates files in the 'rpm' folder."))
rpm_path = os.path.join(path, 'rpm')
# Create/Clean the rpm folder
if not os.path.exists(rpm_path):
os.makedirs(rpm_path)
# Place template files
group = 'bloom.generators.rpm'
__place_template_folder(group, 'templates', rpm_path, gbp)
def copy_track(src, dst):
tracks_dict = get_tracks_dict_raw()
if src not in tracks_dict['tracks']:
error("Track '{0}' does not exist.".format(src), exit=True)
if dst in tracks_dict['tracks']:
error("Track '{0}' already exists.".format(dst), exit=True)
tracks_dict['tracks'][dst] = copy.deepcopy(tracks_dict['tracks'][src])
info("Saving '{0}' track.".format(dst))
write_tracks_dict_raw(tracks_dict)
def copy_track(src, dst):
tracks_dict = get_tracks_dict_raw()
if src not in tracks_dict['tracks']:
error("Track '{0}' does not exist.".format(src), exit=True)
if dst in tracks_dict['tracks']:
error("Track '{0}' already exists.".format(dst), exit=True)
tracks_dict['tracks'][dst] = copy.deepcopy(tracks_dict['tracks'][src])
info("Saving '{0}' track.".format(dst))
write_tracks_dict_raw(tracks_dict)
def main(args=None, get_subs_fn=None):
get_subs_fn = get_subs_fn or get_subs
_place_template_files = True
_process_template_files = True
package_path = os.getcwd()
if args is not None:
package_path = args.package_path or os.getcwd()
_place_template_files = args.place_template_files
_process_template_files = args.process_template_files
pkgs_dict = find_packages(package_path)
if len(pkgs_dict) == 0:
sys.exit("No packages found in path: '{0}'".format(package_path))
if len(pkgs_dict) > 1:
sys.exit("Multiple packages found, " "this tool only supports one package at a time.")
os_data = create_default_installer_context().get_os_name_and_version()
os_name, os_version = os_data
ros_distro = os.environ.get("ROS_DISTRO", "indigo")
# Allow args overrides
os_name = args.os_name or os_name
os_version = args.os_version or os_version
ros_distro = args.ros_distro or ros_distro
# Summarize
info(
fmt("@!@{gf}==> @|")
+ fmt(
"Generating debs for @{cf}%s:%s@| for package(s) %s"
% (os_name, os_version, [p.name for p in pkgs_dict.values()])
)
)
for path, pkg in pkgs_dict.items():
template_files = None
try:
subs = get_subs_fn(pkg, os_name, os_version, ros_distro)
if _place_template_files:
# Place template files
place_template_files(path)
if _process_template_files:
# Just process existing template files
template_files = process_template_files(path, subs)
if not _place_template_files and not _process_template_files:
# If neither, do both
place_template_files(path)
template_files = process_template_files(path, subs)
if template_files is not None:
for template_file in template_files:
os.remove(os.path.normpath(template_file))
except Exception as exc:
debug(traceback.format_exc())
error(type(exc).__name__ + ": " + str(exc), exit=True)
except (KeyboardInterrupt, EOFError):
sys.exit(1)
def place_template_files(path, build_type, gbp=False):
info(fmt("@!@{bf}==>@| Placing templates files in the 'rpm' folder."))
rpm_path = os.path.join(path, 'rpm')
# Create/Clean the rpm folder
if not os.path.exists(rpm_path):
os.makedirs(rpm_path)
# Place template files
group = 'bloom.generators.rpm'
templates = os.path.join('templates', build_type)
__place_template_folder(group, templates, rpm_path, gbp)
def build_debian_pkg(args=None, get_subs_fn=None):
get_subs_fn = get_subs_fn or get_subs
_place_template_files = True
_process_template_files = True
package_path = os.getcwd()
if args is not None:
package_path = args.package_path or os.getcwd()
_place_template_files = args.place_template_files
_process_template_files = args.process_template_files
pkgs_dict = find_packages(package_path)
if len(pkgs_dict) == 0:
sys.exit("No packages found in path: '{0}'".format(package_path))
# if len(pkgs_dict) > 1:
# sys.exit("Multiple packages found, "
# "this tool only supports one package at a time.")
os_data = create_default_installer_context().get_os_name_and_version()
os_name, os_version = os_data
ros_distro = os.environ.get('ROS_DISTRO', 'indigo')
# Allow args overrides
os_name = args.os_name or os_name
os_version = args.os_version or os_version
ros_distro = args.ros_distro or ros_distro
install_prefix = args.install_prefix or "/opt"
# Summarize
info(fmt("@!@{gf}==> @|") +
fmt("Generating debs for @{cf}%s:%s@| for package(s) %s" %
(os_name, os_version, [p.name for p in pkgs_dict.values()])))
# Test Creating single
all_subs = merge_packages(pkgs_dict, get_subs_fn, os_name, os_version, ros_distro, install_prefix, args.native)
path = ''
build_type = 'cmake'
try:
if _place_template_files:
# Place template files
place_template_files(path, build_type)
if _process_template_files:
# Just process existing template files
template_files = process_template_files(path, all_subs)
if not _place_template_files and not _process_template_files:
# If neither, do both
place_template_files(path, build_type)
template_files = process_template_files(path, all_subs)
if template_files is not None:
for template_file in template_files:
os.remove(os.path.normpath(template_file))
except Exception as exc:
debug(traceback.format_exc())
error(type(exc).__name__ + ": " + str(exc), exit=True)
except (KeyboardInterrupt, EOFError):
sys.exit(1)
def get_release_repo(repository, distro):
global _repositories
url = get_repo_uri(repository, distro)
if repository not in _repositories.values():
temp_dir = tempfile.mkdtemp()
_repositories[repository] = get_vcs_client('git', temp_dir)
info(
fmt("@{gf}@!==> @|") +
"Fetching '{0}' repository from '{1}'".format(repository, url))
_repositories[repository].checkout(url, 'master')
return _repositories[repository]
def place_template_files(path, build_type, gbp=False):
info(fmt("@!@{bf}==>@| Placing templates files in the 'debian' folder."))
debian_path = os.path.join(path, 'debian')
# Remove the debian folder if it exist
if os.path.exists(debian_path):
shutil.rmtree(debian_path)
# Place template files
templates = os.path.join(os.curdir, os.path.join('templates', build_type))
shutil.copytree(os.path.join('templates', build_type), debian_path)
if not gbp:
os.remove(os.path.join(debian_path, 'gbp.conf.em'))
def upconvert_bloom_to_config_branch():
global _has_checked_bloom_branch
if _has_checked_bloom_branch:
return
# Assert that this repository does not have multiple remotes
check_for_multiple_remotes()
if get_root() is None:
# Not a git repository
return
track_branches(['bloom', BLOOM_CONFIG_BRANCH])
if show('bloom', PLACEHOLDER_FILE) is not None:
return
if show('bloom', 'bloom.conf') is not None:
# Wait for the bloom.conf upconvert...
return
if not branch_exists('bloom'):
return
_has_checked_bloom_branch = True
info("Moving configurations from deprecated 'bloom' branch "
"to the '{0}' branch.".format(BLOOM_CONFIG_BRANCH))
tmp_dir = mkdtemp()
git_root = get_root()
try:
# Copy the new upstream source into the temporary directory
with inbranch('bloom'):
ignores = ('.git', '.gitignore', '.svn', '.hgignore', '.hg', 'CVS')
configs = os.path.join(tmp_dir, 'configs')
my_copytree(git_root, configs, ignores)
if [x for x in os.listdir(os.getcwd()) if x not in ignores]:
execute_command('git rm -rf ./*')
with open(PLACEHOLDER_FILE, 'w') as f:
f.write("""\
This branch ('bloom') has been deprecated in favor of storing settings and overlay files in the master branch.
Please goto the master branch for anything which referenced the bloom branch.
You can delete this branch at your convenience.
""")
execute_command('git add ' + PLACEHOLDER_FILE)
if has_changes():
execute_command('git commit -m "DEPRECATING BRANCH"')
if not branch_exists(BLOOM_CONFIG_BRANCH):
info("Creating '{0}' branch.".format(BLOOM_CONFIG_BRANCH))
create_branch(BLOOM_CONFIG_BRANCH, orphaned=True)
with inbranch(BLOOM_CONFIG_BRANCH):
my_copytree(configs, git_root)
execute_command('git add ./*')
if has_changes():
execute_command('git commit -m '
'"Moving configs from bloom branch"')
finally:
# Clean up
if os.path.exists(tmp_dir):
shutil.rmtree(tmp_dir)
def print_exc(exc):
exc_str = ''.join(exc)
try:
from pygments import highlight
from pygments.lexers import PythonTracebackLexer
from pygments.formatters import TerminalFormatter
exc_str = highlight(exc_str, PythonTracebackLexer(),
TerminalFormatter())
except ImportError:
pass
info(exc_str, file=sys.stderr, use_prefix=False)
