def get_index_url():
global _rosdistro_index_commit
index_url = rosdistro.get_index_url()
pr = urlparse(index_url)
if pr.netloc == 'raw.github.com':
# Try to determine what the commit hash was
tokens = [x for x in pr.path.split('/') if x]
if len(tokens) <= 3:
debug("Failed to get commit for rosdistro index file: index url")
debug(tokens)
return index_url
owner = tokens[0]
repo = tokens[1]
branch = tokens[2]
gh = get_github_interface(quiet=True)
if gh is None:
# Failed to get it with auth, try without auth (may fail)
gh = Github(username=None, auth=None)
try:
data = gh.get_branch(owner, repo, branch)
except GithubException:
debug(traceback.format_exc())
debug("Failed to get commit for rosdistro index file: api")
return index_url
_rosdistro_index_commit = data.get('commit', {}).get('sha', None)
if _rosdistro_index_commit is not None:
info("ROS Distro index file associate with commit '{0}'"
.format(_rosdistro_index_commit))
else:
debug("Failed to get commit for rosdistro index file: json")
return index_url
def merge_packages(pkgs_dict,
get_subs_fn,
os_name,
os_version,
ros_distro,
install_prefix,
native=False):
all_subs = {}
for path, pkg in pkgs_dict.items():
try:
subs = get_subs_fn(pkg, os_name, os_version, ros_distro,
install_prefix, native)
all_subs[subs['Name']] = subs
except Exception as exc:
debug(traceback.format_exc())
error(type(exc).__name__ + ": " + str(exc), exit=True)
except (KeyboardInterrupt, EOFError):
sys.exit(1)
repo_header = {}
cnt = 0
for pkg, sub in all_subs.items():
try:
if (0 == cnt):
repo_header['Package'] = convertToUnicode(
sanitize_package_name('tesseract_core'))
repo_header['DebianInc'] = sub['DebianInc']
repo_header['format'] = sub['format']
repo_header['InstallationPrefix'] = sub['InstallationPrefix']
repo_header['Maintainer'] = sub['Maintainers'][0]
repo_header['Maintainers'] = sub['Maintainers']
repo_header['BuildDepends'] = sub['BuildDepends']
repo_header['Homepage'] = sub['Homepage']
repo_header['Copyright'] = sub['Copyright']
repo_header['debhelper_version'] = sub['debhelper_version']
repo_header['changelogs'] = sub['changelogs']
repo_header['Distribution'] = sub['Distribution']
else:
repo_header['Maintainers'].join(', '.join(sub['Maintainers']))
repo_header['BuildDepends'].extend(sub['BuildDepends'])
repo_header['Copyright'].join(sub['Copyright'])
cnt = cnt + 1
except Exception as exc:
debug(traceback.format_exc())
error(type(exc).__name__ + ": " + str(exc), exit=True)
except (KeyboardInterrupt, EOFError):
sys.exit(1)
# Remove build depends in this repository
repo_header['BuildDepends'] = [
x for x in repo_header['BuildDepends'] if x not in all_subs.keys()
]
# Remove duplicates
repo_header['BuildDepends'] = list(
dict.fromkeys(repo_header['BuildDepends']))
# TODO Remove Duplicates from repo_header['Maintainers']
all_subs[repo_header['Package']] = repo_header
return all_subs
def set_upstream(upstream_repo, upstream_repo_type, upstream_repo_branch):
# Check for a bloom branch
if branch_exists('bloom', False):
# Found a bloom branch
debug("Found a bloom branch, checking out.")
# Check out the bloom branch
checkout('bloom')
else:
# No bloom branch found, create one
create_branch('bloom', changeto=True)
# Now set the upstream using the bloom config
cmd = 'git config -f bloom.conf bloom.upstream "{0}"'.format(upstream_repo)
execute_command(cmd)
cmd = 'git config -f bloom.conf ' \
+ 'bloom.upstreamtype "{0}"'.format(upstream_repo_type)
execute_command(cmd)
cmd = 'git config -f bloom.conf ' \
+ 'bloom.upstreambranch "{0}"'.format(upstream_repo_branch)
execute_command(cmd)
execute_command('git add bloom.conf')
if has_changes():
cmd = 'git commit -m "bloom branch update by git-bloom-config"'
execute_command(cmd)
else:
debug("No chages, nothing to commit.")
def remove_patches(directory=None):
# Get the current branch
current_branch = get_current_branch(directory)
# Ensure the current branch is valid
if current_branch is None:
error("Could not determine current branch, are you in a git repo?")
return 1
# Construct the patches branch
patches_branch = 'patches/' + current_branch
try:
# See if the patches branch exists
if branch_exists(patches_branch, False, directory=directory):
if not branch_exists(patches_branch, True, directory=directory):
track_branches(patches_branch, directory)
else:
error("No patches branch (" + patches_branch + ") found, cannot "
"remove patches.")
return 1
# Get the parent branch from the patches branch
config = get_patch_config(patches_branch, directory=directory)
parent, spec = config['parent'], config['base']
if None in [parent, spec]:
error("Could not retrieve patches info.")
return 1
debug("Removing patches from " + current_branch + " back to base "
"commit " + spec)
# Reset this branch using git reset --hard spec
execute_command('git reset --hard ' + spec, cwd=directory)
finally:
if current_branch:
checkout(current_branch, directory=directory)
return 0
def _set_trim_sub_dir(sub_dir, force, config, directory):
debug("_set_trim_sub_dir(" + str(sub_dir) + ", " + str(force) + ", " + \
str(config) + ", " + str(directory) + ")")
if sub_dir is not None:
if config['trim'] != '' and config['trim'] != sub_dir:
warning("You are trying to set the trim sub directory to " + \
sub_dir + ", but it is already set to " + \
config['trim'] + ".")
if not force:
warning("Changing the sud directory is not advised. "
"If you are sure you want to do this, use "
"'--force'")
return None
else:
warning("Forcing the change of the sub directory.")
# Make the sub_dir absolute
git_root = get_root(directory)
sub_dir_abs = os.path.join(git_root, sub_dir)
# Make sure it is a directory
if not os.path.isdir(sub_dir_abs):
error("The given sub directory, (" + sub_dir + ") does not "
"exist in the git repository at " + git_root)
return None
# Set the trim sub directory
config['trim'] = sub_dir
return config
def run_generator(generator, arguments):
try:
gen = generator
try_execute('generator handle arguments', '', gen.handle_arguments,
arguments)
try_execute('generator summarize', '', gen.summarize)
if arguments.interactive:
if not maybe_continue('y'):
error("Answered no to continue, aborting.", exit=True)
for branch_args in generator.get_branching_arguments():
parsed_branch_args = parse_branch_args(branch_args,
arguments.interactive)
destination, source, interactive = parsed_branch_args
# Summarize branch command
msg = summarize_branch_cmd(destination, source, interactive)
### Run pre - branch - post
# Pre branch
try_execute('generator pre_branch', msg, gen.pre_branch,
destination, source)
# Branch
try_execute('git-bloom-branch', msg, execute_branch, source,
destination, interactive)
# Post branch
try_execute('generator post_branch', msg, gen.post_branch,
destination, source)
### Run pre - export patches - post
# Pre patch
try_execute('generator pre_export_patches', msg,
gen.pre_export_patches, destination)
# Export patches
try_execute('git-bloom-patch export', msg, export_patches)
# Post branch
try_execute('generator post_export_patches', msg,
gen.post_export_patches, destination)
### Run pre - rebase - post
# Pre rebase
try_execute('generator pre_rebase', msg, gen.pre_rebase,
destination)
# Rebase
ret = try_execute('git-bloom-patch rebase', msg, rebase_patches)
# Post rebase
try_execute('generator post_rebase', msg, gen.post_rebase,
destination)
### Run pre - import patches - post
# Pre patch
try_execute('generator pre_patch', msg, gen.pre_patch, destination)
if ret == 0:
# Import patches
try_execute('git-bloom-patch import', msg, import_patches)
elif ret < 0:
debug("Skipping patching because rebase did not run.")
# Post branch
try_execute('generator post_patch', msg, gen.post_patch,
destination)
except CommandFailed as err:
sys.exit(err.returncode or 1)
def execute_command(cmd, shell=True, autofail=True, silent=True,
silent_error=False, cwd=None, return_io=False):
"""
Executes a given command using vcstools' run_shell_command function.
"""
io_type = None
result = 0
if silent:
io_type = PIPE
debug(((cwd) if cwd else os.getcwd()) + ":$ " + str(cmd))
p = Popen(cmd, shell=True, cwd=cwd, stdout=io_type, stderr=io_type)
out, err = p.communicate()
result = p.returncode
if result != 0:
if not silent_error:
error("'execute_command' failed to call '{0}'".format(cmd) + \
" which had a return code ({0}):".format(result))
if out:
error(" stdout:\n" + ansi('reset') + str(out))
error("end stdout")
if err:
error(" stderr:\n" + ansi('reset') + str(err))
error("end stderr")
if autofail:
raise CalledProcessError(cmd=cmd, output=out, returncode=result)
if return_io:
return result, out, err
else:
return result
def execute_command(cmd, shell=True, autofail=True, silent=True,
silent_error=False, cwd=None, return_io=False):
"""
Executes a given command using vcstools' run_shell_command function.
"""
out_io = None
err_io = None
result = 0
if silent:
out_io = PIPE
err_io = STDOUT
debug(((cwd) if cwd else os.getcwd()) + ":$ " + str(cmd))
p = Popen(cmd, shell=True, cwd=cwd, stdout=out_io, stderr=err_io)
out, err = p.communicate()
if out is not None and not isinstance(out, str):
out = out.decode('utf-8')
if err is not None and not isinstance(err, str):
err = err.decode('utf-8')
result = p.returncode
if result != 0:
if not silent_error:
error("'execute_command' failed to call '{0}'".format(cmd) +
" which had a return code ({0}):".format(result))
error("```")
info(out, use_prefix=False)
error("```")
if autofail:
raise CalledProcessError(result, cmd)
if return_io:
return result, out, err
else:
return result
def execute_command(cmd, shell=True, autofail=True, silent=True,
silent_error=False, cwd=None, return_io=False):
"""
Executes a given command using Popen.
"""
out_io = None
err_io = None
result = 0
if silent:
out_io = PIPE
err_io = STDOUT
debug(((cwd) if cwd else os.getcwd()) + ":$ " + str(cmd))
env = __get_env_for_cmd(cmd)
p = Popen(cmd, shell=shell, cwd=cwd, stdout=out_io, stderr=err_io, env=env)
out, err = p.communicate()
if out is not None and not isinstance(out, str):
out = out.decode('utf-8')
if err is not None and not isinstance(err, str):
err = err.decode('utf-8')
result = p.returncode
if result != 0:
if not silent_error:
error("'execute_command' failed to call '{0}'".format(cmd) +
" which had a return code ({0}):".format(result))
error("```")
info(out, use_prefix=False)
error("```")
if autofail:
raise CalledProcessError(result, cmd)
if return_io:
return result, out, err
else:
return result
def resolve_rosdep_key(key, os_name, os_version, ros_distro=None, ignored=None, retry=True):
ignored = ignored or []
ctx = create_default_installer_context()
try:
installer_key = ctx.get_default_os_installer_key(os_name)
except KeyError:
BloomGenerator.exit("Could not determine the installer for '{0}'".format(os_name))
installer = ctx.get_installer(installer_key)
ros_distro = ros_distro or DEFAULT_ROS_DISTRO
view = get_view(os_name, os_version, ros_distro)
try:
return resolve_more_for_os(key, view, installer, os_name, os_version)
except (KeyError, ResolutionError) as exc:
debug(traceback.format_exc())
if key in ignored:
return None, None, None
if isinstance(exc, KeyError):
error("Could not resolve rosdep key '{0}'".format(key))
returncode = code.GENERATOR_NO_SUCH_ROSDEP_KEY
else:
error("Could not resolve rosdep key '{0}' for distro '{1}':".format(key, os_version))
info(str(exc), use_prefix=False)
returncode = code.GENERATOR_NO_ROSDEP_KEY_FOR_DISTRO
if retry:
error("Try to resolve the problem with rosdep and then continue.")
if maybe_continue():
update_rosdep()
invalidate_view_cache()
return resolve_rosdep_key(key, os_name, os_version, ros_distro, ignored, retry=True)
BloomGenerator.exit("Failed to resolve rosdep key '{0}', aborting.".format(key), returncode=returncode)
def execute_command(cmd, shell=True, autofail=True, silent=True,
silent_error=False, cwd=None, return_io=False):
"""
Executes a given command using vcstools' run_shell_command function.
"""
out_io = None
err_io = None
result = 0
if silent:
out_io = PIPE
err_io = STDOUT
debug(((cwd) if cwd else os.getcwd()) + ":$ " + str(cmd))
p = Popen(cmd, shell=True, cwd=cwd, stdout=out_io, stderr=err_io)
out, err = p.communicate()
result = p.returncode
if result != 0:
if not silent_error:
error("'execute_command' failed to call '{0}'".format(cmd) +
" which had a return code ({0}):".format(result))
error("```")
info(out, use_prefix=False)
error("```")
if autofail:
raise CalledProcessError(result, cmd)
if return_io:
return result, out, err
else:
return result
def generate_ros_distro_diff(track, repository, distro, distro_file_url, distro_file, distro_file_raw):
with inbranch('upstream'):
# Check for package.xml(s)
try:
from catkin_pkg.packages import find_packages
except ImportError:
debug(traceback.format_exc())
error("catkin_pkg was not detected, please install it.",
file=sys.stderr, exit=True)
packages = find_packages(os.getcwd())
if len(packages) == 0:
warning("No packages found, will not generate 'package: path' entries for rosdistro.")
track_dict = get_tracks_dict_raw()['tracks'][track]
last_version = track_dict['last_version']
release_inc = track_dict['release_inc']
if repository not in distro_file['repositories']:
global _user_provided_release_url
distro_file['repositories'][repository] = {'url': _user_provided_release_url or ''}
distro_file['repositories'][repository]['version'] = '{0}-{1}'.format(last_version, release_inc)
if packages and (len(packages) > 1 or packages.keys()[0] != '.'):
distro_file['repositories'][repository]['packages'] = {}
for path, package in packages.iteritems():
if os.path.basename(path) == package.name:
distro_file['repositories'][repository]['packages'][package.name] = None
else:
distro_file['repositories'][repository]['packages'][package.name] = path
distro_file_name = os.path.join('release', distro_file_url.split('/')[-1])
distro_dump = yaml.dump(distro_file, indent=2, default_flow_style=False)
if distro_file_raw != distro_dump:
udiff = difflib.unified_diff(distro_file_raw.splitlines(), distro_dump.splitlines(),
fromfile=distro_file_name, tofile=distro_file_name)
temp_dir = tempfile.mkdtemp()
version = distro_file['repositories'][repository]['version']
udiff_file = os.path.join(temp_dir, repository + '-' + version + '.patch')
udiff_raw = ''
info("Unified diff for the ROS distro file located at '{0}':".format(udiff_file))
for line in udiff:
if line.startswith('@@'):
udiff_raw += line
line = fmt('@{cf}' + line)
if line.startswith('+'):
if not line.startswith('+++'):
line += '\n'
udiff_raw += line
line = fmt('@{gf}' + line)
if line.startswith('-'):
if not line.startswith('---'):
line += '\n'
udiff_raw += line
line = fmt('@{rf}' + line)
if line.startswith(' '):
line += '\n'
udiff_raw += line
info(line, use_prefix=False, end='')
with open(udiff_file, 'w+') as f:
f.write(udiff_raw)
return udiff_file, distro_dump
else:
warning("This release resulted in no changes to the ROS distro file...")
return None, None
def _set_trim_sub_dir(sub_dir, force, config, directory):
debug("_set_trim_sub_dir(" + str(sub_dir) + ", " + str(force) + ", " +
str(config) + ", " + str(directory) + ")")
if sub_dir is not None:
if config['trim'] != '' and config['trim'] != sub_dir:
warning("You are trying to set the trim sub directory to " +
sub_dir + ", but it is already set to " + config['trim'] +
".")
if not force:
warning("Changing the sud directory is not advised. "
"If you are sure you want to do this, use "
"'--force'")
return None
else:
warning("Forcing the change of the sub directory.")
# Make the sub_dir absolute
git_root = get_root(directory)
sub_dir_abs = os.path.join(git_root, sub_dir)
# Make sure it is a directory
if not os.path.isdir(sub_dir_abs):
error("The given sub directory, (" + sub_dir + ") does not "
"exist in the git repository at " + git_root)
return None
# Set the trim sub directory
config['trim'] = sub_dir
return config
def main(args=None, get_subs_fn=None):
get_subs_fn = get_subs_fn or get_subs
_place_template_files = True
_process_template_files = True
package_path = os.getcwd()
if args is not None:
package_path = args.package_path or os.getcwd()
_place_template_files = args.place_template_files
_process_template_files = args.process_template_files
pkgs_dict = find_packages(package_path)
if len(pkgs_dict) == 0:
sys.exit("No packages found in path: '{0}'".format(package_path))
if len(pkgs_dict) > 1:
sys.exit("Multiple packages found, " "this tool only supports one package at a time.")
os_data = create_default_installer_context().get_os_name_and_version()
os_name, os_version = os_data
ros_distro = os.environ.get("ROS_DISTRO", "indigo")
# Allow args overrides
os_name = args.os_name or os_name
os_version = args.os_version or os_version
ros_distro = args.ros_distro or ros_distro
# Summarize
info(
fmt("@!@{gf}==> @|")
+ fmt(
"Generating debs for @{cf}%s:%s@| for package(s) %s"
% (os_name, os_version, [p.name for p in pkgs_dict.values()])
)
)
for path, pkg in pkgs_dict.items():
template_files = None
try:
subs = get_subs_fn(pkg, os_name, os_version, ros_distro)
if _place_template_files:
# Place template files
place_template_files(path)
if _process_template_files:
# Just process existing template files
template_files = process_template_files(path, subs)
if not _place_template_files and not _process_template_files:
# If neither, do both
place_template_files(path)
template_files = process_template_files(path, subs)
if template_files is not None:
for template_file in template_files:
os.remove(os.path.normpath(template_file))
except Exception as exc:
debug(traceback.format_exc())
error(type(exc).__name__ + ": " + str(exc), exit=True)
except (KeyboardInterrupt, EOFError):
sys.exit(1)
def checkout(reference, raise_exc=False, directory=None, show_git_status=True):
"""
Returns True if the checkout to a the reference was successful, else False
:param reference: branch, tag, or commit hash to checkout to
:param directory: directory in which to run this command
:returns: True if the checkout was successful, else False
"""
def checkout_summarize(fail_msg, branch, directory):
branch = "(no branch)" if branch is None else branch
directory = os.getcwd() if directory is None else directory
error(
"Failed to checkout to '{0}'".format(str(reference))
+ " because the working directory {0}".format(str(fail_msg))
)
debug(" Working directory: '{0}'".format(str(directory)))
debug(" Working branch: '{0}'".format(str(branch)))
debug(" Has local changes: '{0}'".format(str(changes)))
debug(" Has untrakced files: '{0}'".format(str(untracked)))
pdb_hook()
if not bloom.util._quiet and show_git_status:
print("\n++ git status:\n")
os.system("git status")
return 1
debug("Checking out to " + str(reference))
if reference == get_current_branch(directory):
debug("Requested checkout reference is the same as the current branch")
return 0
fail_msg = ""
git_root = get_root(directory)
if git_root is not None:
changes = has_changes(directory)
untracked = has_untracked_files(directory)
branch = get_current_branch(directory)
else:
fail_msg = "is not a git repository"
if fail_msg == "" and changes:
fail_msg = "has local changes"
if fail_msg == "" and untracked:
fail_msg = "has untracked files"
try:
if not changes and not untracked:
execute_command('git checkout "{0}"'.format(str(reference)), cwd=directory)
except CalledProcessError as err:
fail_msg = "CalledProcessError: " + str(err)
if raise_exc:
checkout_summarize(fail_msg, branch, directory)
raise
if fail_msg != "":
return checkout_summarize(fail_msg, branch, directory)
else:
return 0
def set_releaser_history(self, history):
# Assumes that this is called in the target branch
patches_branch = 'patches/' + get_current_branch()
debug("Writing release history to '{0}' branch".format(patches_branch))
with inbranch(patches_branch):
with open('releaser_history.json', 'w') as f:
f.write(json.dumps(history))
execute_command('git add releaser_history.json')
if has_changes():
execute_command('git commit -m "Store releaser history"')
def set_releaser_history(self, history):
# Assumes that this is called in the target branch
patches_branch = 'patches/' + get_current_branch()
debug("Writing release history to '{0}' branch".format(patches_branch))
with inbranch(patches_branch):
with open('releaser_history.json', 'w') as f:
f.write(json.dumps(history))
execute_command('git add releaser_history.json')
if has_changes():
execute_command('git commit -m "Store releaser history"')
def build_debian_pkg(args=None, get_subs_fn=None):
get_subs_fn = get_subs_fn or get_subs
_place_template_files = True
_process_template_files = True
package_path = os.getcwd()
if args is not None:
package_path = args.package_path or os.getcwd()
_place_template_files = args.place_template_files
_process_template_files = args.process_template_files
pkgs_dict = find_packages(package_path)
if len(pkgs_dict) == 0:
sys.exit("No packages found in path: '{0}'".format(package_path))
# if len(pkgs_dict) > 1:
# sys.exit("Multiple packages found, "
# "this tool only supports one package at a time.")
os_data = create_default_installer_context().get_os_name_and_version()
os_name, os_version = os_data
ros_distro = os.environ.get('ROS_DISTRO', 'indigo')
# Allow args overrides
os_name = args.os_name or os_name
os_version = args.os_version or os_version
ros_distro = args.ros_distro or ros_distro
install_prefix = args.install_prefix or "/opt"
# Summarize
info(fmt("@!@{gf}==> @|") +
fmt("Generating debs for @{cf}%s:%s@| for package(s) %s" %
(os_name, os_version, [p.name for p in pkgs_dict.values()])))
# Test Creating single
all_subs = merge_packages(pkgs_dict, get_subs_fn, os_name, os_version, ros_distro, install_prefix, args.native)
path = ''
build_type = 'cmake'
try:
if _place_template_files:
# Place template files
place_template_files(path, build_type)
if _process_template_files:
# Just process existing template files
template_files = process_template_files(path, all_subs)
if not _place_template_files and not _process_template_files:
# If neither, do both
place_template_files(path, build_type)
template_files = process_template_files(path, all_subs)
if template_files is not None:
for template_file in template_files:
os.remove(os.path.normpath(template_file))
except Exception as exc:
debug(traceback.format_exc())
error(type(exc).__name__ + ": " + str(exc), exit=True)
except (KeyboardInterrupt, EOFError):
sys.exit(1)
def checkout(reference, raise_exc=False, directory=None, show_git_status=True):
"""
Returns True if the checkout to a the reference was successful, else False
:param reference: branch, tag, or commit hash to checkout to
:param directory: directory in which to run this command
:returns: True if the checkout was successful, else False
"""
def checkout_summarize(fail_msg, branch, directory):
branch = '(no branch)' if branch is None else branch
directory = os.getcwd() if directory is None else directory
error("Failed to checkout to '{0}'".format(str(reference)) +
" because the working directory {0}".format(str(fail_msg)))
debug(" Working directory: '{0}'".format(str(directory)))
debug(" Working branch: '{0}'".format(str(branch)))
debug(" Has local changes: '{0}'".format(str(changes)))
debug(" Has untrakced files: '{0}'".format(str(untracked)))
pdb_hook()
if not bloom.util._quiet and show_git_status:
info('\n++ git status:\n', use_prefix=False)
os.system('git status')
return False
debug("Checking out to " + str(reference))
if reference == get_current_branch(directory):
debug("Requested checkout reference is the same as the current branch")
return True
fail_msg = ''
git_root = get_root(directory)
if git_root is not None:
changes = has_changes(directory)
untracked = has_untracked_files(directory)
branch = get_current_branch(directory) or 'could not determine branch'
else:
fail_msg = "is not a git repository"
if fail_msg == '' and changes:
fail_msg = "has local changes"
if fail_msg == '' and untracked:
fail_msg = "has untracked files"
try:
if not changes and not untracked:
execute_command('git checkout "{0}"'.format(str(reference)),
cwd=directory)
except CalledProcessError as err:
fail_msg = "CalledProcessError: " + str(err)
if raise_exc:
checkout_summarize(fail_msg, branch, directory)
raise
if fail_msg != '':
return checkout_summarize(fail_msg, branch, directory)
else:
return True
def match_branches_with_prefix(prefix, get_branches):
debug("match_branches_with_prefix(" + str(prefix) + ", " +
str(get_branches()) + ")")
branches = []
# Match branches
existing_branches = get_branches()
for branch in existing_branches:
if branch.startswith('remotes/origin/'):
branch = branch.split('/', 2)[-1]
if branch.startswith(prefix):
branches.append(branch)
return list(set(branches))
def match_branches_with_prefix(prefix, get_branches):
debug("match_branches_with_prefix(" + str(prefix) + ", " +
str(get_branches()) + ")")
branches = []
# Match branches
existing_branches = get_branches()
for branch in existing_branches:
if branch.startswith('remotes/origin/'):
branch = branch.split('/', 2)[-1]
if branch.startswith(prefix):
branches.append(branch)
return list(set(branches))
def _undo(config, directory):
debug("_undo(" + str(config) + ", " + str(directory) + ")")
# TODO: handle repo with changes
# TODO: handle repo with patches applied
if config['trimbase'] == '':
debug("Branch has not been trimmed previously, undo not required.")
return None
# Reset with git-reset
execute_command('git reset --hard ' + config['trimbase'], cwd=directory)
# Unset the trimbase
config['trimbase'] = ''
return config
def try_execute(msg, err_msg, func, *args, **kwargs):
try:
retcode = func(*args, **kwargs)
retcode = retcode if retcode is not None else 0
except CalledProcessError as err:
print_exc(traceback.format_exc())
error("Error calling {0}: ".format(msg) + str(err))
retcode = err.returncode
ret_msg = msg + " returned exit code ({0})".format(str(retcode))
if retcode > 0:
error(ret_msg)
raise CommandFailed(retcode)
elif retcode < 0:
debug(ret_msg)
def __place_template_folder(group, src, dst, gbp=False):
template_files = pkg_resources.resource_listdir(group, src)
# For each template, place
for template_file in template_files:
if not gbp and os.path.basename(template_file) == 'gbp.conf.em':
debug("Skipping template '{0}'".format(template_file))
continue
template_path = os.path.join(src, template_file)
template_dst = os.path.join(dst, template_file)
if pkg_resources.resource_isdir(group, template_path):
debug("Recursing on folder '{0}'".format(template_path))
__place_template_folder(group, template_path, template_dst, gbp)
else:
try:
debug("Placing template '{0}'".format(template_path))
template = pkg_resources.resource_string(group, template_path)
template_abs_path = pkg_resources.resource_filename(group, template_path)
except IOError as err:
error("Failed to load template "
"'{0}': {1}".format(template_file, str(err)), exit=True)
if not os.path.exists(dst):
os.makedirs(dst)
if os.path.exists(template_dst):
debug("Removing existing file '{0}'".format(template_dst))
os.remove(template_dst)
with open(template_dst, 'w') as f:
if not isinstance(template, str):
template = template.decode('utf-8')
f.write(template)
shutil.copystat(template_abs_path, template_dst)
def __place_template_folder(group, src, dst, gbp=False):
template_files = pkg_resources.resource_listdir(group, src)
# For each template, place
for template_file in template_files:
if not gbp and os.path.basename(template_file) == 'gbp.conf.em':
debug("Skipping template '{0}'".format(template_file))
continue
template_path = os.path.join(src, template_file)
template_dst = os.path.join(dst, template_file)
if pkg_resources.resource_isdir(group, template_path):
debug("Recursing on folder '{0}'".format(template_path))
__place_template_folder(group, template_path, template_dst, gbp)
else:
try:
debug("Placing template '{0}'".format(template_path))
template = pkg_resources.resource_string(group, template_path)
template_abs_path = pkg_resources.resource_filename(
group, template_path)
except IOError as err:
error("Failed to load template "
"'{0}': {1}".format(template_file, str(err)),
exit=True)
if not os.path.exists(dst):
os.makedirs(dst)
if os.path.exists(template_dst):
debug("Removing existing file '{0}'".format(template_dst))
os.remove(template_dst)
with open(template_dst, 'w') as f:
if not isinstance(template, str):
template = template.decode('utf-8')
f.write(template)
shutil.copystat(template_abs_path, template_dst)
def _undo(config, directory):
debug("_undo(" + str(config) + ", " + str(directory) + ")")
# TODO: handle repo with changes
# TODO: handle repo with patches applied
if config['trimbase'] == '':
debug("Branch has not been trimmed previously, undo not required.")
return None
# Reset with git-revert
cmt = get_commit_hash(get_current_branch(directory), directory)
cmd = 'git revert --no-edit -Xtheirs ' + config['trimbase'] + '..' + cmt
execute_command(cmd, cwd=directory)
# Unset the trimbase
config['trimbase'] = ''
return config
def import_patches(directory=None):
# Get current branch
current_branch = get_current_branch(directory)
# Construct the patches branch name
patches_branch = 'patches/' + current_branch
# Ensure the patches branch exists and is tracked
if branch_exists(patches_branch, False, directory=directory):
if not branch_exists(patches_branch, True, directory=directory):
track_branches(patches_branch, directory)
else:
error("The patches branch ({0}) does not ".format(patches_branch) + \
"exist, did you use git-bloom-branch?")
return code.BRANCH_DOES_NOT_EXIST
# Create a swap space
tmp_dir = tempfile.mkdtemp()
try:
# Get parent branch and base commit from patches branch
config = get_patch_config(patches_branch, directory)
parent_branch, commit = config['parent'], config['base']
if commit != get_commit_hash(current_branch, directory):
warning("The current commit is not the same as the most recent "
"rebase commit. This might mean that you have committed "
"since the last time you did 'git-bloom-patch export'.")
return code.PATCHES_NOT_EXPORTED
# Checkout to the patches branch
checkout(patches_branch, directory=directory)
# Copy the patches to a temp location
patches = list_patches(directory)
if len(patches) == 0:
debug("No patches in the patches branch, nothing to do")
return code.NOTHING_TO_DO
tmp_dir_patches = []
for patch in patches:
tmp_dir_patches.append(os.path.join(tmp_dir, patch))
if directory is not None:
patch = os.path.join(directory, patch)
shutil.copy(patch, tmp_dir)
# Now checkout back to the original branch and import them
checkout(current_branch, directory=directory)
cmd = 'git am {0}*.patch'.format(tmp_dir + os.sep)
execute_command(cmd, cwd=directory)
# Notify the user
info("Applied {0} patches".format(len(patches)))
finally:
if current_branch:
checkout(current_branch, directory=directory)
if os.path.exists(tmp_dir):
shutil.rmtree(tmp_dir)
return 0
def main(args=None, get_subs_fn=None):
get_subs_fn = get_subs_fn or get_subs
_place_template_files = True
_process_template_files = True
package_path = os.getcwd()
if args is not None:
package_path = args.package_path or os.getcwd()
_place_template_files = args.place_template_files
_process_template_files = args.process_template_files
pkgs_dict = find_packages(package_path)
if len(pkgs_dict) == 0:
sys.exit("No packages found in path: '{0}'".format(package_path))
if len(pkgs_dict) > 1:
sys.exit("Multiple packages found, this tool only supports one package at a time.")
ros_distro = os.environ.get('ROS_DISTRO', 'groovy')
# Allow args overrides
ros_distro = args.ros_distro or ros_distro
# Summarize
info(fmt("@!@{gf}==> @|") +
fmt("Generating Homebrew formula for package(s) %s" %
([p.name for p in pkgs_dict.values()])))
for path, pkg in pkgs_dict.items():
template_files = None
try:
subs = get_subs_fn(pkg, ros_distro)
if _place_template_files:
# Place template files
place_template_files(path)
if _process_template_files:
# Just process existing template files
template_files = process_template_files(path, subs)
if not _place_template_files and not _process_template_files:
# If neither, do both
place_template_files(path)
template_files = process_template_files(path, subs)
if template_files is not None:
for template_file in template_files:
os.remove(os.path.normpath(template_file))
except Exception as exc:
debug(traceback.format_exc())
error(type(exc).__name__ + ": " + str(exc), exit=True)
except (KeyboardInterrupt, EOFError):
sys.exit(1)
def try_execute(msg, err_msg, func, *args, **kwargs):
retcode = 0
try:
retcode = func(*args, **kwargs)
retcode = retcode if retcode is not None else 0
except CalledProcessError as err:
print_exc(traceback.format_exc())
error("Error calling {0}: {1}".format(msg, str(err)))
retcode = err.returncode
ret_msg = msg + " returned exit code ({0})".format(str(retcode))
if retcode > 0:
error(ret_msg)
raise CommandFailed(retcode)
elif retcode < 0:
debug(ret_msg)
return retcode
def _undo(config, directory):
debug("_undo(" + str(config) + ", " + str(directory) + ")")
# TODO: handle repo with changes
# TODO: handle repo with patches applied
if config['trimbase'] == '':
debug("Branch has not been trimmed previously, undo not required.")
return None
# Reset with git-revert
current_branch = get_current_branch(directory)
if current_branch is None:
error("Could not determine current branch.", exit=True)
cmt = get_commit_hash(current_branch, directory)
cmd = 'git revert --no-edit -Xtheirs ' + config['trimbase'] + '..' + cmt
execute_command(cmd, cwd=directory)
# Unset the trimbase
config['trimbase'] = ''
return config
def _undo(config, directory):
debug("_undo(" + str(config) + ", " + str(directory) + ")")
# TODO: handle repo with changes
# TODO: handle repo with patches applied
if config["trimbase"] == "":
debug("Branch has not been trimmed previously, undo not required.")
return None
# Reset with git-revert
current_branch = get_current_branch(directory)
if current_branch is None:
error("Could not determine current branch.", exit=True)
cmt = get_commit_hash(current_branch, directory)
cmd = "git revert --no-edit -Xtheirs " + config["trimbase"] + ".." + cmt
execute_command(cmd, cwd=directory)
# Unset the trimbase
config["trimbase"] = ""
return config
def track_branches(branches=None, directory=None):
"""
Tracks all specified branches.
:param branches: a list of branches that are to be tracked if not already
tracked. If this is set to None then all remote branches will be tracked.
:param directory: directory in which to run all commands
:raises: subprocess.CalledProcessError if git command fails
"""
if type(branches) == str:
branches = [branches]
debug("track_branches(" + str(branches) + ", " + str(directory) + ")")
if branches == []:
return
# Save the current branch
current_branch = get_current_branch(directory)
try:
# Get the local branches
local_branches = get_branches(local_only=True, directory=directory)
# Get the remote and local branches
all_branches = get_branches(local_only=False, directory=directory)
# Calculate the untracked branches
untracked_branches = []
for branch in all_branches:
if branch.startswith('remotes/'):
if branch.count('/') >= 2:
branch = '/'.join(branch.split('/')[2:])
if branch not in local_branches:
untracked_branches.append(branch)
# Prune any untracked branches by specified branches
if branches is not None:
branches_to_track = []
for untracked in untracked_branches:
if untracked in branches:
branches_to_track.append(untracked)
else:
branches_to_track = untracked_branches
# Track branches
debug("Tracking branches: " + str(branches_to_track))
for branch in branches_to_track:
checkout(branch, directory=directory)
finally:
if current_branch:
checkout(current_branch, directory=directory)
def rebase_patches(without_git_rebase=True, directory=None):
### Ensure a clean/valid working environment
ret = ensure_clean_working_env(git_status=True, directory=directory)
if ret != 0:
return ret
### Make sure we need to actually call this
# Get the current branch
current_branch = get_current_branch(directory)
# Get the patches branch
patches_branch = "patches/" + current_branch
# Get the current patches.conf
config = get_patch_config(patches_branch, directory=directory)
# Get the current upstream commit hash
upstream_commit_hash = get_commit_hash(config["parent"], directory)
# If the current upstream commit hash is the same as the stored one, noop
if upstream_commit_hash == config["previous"]:
debug(
"Nothing to do: Current branch (" + current_branch + ")'s "
"base commit hash is the same as the source branch (" + config["parent"] + ")'s commit hash."
)
debug(" Did you forget to update the parent branch first?")
debug(
" Updating the parent branch can be done by calling "
"'git-bloom-patch rebase' on it, or 'git-bloom-import-upsteam'"
" if the parent branch is the upstream branch."
)
return 0
else:
debug(
"rebase_patches: "
+ upstream_commit_hash
+ " == "
+ config["previous"]
+ ": "
+ str(upstream_commit_hash == config["previous"])
)
### Execute the rebase
if without_git_rebase:
non_git_rebase(config["parent"], directory=directory)
else:
git_rebase(config["parent"], directory=directory)
### Update the patches information
# Get the latest configs
config = get_patch_config(patches_branch, directory)
# Set the base to the current hash (before patches)
config["base"] = get_commit_hash(current_branch, directory)
# Set the new upstream hash to the previous upstream hash
config["previous"] = get_commit_hash(config["parent"], directory)
# Clear the trimbase (it needs to be reapplied)
config["trimbase"] = ""
# Write the new configs
set_patch_config(patches_branch, config, directory)
return 0
def resolve_rosdep_key(key,
os_name,
os_version,
ros_distro=None,
ignored=None,
retry=True):
ignored = ignored or []
ctx = create_default_installer_context()
try:
installer_key = ctx.get_default_os_installer_key(os_name)
except KeyError:
BloomGenerator.exit(
"Could not determine the installer for '{0}'".format(os_name))
installer = ctx.get_installer(installer_key)
ros_distro = ros_distro or DEFAULT_ROS_DISTRO
view = get_view(os_name, os_version, ros_distro)
try:
return resolve_more_for_os(key, view, installer, os_name, os_version)
except (KeyError, ResolutionError) as exc:
debug(traceback.format_exc())
if key in ignored:
return None, None, None
if isinstance(exc, KeyError):
error("Could not resolve rosdep key '{0}'".format(key))
returncode = code.GENERATOR_NO_SUCH_ROSDEP_KEY
else:
error(
"Could not resolve rosdep key '{0}' for distro '{1}':".format(
key, os_version))
info(str(exc), use_prefix=False)
returncode = code.GENERATOR_NO_ROSDEP_KEY_FOR_DISTRO
if retry:
error("Try to resolve the problem with rosdep and then continue.")
if maybe_continue():
update_rosdep()
invalidate_view_cache()
return resolve_rosdep_key(key,
os_name,
os_version,
ros_distro,
ignored,
retry=True)
BloomGenerator.exit(
"Failed to resolve rosdep key '{0}', aborting.".format(key),
returncode=returncode)
def track_branches(branches=None, directory=None):
"""
Tracks all specified branches.
:param branches: a list of branches that are to be tracked if not already
tracked. If this is set to None then all remote branches will be tracked.
:param directory: directory in which to run all commands
:raises: subprocess.CalledProcessError if git command fails
"""
if type(branches) == str:
branches = [branches]
debug("track_branches(" + str(branches) + ", " + str(directory) + ")")
if branches == []:
return
# Save the current branch
current_branch = get_current_branch(directory)
try:
# Get the local branches
local_branches = get_branches(local_only=True, directory=directory)
# Get the remote and local branches
all_branches = get_branches(local_only=False, directory=directory)
# Calculate the untracked branches
untracked_branches = []
for branch in all_branches:
if branch.startswith('remotes/'):
if branch.count('/') >= 2:
branch = '/'.join(branch.split('/')[2:])
if branch not in local_branches:
untracked_branches.append(branch)
# Prune any untracked branches by specified branches
if branches is not None:
branches_to_track = []
for untracked in untracked_branches:
if untracked in branches:
branches_to_track.append(untracked)
else:
branches_to_track = untracked_branches
# Track branches
debug("Tracking branches: " + str(branches_to_track))
for branch in branches_to_track:
checkout(branch, directory=directory)
finally:
if current_branch:
checkout(current_branch, directory=directory)
def export_patches(directory=None):
### Ensure a clean/valid working environment
ensure_clean_working_env(git_status=True, directory=directory)
# Get current branch
current_branch = get_current_branch(directory)
if current_branch is None:
error("Could not determine current branch.", exit=True)
# Construct the patches branch name
patches_branch = 'patches/' + current_branch
# Ensure the patches branch exists
if not branch_exists(patches_branch, False, directory=directory):
error("The patches branch ({0}) does not ".format(patches_branch) +
"exist, did you use git-bloom-branch?",
exit=True)
try:
# Get parent branch and base commit from patches branch
config = get_patch_config(patches_branch, directory)
if config is None:
error("Failed to get patches information.", exit=True)
# Checkout to the patches branch
checkout(patches_branch, directory=directory)
# Notify the user
debug("Exporting patches from "
"{0}...{1}".format(config['base'], current_branch))
# Remove all the old patches
if len(list_patches(directory)) > 0:
cmd = 'git rm ./*.patch'
execute_command(cmd, cwd=directory)
# Create the patches using git format-patch
cmd = "git format-patch -M -B " \
"{0}...{1}".format(config['base'], current_branch)
execute_command(cmd, cwd=directory)
# Report of the number of patches created
patches_list = list_patches(directory)
debug("Created {0} patches".format(len(patches_list)))
# Clean up and commit
if len(patches_list) > 0:
cmd = 'git add ./*.patch'
execute_command(cmd, cwd=directory)
if has_changes(directory):
cmd = 'git commit -m "Updating patches."'
execute_command(cmd, cwd=directory)
finally:
if current_branch:
checkout(current_branch, directory=directory)
def export_patches(directory=None):
### Ensure a clean/valid working environment
ensure_clean_working_env(git_status=True, directory=directory)
# Get current branch
current_branch = get_current_branch(directory)
if current_branch is None:
error("Could not determine current branch.", exit=True)
# Construct the patches branch name
patches_branch = 'patches/' + current_branch
# Ensure the patches branch exists
if not branch_exists(patches_branch, False, directory=directory):
error("The patches branch ({0}) does not ".format(patches_branch) +
"exist, did you use git-bloom-branch?", exit=True)
try:
# Get parent branch and base commit from patches branch
config = get_patch_config(patches_branch, directory)
if config is None:
error("Failed to get patches information.", exit=True)
# Checkout to the patches branch
checkout(patches_branch, directory=directory)
# Notify the user
debug("Exporting patches from "
"{0}...{1}".format(config['base'], current_branch))
# Remove all the old patches
if len(list_patches(directory)) > 0:
cmd = 'git rm ./*.patch'
execute_command(cmd, cwd=directory)
# Create the patches using git format-patch
cmd = "git format-patch -M -B " \
"{0}...{1}".format(config['base'], current_branch)
execute_command(cmd, cwd=directory)
# Report of the number of patches created
patches_list = list_patches(directory)
debug("Created {0} patches".format(len(patches_list)))
# Clean up and commit
if len(patches_list) > 0:
cmd = 'git add ./*.patch'
execute_command(cmd, cwd=directory)
if has_changes(directory):
cmd = 'git commit -m "Updating patches."'
execute_command(cmd, cwd=directory)
finally:
if current_branch:
checkout(current_branch, directory=directory)
def get_index_url():
global _rosdistro_index_commit, _rosdistro_index_original_branch
index_url = rosdistro.get_index_url()
pr = urlparse(index_url)
if pr.netloc in ['raw.github.com', 'raw.githubusercontent.com']:
# Try to determine what the commit hash was
tokens = [x for x in pr.path.split('/') if x]
if len(tokens) <= 3:
debug("Failed to get commit for rosdistro index file: index url")
debug(tokens)
return index_url
owner = tokens[0]
repo = tokens[1]
branch = tokens[2]
gh = get_github_interface(quiet=True)
if gh is None:
# Failed to get it with auth, try without auth (may fail)
gh = Github(username=None, auth=None)
try:
data = gh.get_branch(owner, repo, branch)
except GithubException:
debug(traceback.format_exc())
debug("Failed to get commit for rosdistro index file: api")
return index_url
_rosdistro_index_commit = data.get('commit', {}).get('sha', None)
if _rosdistro_index_commit is not None:
info("ROS Distro index file associate with commit '{0}'".format(
_rosdistro_index_commit))
# Also mutate the index_url to use the commit (rather than the moving branch name)
base_info = get_gh_info(index_url)
base_branch = base_info['branch']
rosdistro_index_commit = _rosdistro_index_commit # Copy global into local for substitution
middle = "{org}/{repo}".format(**base_info)
index_url = index_url.replace(
"{pr.netloc}/{middle}/{base_branch}/".format(**locals()),
"{pr.netloc}/{middle}/{rosdistro_index_commit}/".format(
**locals()))
info("New ROS Distro index url: '{0}'".format(index_url))
_rosdistro_index_original_branch = base_branch
else:
debug("Failed to get commit for rosdistro index file: json")
return index_url
def match_branches_with_prefix(prefix, get_branches, prune=False):
debug("match_branches_with_prefix(" + str(prefix) + ", " +
str(get_branches()) + ")")
branches = []
# Match branches
existing_branches = get_branches()
for branch in existing_branches:
if branch.startswith('remotes/origin/'):
branch = branch.split('/', 2)[-1]
if branch.startswith(prefix):
branches.append(branch)
branches = list(set(branches))
if prune:
# Prune listed branches by packages in latest upstream
with inbranch('upstream'):
pkg_names, version, pkgs_dict = get_package_data('upstream')
for branch in branches:
if branch.split(prefix)[-1].strip('/') not in pkg_names:
branches.remove(branch)
return branches
def match_branches_with_prefix(prefix, get_branches, prune=False):
debug("match_branches_with_prefix(" + str(prefix) + ", " +
str(get_branches()) + ")")
branches = []
# Match branches
existing_branches = get_branches()
for branch in existing_branches:
if branch.startswith('remotes/origin/'):
branch = branch.split('/', 2)[-1]
if branch.startswith(prefix):
branches.append(branch)
branches = list(set(branches))
if prune:
# Prune listed branches by packages in latest upstream
with inbranch('upstream'):
pkg_names, version, pkgs_dict = get_package_data('upstream')
for branch in branches:
if branch.split(prefix)[-1].strip('/') not in pkg_names:
branches.remove(branch)
return branches
def _trim(config, force, directory):
debug("_trim(" + str(config) + ", " + str(force) + ", " + \
str(directory) + ")")
if config['trimbase'] != '':
warning("It looks like the trim operation has already been done, "
"nested trimming is not supported.")
if force:
warning("Proceeding anyways because of '--force'")
else:
warning("If you would like to continue anyways use '--force'")
return None
config['trimbase'] = get_commit_hash(get_current_branch(directory))
tmp_dir = tempfile.mkdtemp()
try:
# Buckup trim sub directory
git_root = get_root()
sub_dir = os.path.join(git_root, config['trim'])
storage = os.path.join(tmp_dir, config['trim'])
shutil.copytree(sub_dir, storage)
# Clear out the git repo
execute_command('git rm -rf ./*', cwd=directory)
# Copy the sub directory back
for item in os.listdir(storage):
src = os.path.join(storage, item)
dst = os.path.join(git_root, item)
if os.path.isdir(src):
shutil.copytree(src, dst)
else:
shutil.copy(src, dst)
# Stage
execute_command('git add ./*', cwd=directory)
# Commit
cmd = 'git commit -m "Trimmed the branch to only the ' + \
config['trim'] + ' sub directory"'
execute_command(cmd, cwd=directory)
# Update the patch base to be this commit
config['base'] = get_commit_hash(get_current_branch(directory))
finally:
if os.path.exists(tmp_dir):
shutil.rmtree(tmp_dir)
return config
def checkout_summarize(fail_msg, branch, directory):
branch = '(no branch)' if branch is None else branch
directory = os.getcwd() if directory is None else directory
error("Failed to checkout to '{0}'".format(str(reference)) +
" because the working directory {0}".format(str(fail_msg)))
debug(" Working directory: '{0}'".format(str(directory)))
debug(" Working branch: '{0}'".format(str(branch)))
debug(" Has local changes: '{0}'".format(str(changes)))
debug(" Has untrakced files: '{0}'".format(str(untracked)))
pdb_hook()
if not bloom.util._quiet and show_git_status:
info('\n++ git status:\n', use_prefix=False)
os.system('git status')
return False
def rebase_patches(without_git_rebase=True, directory=None):
### Ensure a clean/valid working environment
ensure_clean_working_env(git_status=True, directory=directory)
### Make sure we need to actually call this
# Get the current branch
current_branch = get_current_branch(directory)
if current_branch is None:
error("Could not determine current branch.", exit=True)
# Get the patches branch
patches_branch = 'patches/' + current_branch
# Get the current patches.conf
config = get_patch_config(patches_branch, directory=directory)
### Execute the rebase
if without_git_rebase:
non_git_rebase(config['parent'], directory=directory)
else:
git_rebase(config['parent'], directory=directory)
### Update the patches information
# Get the latest configs
config = get_patch_config(patches_branch, directory)
# Set the base to the current hash (before patches)
current_branch_ = get_current_branch(directory)
debug('Current branch: ' + current_branch_ or 'could not determine branch')
config['base'] = get_commit_hash(current_branch_, directory)
debug('New current commit hash after rebase: ' + config['base'])
# Set the new upstream hash to the previous upstream hash
config['previous'] = get_commit_hash(config['parent'], directory)
debug('New parent commit hash after rebase: ' + config['previous'])
# Clear the trimbase (it needs to be reapplied)
config['trimbase'] = ''
# Write the new configs
set_patch_config(patches_branch, config, directory)
def checkout_summarize(fail_msg, branch, directory):
branch = '(no branch)' if branch is None else branch
directory = os.getcwd() if directory is None else directory
error("Failed to checkout to '{0}'".format(str(reference)) +
" because the working directory {0}".format(str(fail_msg)))
debug(" Working directory: '{0}'".format(str(directory)))
debug(" Working branch: '{0}'".format(str(branch)))
debug(" Has local changes: '{0}'".format(str(changes)))
debug(" Has untrakced files: '{0}'".format(str(untracked)))
pdb_hook()
if not bloom.util._quiet and show_git_status:
print('\n++ git status:\n')
os.system('git status')
return False
def rebase_patches(without_git_rebase=True, directory=None):
### Ensure a clean/valid working environment
ensure_clean_working_env(git_status=True, directory=directory)
### Make sure we need to actually call this
# Get the current branch
current_branch = get_current_branch(directory)
if current_branch is None:
error("Could not determine current branch.", exit=True)
# Get the patches branch
patches_branch = 'patches/' + current_branch
# Get the current patches.conf
config = get_patch_config(patches_branch, directory=directory)
### Execute the rebase
if without_git_rebase:
non_git_rebase(config['parent'], directory=directory)
else:
git_rebase(config['parent'], directory=directory)
### Update the patches information
# Get the latest configs
config = get_patch_config(patches_branch, directory)
# Set the base to the current hash (before patches)
current_branch_ = get_current_branch(directory)
debug('Current branch: ' + current_branch_ or 'could not determine branch')
config['base'] = get_commit_hash(current_branch_, directory)
debug('New current commit hash after rebase: ' + config['base'])
# Set the new upstream hash to the previous upstream hash
config['previous'] = get_commit_hash(config['parent'], directory)
debug('New parent commit hash after rebase: ' + config['previous'])
# Clear the trimbase (it needs to be reapplied)
config['trimbase'] = ''
# Write the new configs
set_patch_config(patches_branch, config, directory)
def check_for_bloom():
"""
Checks for the bloom branch, else looks for and converts the catkin branch.
Then it checks for the bloom branch and that it contains a bloom.conf file.
"""
if branch_exists('catkin') and not branch_exists('bloom'):
# Found catkin branch, migrate it to bloom
info('catkin branch detected, up converting to the bloom branch')
convert_catkin_to_bloom()
return
# Check for bloom.conf
if os.path.exists('bloom'):
error("File or directory bloom prevents checking out to the bloom "
"branch, remove it.")
sys.exit(1)
if not branch_exists('bloom'):
debug('no bloom branch')
not_a_bloom_release_repo()
with inbranch('bloom'):
if not os.path.exists(os.path.join(os.getcwd(), 'bloom.conf')):
debug('no bloom.conf file')
not_a_bloom_release_repo()
def resolve_rosdep_key(key,
os_name,
os_version,
ros_distro=None,
ignored=None,
retry=True):
ignored = ignored or []
if os_name not in default_installers:
BloomGenerator.exit(
"Could not determine the installer for '{0}'".format(os_name))
installer = get_installer(default_installers[os_name][0])
ros_distro = ros_distro or DEFAULT_ROS_DISTRO
view = get_view(os_name, os_version, ros_distro)
try:
return resolve_for_os(key, view, installer, os_name, os_version)
except (KeyError, ResolutionError) as exc:
debug(traceback.format_exc())
if key in ignored:
return None
if isinstance(exc, KeyError):
error("Could not resolve rosdep key '{0}'".format(key))
else:
error(
"Could not resolve rosdep key '{0}' for distro '{1}':".format(
key, os_version))
info(str(exc), use_prefix=False)
if retry:
error("Try to resolve the problem with rosdep and then continue.")
if maybe_continue():
update_rosdep()
invalidate_view_cache()
return resolve_rosdep_key(key,
os_name,
os_version,
ros_distro,
ignored,
retry=True)
BloomGenerator.exit(
"Failed to resolve rosdep key '{0}', aborting.".format(key))
def remove_patches(directory=None):
# Get the current branch
current_branch = get_current_branch(directory)
if current_branch is None:
error("Could not determine current branch.", exit=True)
# Ensure the current branch is valid
if current_branch is None:
error("Could not determine current branch, are you in a git repo?",
exit=True)
# Construct the patches branch
patches_branch = 'patches/' + current_branch
try:
# See if the patches branch exists
if branch_exists(patches_branch, False, directory=directory):
if not branch_exists(patches_branch, True, directory=directory):
track_branches(patches_branch, directory)
else:
error("No patches branch (" + patches_branch + ") found, cannot "
"remove patches.",
exit=True)
# Get the parent branch from the patches branch
config = get_patch_config(patches_branch, directory=directory)
parent, spec = config['parent'], config['base']
if None in [parent, spec]:
error("Could not retrieve patches info.", exit=True)
debug("Removing patches from " + current_branch + " back to base "
"commit " + spec)
# Reset this branch using git revert --no-edit spec
current_commit = get_commit_hash(current_branch, directory)
command_spec = spec + '..' + current_commit
execute_command('git revert --no-edit -Xtheirs ' + command_spec,
cwd=directory)
# Update the base
config['base'] = get_commit_hash(current_branch, directory)
set_patch_config(patches_branch, config, directory=directory)
finally:
if current_branch:
checkout(current_branch, directory=directory)
def remove_patches(directory=None):
# Get the current branch
current_branch = get_current_branch(directory)
if current_branch is None:
error("Could not determine current branch.", exit=True)
# Ensure the current branch is valid
if current_branch is None:
error("Could not determine current branch, are you in a git repo?",
exit=True)
# Construct the patches branch
patches_branch = 'patches/' + current_branch
try:
# See if the patches branch exists
if branch_exists(patches_branch, False, directory=directory):
if not branch_exists(patches_branch, True, directory=directory):
track_branches(patches_branch, directory)
else:
error("No patches branch (" + patches_branch + ") found, cannot "
"remove patches.", exit=True)
# Get the parent branch from the patches branch
config = get_patch_config(patches_branch, directory=directory)
parent, spec = config['parent'], config['base']
if None in [parent, spec]:
error("Could not retrieve patches info.", exit=True)
debug("Removing patches from " + current_branch + " back to base "
"commit " + spec)
# Reset this branch using git revert --no-edit spec
current_commit = get_commit_hash(current_branch, directory)
command_spec = spec + '..' + current_commit
execute_command(
'git revert --no-edit -Xtheirs ' + command_spec, cwd=directory
)
# Update the base
config['base'] = get_commit_hash(current_branch, directory)
set_patch_config(patches_branch, config, directory=directory)
finally:
if current_branch:
checkout(current_branch, directory=directory)
def resolve_rosdep_key(
key,
os_name,
os_version,
ros_distro=None,
ignored=None,
retry=True
):
ignored = ignored or []
if os_name not in default_installers:
BloomGenerator.exit("Could not determine the installer for '{0}'"
.format(os_name))
installer = get_installer(default_installers[os_name][0])
ros_distro = ros_distro or DEFAULT_ROS_DISTRO
view = get_view(os_name, os_version, ros_distro)
try:
return resolve_for_os(key, view, installer, os_name, os_version)
except (KeyError, ResolutionError) as exc:
debug(traceback.format_exc())
if key in ignored:
return None
if isinstance(exc, KeyError):
error("Could not resolve rosdep key '{0}'".format(key))
else:
error("Could not resolve rosdep key '{0}' for distro '{1}':"
.format(key, os_version))
info(str(exc), use_prefix=False)
if retry:
error("Try to resolve the problem with rosdep and then continue.")
if maybe_continue():
update_rosdep()
invalidate_view_cache()
return resolve_rosdep_key(key, os_name, os_version, ros_distro,
ignored, retry=True)
BloomGenerator.exit("Failed to resolve rosdep key '{0}', aborting."
.format(key))
def convert_old_bloom_conf(prefix=None):
prefix = prefix if prefix is not None else 'convert'
tracks_dict = get_tracks_dict_raw()
track = prefix
track_count = 0
while track in tracks_dict['tracks']:
track_count += 1
track = prefix + str(track_count)
track_dict = copy.copy(DEFAULT_TEMPLATE)
cmd = 'git config -f bloom.conf bloom.upstream'
upstream_repo = check_output(cmd, shell=True).strip()
cmd = 'git config -f bloom.conf bloom.upstreamtype'
upstream_type = check_output(cmd, shell=True).strip()
try:
cmd = 'git config -f bloom.conf bloom.upstreambranch'
upstream_branch = check_output(cmd, shell=True).strip()
except subprocess.CalledProcessError:
upstream_branch = ''
for key in template_entry_order:
if key == 'vcs_uri':
track_dict[key] = upstream_repo
continue
if key == 'vcs_type':
track_dict[key] = upstream_type
continue
if key == 'vcs_uri':
track_dict[key] = upstream_branch or None
continue
track_dict[key] = track_dict[key].default
debug('Converted bloom.conf:')
with open('bloom.conf', 'r') as f:
debug(f.read())
debug('To this track:')
debug(str({track: track_dict}))
tracks_dict['tracks'][track] = track_dict
write_tracks_dict_raw(tracks_dict)
execute_command('git rm bloom.conf', shell=True)
execute_command('git commit -m "Removed bloom.conf"', shell=True)
# Now move the old bloom branch into master
upconvert_bloom_to_config_branch()
def convert_old_bloom_conf(prefix=None):
prefix = prefix if prefix is not None else 'convert'
tracks_dict = get_tracks_dict_raw()
track = prefix
track_count = 0
while track in tracks_dict['tracks']:
track_count += 1
track = prefix + str(track_count)
track_dict = copy.copy(DEFAULT_TEMPLATE)
cmd = 'git config -f bloom.conf bloom.upstream'
upstream_repo = check_output(cmd, shell=True).strip()
cmd = 'git config -f bloom.conf bloom.upstreamtype'
upstream_type = check_output(cmd, shell=True).strip()
try:
cmd = 'git config -f bloom.conf bloom.upstreambranch'
upstream_branch = check_output(cmd, shell=True).strip()
except subprocess.CalledProcessError:
upstream_branch = ''
for key in template_entry_order:
if key == 'vcs_uri':
track_dict[key] = upstream_repo
continue
if key == 'vcs_type':
track_dict[key] = upstream_type
continue
if key == 'vcs_uri':
track_dict[key] = upstream_branch or None
continue
track_dict[key] = track_dict[key].default
debug('Converted bloom.conf:')
with open('bloom.conf', 'r') as f:
debug(f.read())
debug('To this track:')
debug(str({track: track_dict}))
tracks_dict['tracks'][track] = track_dict
write_tracks_dict_raw(tracks_dict)
execute_command('git rm bloom.conf', shell=True)
execute_command('git commit -m "Removed bloom.conf"', shell=True)
# Now move the old bloom branch into master
upconvert_bloom_to_config_branch()
from bloom.logging import debug
from bloom.logging import error
from bloom.logging import fmt
from bloom.logging import info
from debian_generator import generate_substitutions_from_package
from debian_generator import merge_packages
from debian_generator import place_template_files
from debian_generator import process_template_files
from bloom.util import get_distro_list_prompt
try:
from rosdep2 import create_default_installer_context
except ImportError:
debug(traceback.format_exc())
error("rosdep was not detected, please install it.", exit=True)
try:
from catkin_pkg.packages import find_packages
except ImportError:
debug(traceback.format_exc())
error("catkin_pkg was not detected, please install it.", exit=True)
def prepare_arguments(parser):
add = parser.add_argument
add('package_path', nargs='?',
help="path to or containing the package.xml of a package")
action = parser.add_mutually_exclusive_group(required=False)
add = action.add_argument
def perform_release(repository, track, distro, new_track, interactive,
pretend):
release_repo = get_release_repo(repository, distro)
with change_directory(release_repo.get_path()):
# Check to see if the old bloom.conf exists
if check_for_bloom_conf(repository):
# Convert to a track
info("Old bloom.conf file detected.")
info(fmt("@{gf}@!==> @|Converting to bloom.conf to track"))
convert_old_bloom_conf(None if new_track else distro)
upconvert_bloom_to_config_branch()
# Check that the track is valid
tracks_dict = get_tracks_dict_raw()
# If new_track, create the new track first
if new_track:
if not track:
error("You must specify a track when creating a new one.",
exit=True)
if track in tracks_dict['tracks']:
warning("Track '{0}' exists, editing...".format(track))
edit_track_cmd(track)
tracks_dict = get_tracks_dict_raw()
else:
# Create a new track called <track>,
# copying an existing track if possible,
# and overriding the ros_distro
warning("Creating track '{0}'...".format(track))
overrides = {'ros_distro': distro}
new_track_cmd(track, copy_track='', overrides=overrides)
tracks_dict = get_tracks_dict_raw()
if track and track not in tracks_dict['tracks']:
error("Given track '{0}' does not exist in release repository.".
format(track))
error("Available tracks: " + str(tracks_dict['tracks'].keys()),
exit=True)
elif not track:
tracks = tracks_dict['tracks'].keys()
# Error out if there are no tracks
if len(tracks) == 0:
error("Release repository has no tracks.")
info("Manually clone the repository:")
info(" git clone {0}".format(release_repo.get_url()))
info("And then create a new track:")
info(" git-bloom-config new <track name>")
error("Run again after creating a track.", exit=True)
# Error out if there is more than one track
if len(tracks) != 1:
error("No track specified and there is not just one track.")
error("Please specify one of the available tracks: " +
str(tracks),
exit=True)
# Get the only track
track = tracks[0]
start_summary(track)
# Ensure the track is complete
track_dict = tracks_dict['tracks'][track]
track_dict = update_track(track_dict)
tracks_dict['tracks'][track] = track_dict
# Set the release repositories' remote if given
release_repo_url = track_dict.get('release_repo_url', None)
if release_repo_url is not None:
info(
fmt("@{gf}@!==> @|") +
"Setting release repository remote url to '{0}'".format(
release_repo_url))
cmd = 'git remote set-url origin ' + release_repo_url
info(fmt("@{bf}@!==> @|@!") + str(cmd))
try:
subprocess.check_call(cmd, shell=True)
except subprocess.CalledProcessError:
error("Setting the remote url failed, exiting.", exit=True)
# Check for push permissions
try:
info(
fmt("@{gf}@!==> @|Testing for push permission on release repository"
))
cmd = 'git remote -v'
info(fmt("@{bf}@!==> @|@!") + str(cmd))
subprocess.check_call(cmd, shell=True)
# Dry run will authenticate, but not push
cmd = 'git push --dry-run'
info(fmt("@{bf}@!==> @|@!") + str(cmd))
subprocess.check_call(cmd, shell=True)
except subprocess.CalledProcessError:
error("Cannot push to remote release repository.", exit=True)
# Write the track config before releasing
write_tracks_dict_raw(tracks_dict)
# Run the release
info(
fmt("@{gf}@!==> @|") +
"Releasing '{0}' using release track '{1}'".format(
repository, track))
cmd = 'git-bloom-release ' + str(track)
if pretend:
cmd += ' --pretend'
info(fmt("@{bf}@!==> @|@!" + str(cmd)))
try:
subprocess.check_call(cmd, shell=True)
except subprocess.CalledProcessError:
error("Release failed, exiting.", exit=True)
info(
fmt(_success) +
"Released '{0}' using release track '{1}' successfully".format(
repository, track))
# Commit the summary
update_summary(track, repository, distro)
commit_summary()
# Check for pushing
if interactive:
info("Releasing complete, push?")
if not maybe_continue():
error("User answered no to continue prompt, aborting.",
exit=True)
# Push changes to the repository
info(
fmt("@{gf}@!==> @|") +
"Pushing changes to release repository for '{0}'".format(
repository))
cmd = 'git push --all'
if pretend:
cmd += ' --dry-run'
info(fmt("@{bf}@!==> @|@!" + str(cmd)))
try:
subprocess.check_call(cmd, shell=True)
except subprocess.CalledProcessError:
error(
"Pushing changes failed, would you like to add '--force' to 'git push --all'?"
)
if not maybe_continue():
error("Pushing changes failed, exiting.", exit=True)
cmd += ' --force'
info(fmt("@{bf}@!==> @|@!" + str(cmd)))
try:
subprocess.check_call(cmd, shell=True)
except subprocess.CalledProcessError:
error("Pushing changes failed, exiting.", exit=True)
info(fmt(_success) + "Pushed changes successfully")
# Push tags to the repository
info(
fmt("@{gf}@!==> @|") +
"Pushing tags to release repository for '{0}'".format(repository))
cmd = 'git push --tags'
if pretend:
cmd += ' --dry-run'
info(fmt("@{bf}@!==> @|@!" + str(cmd)))
try:
subprocess.check_call(cmd, shell=True)
except subprocess.CalledProcessError:
error(
"Pushing changes failed, would you like to add '--force' to 'git push --tags'?"
)
if not maybe_continue():
error("Pushing tags failed, exiting.", exit=True)
cmd += ' --force'
info(fmt("@{bf}@!==> @|@!" + str(cmd)))
try:
subprocess.check_call(cmd, shell=True)
except subprocess.CalledProcessError:
error("Pushing tags failed, exiting.", exit=True)
info(fmt(_success) + "Pushed tags successfully")
# Propose github pull request
info(
fmt("@{gf}@!==> @|") +
"Generating pull request to distro file located at '{0}'".format(
get_release_file_url(distro)))
try:
pull_request_url = open_pull_request(track, repository, distro)
if pull_request_url:
info(
fmt(_success) +
"Pull request opened at: {0}".format(pull_request_url))
if 'BLOOM_NO_WEBBROWSER' in os.environ and platform.system(
) not in ['Darwin']:
webbrowser.open(pull_request_url)
else:
info(
"The release of your packages was successful, but the pull request failed."
)
info(
"Please manually open a pull request by editing the file here: '{0}'"
.format(get_release_file_url(distro)))
info(fmt(_error) + "No pull request opened.")
except Exception as e:
debug(traceback.format_exc())
error("Failed to open pull request: {0} - {1}".format(
type(e).__name__, e),
exit=True)
def import_patches(directory=None):
# Get current branch
current_branch = get_current_branch(directory)
if current_branch is None:
error("Could not determine current branch.", exit=True)
# Construct the patches branch name
patches_branch = 'patches/' + current_branch
# Ensure the patches branch exists and is tracked
if branch_exists(patches_branch, False, directory=directory):
if not branch_exists(patches_branch, True, directory=directory):
track_branches(patches_branch, directory)
else:
error("The patches branch ({0}) does not ".format(patches_branch) +
"exist, did you use git-bloom-branch?", exit=True)
# Create a swap space
tmp_dir = tempfile.mkdtemp()
try:
# Get parent branch and base commit from patches branch
config = get_patch_config(patches_branch, directory)
parent_branch, commit = config['parent'], config['base']
if commit != get_commit_hash(current_branch, directory):
debug(
"commit != get_commit_hash(current_branch, directory)"
)
debug(
"{0} != get_commit_hash({1}, {2}) != {3}".format(
commit, current_branch, directory,
get_commit_hash(current_branch, directory)
)
)
os.system('git log')
warning(
"The current commit is not the same as the most recent "
"rebase commit."
)
warning(
"This might mean that you have committed since the last "
"time you did:"
)
warning(
" 'git-bloom-patch rebase' or 'git-bloom-patch remove'"
)
warning(
"Make sure you export any commits you want to save first:"
)
warning(" 'git-bloom-patch export'")
error("Patches not exported", exit=True)
# Checkout to the patches branch
checkout(patches_branch, directory=directory)
# Copy the patches to a temp location
patches = list_patches(directory)
if len(patches) == 0:
debug("No patches in the patches branch, nothing to do")
return -1 # Indicates that nothing was done
tmp_dir_patches = []
for patch in patches:
tmp_dir_patches.append(os.path.join(tmp_dir, patch))
if directory is not None:
patch = os.path.join(directory, patch)
shutil.copy(patch, tmp_dir)
# Now checkout back to the original branch and import them
checkout(current_branch, directory=directory)
try:
cmd = 'git am {0}*.patch'.format(tmp_dir + os.sep)
execute_command(cmd, cwd=directory)
except subprocess.CalledProcessError as e:
warning("Failed to apply one or more patches for the "
"'{0}' branch.".format(str(e)))
info('', use_prefix=False)
info('', use_prefix=False)
info(">>> Resolve any conflicts and when you have resolved this "
"problem run 'git am --resolved' and then exit the "
"shell using 'exit 0'. <<<", use_prefix=False)
info(" To abort use 'exit 1'", use_prefix=False)
if 'bash' in os.environ['SHELL']:
ret = subprocess.call([
"/bin/bash", "-l", "-c",
"""\
/bin/bash --rcfile <(echo "if [ -f /etc/bashrc ]; then source /etc/bashrc; fi; \
if [ -f ~/.bashrc ]; then source ~/.bashrc; fi;PS1='(bloom)$PS1'") -i"""
])
else:
ret = subprocess.call("$SHELL", shell=True)
if ret != 0:
error("User failed to resolve patch conflicts, exiting.")
sys.exit("'git-bloom-patch import' aborted.")
info("User reports that conflicts have been resolved, continuing.")
# Notify the user
info("Applied {0} patches".format(len(patches)))
finally:
if current_branch:
checkout(current_branch, directory=directory)
if os.path.exists(tmp_dir):
shutil.rmtree(tmp_dir)
def execute_branch(src, dst, interactive, directory=None):
"""
Changes to the destination branch, creates branch and patches/branch
if they do not exist.
If the dst branch does not exist yet, then it is created by branching the
current working branch or the specified SRC_BRANCH.
If the patches/dst branch branch does not exist yet then it is created.
If the branches are created successful, then the working branch will be
set to the dst branch, otherwise the working branch will remain unchanged.
:param src: source branch from which to copy
:param dst: destination branch
:param interactive: if True actions are summarized before committing
:param directory: directory in which to preform this action
:raises: subprocess.CalledProcessError if any git calls fail
"""
# Determine if the srouce branch exists
if src is None:
error("No source specified and/or not a branch currently", exit=True)
if branch_exists(src, local_only=False, directory=directory):
if not branch_exists(src, local_only=True, directory=directory):
debug("Tracking source branch: {0}".format(src))
track_branches(src, directory)
elif tag_exists(src):
pass
else:
error("Specified source branch does not exist: {0}".format(src),
exit=True)
# Determine if the destination branch needs to be created
create_dst_branch = False
if branch_exists(dst, local_only=False, directory=directory):
if not branch_exists(dst, local_only=True, directory=directory):
debug("Tracking destination branch: {0}".format(dst))
track_branches(dst, directory)
else:
create_dst_branch = True
# Determine if the destination patches branch needs to be created
create_dst_patches_branch = False
dst_patches = 'patches/' + dst
if branch_exists(dst_patches, False, directory=directory):
if not branch_exists(dst_patches, True, directory=directory):
track_branches(dst_patches, directory)
else:
create_dst_patches_branch = True
# Summarize
if interactive:
info("Summary of changes:")
if create_dst_branch:
info(" " * 22 + "- The specified destination branch, " +
ansi('boldon') + dst + ansi('reset') +
", does not exist; it will be created from the source "
"branch " + ansi('boldon') + src + ansi('reset'))
if create_dst_patches_branch:
info(" " * 22 + "- The destination patches branch, " +
ansi('boldon') + dst_patches + ansi('reset') +
", does not exist; it will be created")
info(" " * 22 + "- The working branch will be set to " +
ansi('boldon') + dst + ansi('reset'))
if not maybe_continue():
error("Answered no to continue, aborting.", exit=True)
# Make changes to the layout
current_branch = get_current_branch(directory)
try:
# Change to the src branch
checkout(src, directory=directory)
# Create the dst branch if needed
if create_dst_branch:
create_branch(dst, changeto=True, directory=directory)
else:
checkout(dst, directory=directory)
# Create the dst patches branch if needed
if create_dst_patches_branch:
create_branch(dst_patches, orphaned=True, directory=directory)
# Create the starting config data if it does not exist
patches_ls = ls_tree(dst_patches, directory=directory)
if 'patches.conf' not in patches_ls:
# Patches config not setup, set it up
config = {
'parent': src,
'previous': '',
'base': get_commit_hash(dst, directory=directory),
'trim': '',
'trimbase': ''
}
set_patch_config(dst_patches, config, directory=directory)
else:
config = get_patch_config(dst_patches, directory=directory)
if config['parent'] != src:
warning("Updated parent to '{0}' from '{1}'".format(
src, config['parent']))
config['parent'] = src
config['base'] = get_commit_hash(dst, directory=directory)
set_patch_config(dst_patches, config, directory=directory)
# Command successful, do not switch back to previous branch
current_branch = None
finally:
if current_branch is not None:
checkout(current_branch, directory=directory)
def import_tarball(tarball_path, target_branch, version, name):
if tarball_path.endswith('.zip'):
error("Zip archives are not yet supported.", exit=True)
# Create the tarfile handle
targz = tarfile.open(tarball_path, 'r:gz')
with inbranch(target_branch):
# Prepare list of members to extract, ignoring some
ignores = ('.git', '.gitignore', '.svn', '.hgignore', '.hg', 'CVS')
members = targz.getmembers()
members = [m for m in members if m.name.split('/')[-1] not in ignores]
# Clear out the local branch
items = []
for item in os.listdir(os.getcwd()):
if item in ['.git', '..', '.']:
continue
items.append(item)
if len(items) > 0:
execute_command('git rm -rf ' + ' '.join(items))
# Clear out any untracked files
execute_command('git clean -fdx')
# Extract the tarball into the clean branch
targz.extractall(os.getcwd(), members)
# Check for folder nesting (mostly hg)
items = []
for item in os.listdir(os.getcwd()):
if not item.startswith('.'):
items.append(item)
tarball_prefix = os.path.basename(tarball_path)[:-len('.tag.gz')]
if tarball_prefix in items:
debug('Removing nested tarball folder: ' + str(tarball_prefix))
tarball_prefix_path = os.path.join(os.getcwd(), tarball_prefix)
for item in os.listdir(tarball_prefix_path):
if item in ['.', '..']:
continue
item_path = os.path.join(os.getcwd(), tarball_prefix, item)
debug(
'moving ' + str(item_path) + ' to ' +
str(os.path.join(os.getcwd(), item))
)
shutil.move(item_path, os.path.join(os.getcwd(), item))
shutil.rmtree(tarball_prefix_path)
else:
debug('No nested tarball folder found.')
# Commit changes to the repository
items = []
for item in os.listdir(os.getcwd()):
if item in ['.git', '..', '.']:
continue
items.append(item)
if len(items) > 0:
execute_command('git add ' + ' '.join(items))
# Remove any straggling untracked files
execute_command('git clean -dXf')
# Only if we have local changes commit
# (not true if the upstream didn't change any files)
if has_changes():
msg = "Imported upstream version '{0}' of '{1}'"
msg = msg.format(version, name or 'upstream')
cmd = 'git commit -m "{0}"'.format(msg)
execute_command(cmd)
def _trim(config, force, directory):
debug("_trim(" + str(config) + ", " + str(force) + ", " + str(directory) +
")")
if config['trimbase'] != '':
warning("It looks like the trim operation has already been done, "
"nested trimming is not supported.")
if force:
warning("Proceeding anyways because of '--force'")
else:
warning("If you would like to continue anyways use '--force'")
return None
current_branch = get_current_branch(directory)
if current_branch is None:
error("Could not determine current branch.", exit=True)
config['trimbase'] = get_commit_hash(current_branch)
tmp_dir = tempfile.mkdtemp()
try:
# Buckup trim sub directory
git_root = get_root()
sub_dir = os.path.join(git_root, config['trim'])
storage = os.path.join(tmp_dir, config['trim'])
shutil.copytree(sub_dir, storage)
# Clear out any untracked files
execute_command('git clean -fdx', cwd=directory)
# Collect al files (excluding .git)
items = []
for item in os.listdir(git_root):
if item in ['.git', '..', '.']:
continue
items.append(item)
# Remove and .* files missed by 'git rm -rf *'
if len(items) > 0:
execute_command('git rm -rf ' + ' '.join(items), cwd=directory)
# Copy the sub directory back
for item in os.listdir(storage):
src = os.path.join(storage, item)
dst = os.path.join(git_root, item)
if os.path.isdir(src):
shutil.copytree(src, dst)
else:
shutil.copy(src, dst)
# Stage
execute_command('git add ./*', cwd=directory)
# Collect .* files
dot_items = []
for item in os.listdir(git_root):
if item in ['.git', '..', '.']:
continue
if item.startswith('.'):
dot_items.append(item)
# Add any .* files missed by 'git add ./*'
if len(dot_items) > 0:
execute_command('git add ' + ' '.join(dot_items), cwd=directory)
# Remove any straggling untracked files
execute_command('git clean -dXf', cwd=directory)
# Commit
cmd = 'git commit -m "Trimmed the branch to only the ' + \
config['trim'] + ' sub directory"'
execute_command(cmd, cwd=directory)
# Update the patch base to be this commit
current_branch = get_current_branch(directory)
if current_branch is None:
error("Could not determine current branch.", exit=True)
config['base'] = get_commit_hash(current_branch)
finally:
if os.path.exists(tmp_dir):
shutil.rmtree(tmp_dir)
return config
def main(args=None, get_subs_fn=None):
get_subs_fn = get_subs_fn or get_subs
_place_template_files = True
_process_template_files = True
package_path = os.getcwd()
skip_package_names = None
if args is not None:
package_path = args.package_path or os.getcwd()
_place_template_files = args.place_template_files
_process_template_files = args.process_template_files
# create dependency-skip package name lists
skip_name_path = args.skip_package_names or ''
if os.path.isfile(os.path.abspath(skip_name_path)):
with open(os.path.abspath(skip_name_path), 'r') as f:
skip_package_names = []
for line in f:
# comment string
if line[:1] == '#':
continue
# add lists
skip_package_names.append(line.strip())
pkgs_dict = find_packages(package_path)
if len(pkgs_dict) == 0:
sys.exit("No packages found in path: '{0}'".format(package_path))
if len(pkgs_dict) > 1:
sys.exit("Multiple packages found, "
"this tool only supports one package at a time.")
os_data = create_default_installer_context().get_os_name_and_version()
os_name, os_version = os_data
ros_distro = os.environ.get('ROS_DISTRO', 'indigo')
# Allow args overrides
os_name = args.os_name or os_name
os_version = args.os_version or os_version
ros_distro = args.ros_distro or ros_distro
# Summarize
info(
fmt("@!@{gf}==> @|") +
fmt("Generating debs for @{cf}%s:%s@| for package(s) %s" %
(os_name, os_version, [p.name for p in pkgs_dict.values()])))
for path, pkg in pkgs_dict.items():
template_files = None
try:
subs = get_subs_fn(pkg, os_name, os_version, ros_distro,
args.native, skip_package_names)
if _place_template_files:
# Place template files
place_template_files(path, pkg.get_build_type())
if _process_template_files:
# Just process existing template files
template_files = process_template_files(path, subs)
if not _place_template_files and not _process_template_files:
# If neither, do both
place_template_files(path, pkg.get_build_type())
template_files = process_template_files(path, subs)
if template_files is not None:
for template_file in template_files:
os.remove(os.path.normpath(template_file))
except Exception as exc:
debug(traceback.format_exc())
error(type(exc).__name__ + ": " + str(exc), exit=True)
except (KeyboardInterrupt, EOFError):
sys.exit(1)
def handle_arguments(self, args):
"""
Hook to handle parsed arguments from argparse
"""
debug("BloomGenerator.handle_arguments: got args -> " + str(args))
