def commit(self):
if self.disabled:
return
info(
fmt("@{bf}<==@| Command successful, committing changes to working copy"
))
current_branch = get_current_branch()
if current_branch is None:
error("Could not determine current branch.", exit=True)
with inbranch(get_commit_hash(get_current_branch())):
with change_directory(self.clone_dir):
new_branches = get_branches()
for branch in self.current_branches:
if branch in new_branches:
new_branches.remove(branch)
for branch in get_branches(local_only=True):
if branch not in new_branches:
with inbranch(branch):
cmd = 'git pull --rebase origin ' + branch
execute_command(cmd)
execute_command('git push --all', silent=False)
try:
execute_command('git push --tags', silent=False)
except subprocess.CalledProcessError:
warning(
"Force pushing tags from clone to working repository, "
"you will have to force push back to origin...")
execute_command('git push --force --tags', silent=False)
self.clean_up()
def main(sysargs=None):
# Check that the current directory is a serviceable git/bloom repo
ensure_clean_working_env()
ensure_git_root()
# Get tracks
tracks_dict = get_tracks_dict_raw()
if not tracks_dict['tracks']:
error("No tracks configured, first create a track with "
"'git-bloom-config new <track_name>'", exit=True)
# Do argparse stuff
parser = get_argument_parser([str(t) for t in tracks_dict['tracks']])
parser = add_global_arguments(parser)
args = parser.parse_args(sysargs)
handle_global_arguments(args)
verify_track(args.track, tracks_dict['tracks'][args.track])
execute_track(args.track, tracks_dict['tracks'][args.track],
args.release_increment, args.pretend, args.debug, args.unsafe)
# Notify the user of success and next action suggestions
print('\n\n')
warning("Tip: Check to ensure that the debian tags created have the same "
"version as the upstream version you are releasing.")
info(fmt("@{gf}@!Everything went as expected, "
"you should check that the new tags match your expectations, and "
"then push to the release repo with:@|"))
info(fmt(" git push --all && git push --tags "
"@{kf}@!# You might have to add --force to the second command if you "
"are over-writing existing flags"))
def convert_package_to_debian_data(self, package):
data = {}
# Name, Version, Description
data['Name'] = package.name
data['Version'] = package.version
data['Description'] = debianize_string(package.description)
# Websites
websites = [str(url) for url in package.urls if url.type == 'website']
homepage = websites[0] if websites else ''
if homepage == '':
warning("No homepage set, defaulting to ''")
data['Homepage'] = homepage
# Build rule templates
if is_meta_package(package):
data['BuildType'] = 'metapackage'
else:
data['BuildType'] = 'cmake'
# Debian Increment Number
data['DebianInc'] = self.debian_inc
# Package name
data['Package'] = self.get_stackage_name(package)
# Installation prefix
data['InstallationPrefix'] = self.install_prefix
# Dependencies
data['Depends'] = set([d.name for d in package.run_depends])
build_deps = (package.build_depends + package.buildtool_depends)
data['BuildDepends'] = set([d.name for d in build_deps])
# Maintainers
maintainers = []
for m in package.maintainers:
maintainers.append(str(m))
data['Maintainer'] = ', '.join(maintainers)
return data
def __exit__(self, exc_type, exc_value, traceback):
if self.current_branch is not None:
checkout(self.current_branch,
raise_exc=True,
directory=self.directory)
else:
warning("Could not determine branch to return to.")
def commit(self):
if self.disabled:
return
info(fmt("@{bf}<==@| Command successful, committing changes to working copy"))
current_branch = get_current_branch()
if current_branch is None:
error("Could not determine current branch.", exit=True)
with inbranch(get_commit_hash(get_current_branch())):
with change_directory(self.clone_dir):
new_branches = get_branches()
for branch in self.current_branches:
if branch in new_branches:
new_branches.remove(branch)
for branch in get_branches(local_only=True):
if branch not in new_branches:
with inbranch(branch):
cmd = 'git pull --rebase origin ' + branch
execute_command(cmd)
execute_command('git push --all', silent=False)
try:
execute_command('git push --tags', silent=False)
except subprocess.CalledProcessError:
warning("Force pushing tags from clone to working repository, "
"you will have to force push back to origin...")
execute_command('git push --force --tags', silent=False)
self.clean_up()
def get_changelogs(package, releaser_history=None):
if releaser_history is None:
warning("No historical releaser history, using current maintainer name "
"and email for each versioned changelog entry.")
releaser_history = {}
if is_debug():
import logging
logging.basicConfig()
import catkin_pkg
catkin_pkg.changelog.log.setLevel(logging.DEBUG)
package_path = os.path.abspath(os.path.dirname(package.filename))
changelog_path = os.path.join(package_path, CHANGELOG_FILENAME)
if os.path.exists(changelog_path):
changelog = get_changelog_from_path(changelog_path)
changelogs = []
maintainer = (package.maintainers[0].name, package.maintainers[0].email)
for version, date, changes in changelog.foreach_version(reverse=True):
changes_str = []
date_str = get_rfc_2822_date(date)
for item in changes:
changes_str.extend([' ' + i for i in to_unicode(item).splitlines()])
# Each entry has (version, date, changes, releaser, releaser_email)
releaser, email = releaser_history.get(version, maintainer)
changelogs.append((
version, date_str, '\n'.join(changes_str), releaser, email
))
return changelogs
else:
warning("No {0} found for package '{1}'"
.format(CHANGELOG_FILENAME, package.name))
return []
def _check_all_keys_are_valid(self, peer_packages, rosdistro):
keys_to_resolve = set()
key_to_packages_which_depends_on = collections.defaultdict(list)
keys_to_ignore = set()
for package in self.packages.values():
evaluate_package_conditions(package, rosdistro)
depends = [
dep for dep in (package.run_depends + package.buildtool_export_depends)
if dep.evaluated_condition is not False]
build_depends = [
dep for dep in (package.build_depends + package.buildtool_depends + package.test_depends)
if dep.evaluated_condition is not False]
unresolved_keys = [
dep for dep in (depends + build_depends + package.replaces + package.conflicts)
if dep.evaluated_condition is not False]
keys_to_ignore = {
dep for dep in keys_to_ignore.union(package.replaces + package.conflicts)
if dep.evaluated_condition is not False}
keys = [d.name for d in unresolved_keys]
keys_to_resolve.update(keys)
for key in keys:
key_to_packages_which_depends_on[key].append(package.name)
for skip_key in self.skip_keys:
try:
keys_to_resolve.remove(skip_key)
except KeyError:
warning("Key '{0}' specified by --skip-keys was not found".format(skip_key))
else:
warning("Skipping dependency key '{0}' per --skip-keys".format(skip_key))
os_name = self.os_name
rosdistro = self.rosdistro
all_keys_valid = True
for key in sorted(keys_to_resolve):
for os_version in self.distros:
try:
extended_peer_packages = peer_packages + [d.name for d in keys_to_ignore]
rule, installer_key, default_installer_key = \
resolve_rosdep_key(key, os_name, os_version, rosdistro, extended_peer_packages,
retry=False)
if rule is None:
continue
if installer_key != default_installer_key:
error("Key '{0}' resolved to '{1}' with installer '{2}', "
"which does not match the default installer '{3}'."
.format(key, rule, installer_key, default_installer_key))
BloomGenerator.exit(
"The RPM generator does not support dependencies "
"which are installed with the '{0}' installer."
.format(installer_key),
returncode=code.GENERATOR_INVALID_INSTALLER_KEY)
except (GeneratorError, RuntimeError) as e:
print(fmt("Failed to resolve @{cf}@!{key}@| on @{bf}{os_name}@|:@{cf}@!{os_version}@| with: {e}")
.format(**locals()))
print(fmt("@{cf}@!{0}@| is depended on by these packages: ").format(key) +
str(list(set(key_to_packages_which_depends_on[key]))))
print(fmt("@{kf}@!<== @{rf}@!Failed@|"))
all_keys_valid = False
return all_keys_valid
def get_changelogs(package, releaser_history=None):
if releaser_history is None:
warning(
"No historical releaser history, using current maintainer name "
"and email for each versioned changelog entry.")
releaser_history = {}
if is_debug():
import logging
logging.basicConfig()
import catkin_pkg
catkin_pkg.changelog.log.setLevel(logging.DEBUG)
package_path = os.path.abspath(os.path.dirname(package.filename))
changelog_path = os.path.join(package_path, CHANGELOG_FILENAME)
if os.path.exists(changelog_path):
changelog = get_changelog_from_path(changelog_path)
changelogs = []
maintainer = (package.maintainers[0].name,
package.maintainers[0].email)
for version, date, changes in changelog.foreach_version(reverse=True):
changes_str = []
date_str = get_rfc_2822_date(date)
for item in changes:
changes_str.extend(
[' ' + i for i in to_unicode(item).splitlines()])
# Each entry has (version, date, changes, releaser, releaser_email)
releaser, email = releaser_history.get(version, maintainer)
changelogs.append(
(version, date_str, '\n'.join(changes_str), releaser, email))
return changelogs
else:
warning("No {0} found for package '{1}'".format(
CHANGELOG_FILENAME, package.name))
return []
def generate_ros_distro_diff(track, repository, distro, distro_file_url, distro_file, distro_file_raw):
with inbranch('upstream'):
# Check for package.xml(s)
try:
from catkin_pkg.packages import find_packages
except ImportError:
debug(traceback.format_exc())
error("catkin_pkg was not detected, please install it.",
file=sys.stderr, exit=True)
packages = find_packages(os.getcwd())
if len(packages) == 0:
warning("No packages found, will not generate 'package: path' entries for rosdistro.")
track_dict = get_tracks_dict_raw()['tracks'][track]
last_version = track_dict['last_version']
release_inc = track_dict['release_inc']
if repository not in distro_file['repositories']:
global _user_provided_release_url
distro_file['repositories'][repository] = {'url': _user_provided_release_url or ''}
distro_file['repositories'][repository]['version'] = '{0}-{1}'.format(last_version, release_inc)
if packages and (len(packages) > 1 or packages.keys()[0] != '.'):
distro_file['repositories'][repository]['packages'] = {}
for path, package in packages.iteritems():
if os.path.basename(path) == package.name:
distro_file['repositories'][repository]['packages'][package.name] = None
else:
distro_file['repositories'][repository]['packages'][package.name] = path
distro_file_name = os.path.join('release', distro_file_url.split('/')[-1])
distro_dump = yaml.dump(distro_file, indent=2, default_flow_style=False)
if distro_file_raw != distro_dump:
udiff = difflib.unified_diff(distro_file_raw.splitlines(), distro_dump.splitlines(),
fromfile=distro_file_name, tofile=distro_file_name)
temp_dir = tempfile.mkdtemp()
version = distro_file['repositories'][repository]['version']
udiff_file = os.path.join(temp_dir, repository + '-' + version + '.patch')
udiff_raw = ''
info("Unified diff for the ROS distro file located at '{0}':".format(udiff_file))
for line in udiff:
if line.startswith('@@'):
udiff_raw += line
line = fmt('@{cf}' + line)
if line.startswith('+'):
if not line.startswith('+++'):
line += '\n'
udiff_raw += line
line = fmt('@{gf}' + line)
if line.startswith('-'):
if not line.startswith('---'):
line += '\n'
udiff_raw += line
line = fmt('@{rf}' + line)
if line.startswith(' '):
line += '\n'
udiff_raw += line
info(line, use_prefix=False, end='')
with open(udiff_file, 'w+') as f:
f.write(udiff_raw)
return udiff_file, distro_dump
else:
warning("This release resulted in no changes to the ROS distro file...")
return None, None
def handle_tree(tree, directory, root_path, version):
for path, kind in tree.items():
if kind == 'directory':
# Path relative to start path
rel_path = os.path.join(directory, path)
# If it is a file, error
if os.path.isfile(rel_path):
error("In patches path '{0}' is a directory".format(rel_path) +
", but it exists in the upstream branch as a file.",
exit=True)
# If it is not already a directory, create it
if not os.path.isdir(rel_path):
info(" Createing directory... '{0}'".format(rel_path))
os.mkdir(rel_path)
# Recurse on the directory
handle_tree(
ls_tree(BLOOM_CONFIG_BRANCH, os.path.join(root_path,
rel_path)), rel_path,
root_path, version)
if kind == 'file':
# Path relative to start path
rel_path = os.path.join(directory, path)
# If the local version is a directory, error
if os.path.isdir(rel_path):
error("In patches path '{0}' is a file, ".format(rel_path) +
"but it exists in the upstream branch as a directory.",
exit=True)
# If the file already exists, warn
if os.path.isfile(rel_path):
warning(" File '{0}' already exists, overwriting...".format(
rel_path))
execute_command('git rm {0}'.format(rel_path), shell=True)
# If package.xml tempalte in version, else grab data
if path in ['stack.xml']:
warning(
" Skipping '{0}' templating, fuerte not supported".format(
rel_path))
if path in ['package.xml']:
info(" Templating '{0}' into upstream branch...".format(
rel_path))
file_data = show(BLOOM_CONFIG_BRANCH,
os.path.join(root_path, rel_path))
file_data = file_data.replace(':{version}', version)
else:
info(" Overlaying '{0}' into upstream branch...".format(
rel_path))
file_data = show(BLOOM_CONFIG_BRANCH,
os.path.join(root_path, rel_path))
# Write file
with open(rel_path, 'wb') as f:
# Python 2 will treat this as an ascii string but
# Python 3 will not re-decode a utf-8 string.
if sys.version_info.major == 2:
file_data = file_data.decode('utf-8').encode('utf-8')
else:
file_data = file_data.encode('utf-8')
f.write(file_data)
# Add it with git
execute_command('git add {0}'.format(rel_path), shell=True)
def segment_version(full_version):
version_list = full_version.split('.')
if len(version_list) != 3:
warning('Invalid version element, expected: '
'<major>.<minor>.<patch>')
if len(version_list) < 3:
sys.exit(code.INVALID_VERSION)
return version_list
def segment_version(full_version):
version_list = full_version.split('.')
if len(version_list) != 3:
warning('Invalid version element, expected: '
'<major>.<minor>.<patch>')
if len(version_list) < 3:
sys.exit(code.INVALID_VERSION)
return version_list
def get_package_data(branch_name=None,
directory=None,
quiet=True,
release_directory=None):
"""
Gets package data about the package(s) in the current branch.
It also ignores the packages in the `packages.ignore` file in the master branch.
:param branch_name: name of the branch you are searching on (log use only)
"""
log = debug if quiet else info
repo_dir = directory or os.getcwd()
if branch_name:
log("Looking for packages in '{0}' branch... ".format(branch_name),
end='')
else:
log("Looking for packages in '{0}'... ".format(directory
or os.getcwd()),
end='')
# Check for package.xml(s)
packages = find_packages(repo_dir)
if type(packages) == dict and packages != {}:
if len(packages) > 1:
log("found " + str(len(packages)) + " packages.", use_prefix=False)
else:
log("found '" + list(packages.values())[0].name + "'.",
use_prefix=False)
ignored_packages = get_ignored_packages(
release_directory=release_directory)
for k, v in dict(packages).items():
# Check for packages with upper case names
if v.name.lower() != v.name:
error(
"Cowardly refusing to release packages with uppercase characters in the name: "
+ v.name)
error("See:")
error(
" https://github.com/ros-infrastructure/bloom/issues/191")
error(
" https://github.com/ros-infrastructure/bloom/issues/76")
error("Invalid package names, aborting.", exit=True)
# Check for ignored packages
if v.name in ignored_packages:
warning(
"Explicitly ignoring package '{0}' because it is in the `{1}.ignored` file."
.format(v.name, os.environ.get('BLOOM_TRACK', 'packages')))
del packages[k]
if packages == {}:
error("All packages that were found were also ignored, aborting.",
exit=True)
version = verify_equal_package_versions(packages.values())
return [p.name for p in packages.values()], version, packages
# Otherwise we have a problem
log("failed.", use_prefix=False)
error("No package.xml(s) found, and '--package-name' not given, aborting.",
use_prefix=False,
exit=True)
def create_generators(generator_names):
generators = {}
for generator_name in generator_names:
generator = load_generator(generator_name)
if generator is not None:
generators[generator_name] = generator()
else:
warning("Failed to load generator: " + str(generator_name))
return generators
def create_generators(generator_names):
generators = {}
for generator_name in generator_names:
generator = load_generator(load_generator_module, generator_name)
if generator is not None:
generators[generator_name] = generator()
else:
warning("Failed to load generator: " + str(generator_name))
return generators
def metapackage_check(self, path, pkg):
if pkg.is_metapackage():
try:
metapackage.validate_metapackage(path, pkg)
except metapackage.InvalidMetapackage as e:
warning("Invalid metapackage:")
warning(" %s\n" % str(e))
error(fmt("Refusing to release invalid metapackage '@|%s@{rf}@!', metapackage requirements:\n @|%s" %
(pkg.name, metapackage.DEFINITION_URL)), exit=True)
def auto_upstream_checkout(upstream_repo, upstream_url, devel_branch):
info("Searching in upstream development branch for the name and version")
info(" Upstream url: " + upstream_url)
info(" Upstream type: " + upstream_repo.get_vcs_type_name())
if devel_branch:
info(" Upstream branch: " + str(devel_branch))
# Handle special svn cases
if upstream_repo.get_vcs_type_name() == 'svn':
if devel_branch == '':
upstream_url += '/trunk'
else:
upstream_url += '/branches/' + str(devel_branch)
devel_branch = ''
# Checkout to the upstream development branch
retcode = try_vcstools_checkout(upstream_repo, upstream_url, devel_branch)
if retcode != 0:
return retcode
# Look into the upstream devel branch for the version
meta = get_upstream_meta(upstream_repo.get_path())
if meta is None or None in meta.values():
error("Failed to get the upstream meta data.")
return 1
# Summarize the package.xml/stack.xml contents
info("Found upstream with version: " + ansi('boldon') + meta['version'] + \
ansi('reset'))
if meta['type'] == 'stack.xml':
info("Upstream contains a stack called: " + ansi('boldon') + \
meta['name'][0] + ansi('reset'))
else:
info("Upstream contains package" + \
('s: ' if len(meta['name']) > 1 else ': ') + ansi('boldon') + \
', '.join(meta['name']) + ansi('reset'))
# If svn recreate upstream_repo and checkout to the tag
if upstream_repo.get_vcs_type_name() == 'svn':
# Remove the /trunk from the url
upstream_url = '/'.join(upstream_url.split('/')[:-1])
upstream_dir = upstream_repo.get_path()
shutil.rmtree(upstream_dir) # Delete old upstream
upstream_repo = get_vcs_client('svn', upstream_dir)
checkout_url = upstream_url + '/tags/' + meta['version']
if not upstream_repo.checkout(checkout_url):
got_it = False
for name in meta['name']:
warning("Didn't find the tagged version at " + checkout_url)
checkout_url = upstream_url + '/tags/' + name + \
'-' + meta['version']
warning("Trying " + checkout_url)
if upstream_repo.checkout(checkout_url):
got_it = True
break
if not got_it:
error("Could not checkout upstream version")
return 1
# Return the meta data
return meta
def main(sysargs=None):
from bloom.config import upconvert_bloom_to_config_branch
upconvert_bloom_to_config_branch()
# Check that the current directory is a serviceable git/bloom repo
ensure_clean_working_env()
ensure_git_root()
# Get tracks
tracks_dict = get_tracks_dict_raw()
if not tracks_dict['tracks']:
error(
"No tracks configured, first create a track with "
"'git-bloom-config new <track_name>'",
exit=True)
# Do argparse stuff
parser = get_argument_parser([str(t) for t in tracks_dict['tracks']])
parser = add_global_arguments(parser)
args = parser.parse_args(sysargs)
handle_global_arguments(args)
os.environ['BLOOM_TRACK'] = args.track
verify_track(args.track, tracks_dict['tracks'][args.track])
git_clone = GitClone()
with git_clone:
quiet_git_clone_warning(True)
disable_git_clone(True)
execute_track(args.track,
tracks_dict['tracks'][args.track],
args.release_increment,
args.pretend,
args.debug,
args.unsafe,
interactive=args.interactive)
disable_git_clone(False)
quiet_git_clone_warning(False)
git_clone.commit()
# Notify the user of success and next action suggestions
info('\n\n', use_prefix=False)
warning("Tip: Check to ensure that the debian tags created have the same "
"version as the upstream version you are releasing.")
info(
fmt("@{gf}@!Everything went as expected, "
"you should check that the new tags match your expectations, and "
"then push to the release repo with:@|"))
info(
fmt(" git push --all && git push --tags "
"@{kf}@!# You might have to add --force to the second command if you "
"are over-writing existing tags"))
def __enter__(self):
if self.disabled:
return
current_branch = get_current_branch()
if current_branch is None:
warning("Could not determine current branch, changing to the bloom branch")
execute_command('git checkout bloom')
self.orig_cwd = os.getcwd()
os.chdir(self.clone_dir)
if self.track_all:
track_branches(directory=self.clone_dir)
return os.getcwd()
def mfa_prompt(oauth_config_path, username):
"""Explain how to create a token for users with Multi-Factor Authentication configured."""
warning("Receiving 401 when trying to create an oauth token can be caused by the user "
"having two-factor authentication enabled.")
warning("If 2FA is enabled, the user will have to create an oauth token manually.")
warning("A token can be created at https://github.com/settings/tokens")
warning("The resulting token can be placed in the '{oauth_config_path}' file as such:"
.format(**locals()))
info("")
warning('{{"github_user": "******", "oauth_token": "TOKEN_GOES_HERE"}}'
.format(**locals()))
info("")
def __enter__(self):
if self.disabled:
return
current_branch = get_current_branch()
if current_branch is None:
warning("Could not determine current branch, changing to the bloom branch")
execute_command('git checkout bloom')
self.orig_cwd = os.getcwd()
os.chdir(self.clone_dir)
if self.track_all:
track_branches(directory=self.clone_dir)
return os.getcwd()
def export_upstream(uri, tag, vcs_type, output_dir, show_uri, name):
tag = tag if tag != ":{none}" else None
output_dir = output_dir or os.getcwd()
if uri.startswith("git@"):
uri_is_path = False
else:
uri_parsed = urlparse(uri)
uri = uri if uri_parsed.scheme else uri_parsed.path
uri_is_path = False if uri_parsed.scheme else True
name = name or "upstream"
with temporary_directory() as tmp_dir:
info(
"Checking out repository at '{0}'".format(show_uri or uri)
+ (" to reference '{0}'.".format(tag) if tag else ".")
)
if uri_is_path:
upstream_repo = get_vcs_client(vcs_type, uri)
else:
repo_path = os.path.join(tmp_dir, "upstream")
upstream_repo = get_vcs_client(vcs_type, repo_path)
if not upstream_repo.checkout(uri, tag or ""):
error(
"Failed to clone repository at '{0}'".format(uri)
+ (" to reference '{0}'.".format(tag) if tag else "."),
exit=True,
)
if get_root() is not None and has_submodules(upstream_repo.get_path()):
error(
"""\
bloom does not support exporting git repositories with submodules, see:
- https://github.com/ros-infrastructure/bloom/issues/202
- https://github.com/ros-infrastructure/bloom/issues/217
- https://github.com/vcstools/vcstools/issues/84
""",
exit=True,
)
tarball_prefix = "{0}-{1}".format(name, tag) if tag else name
tarball_path = os.path.join(output_dir, tarball_prefix)
full_tarball_path = tarball_path + ".tar.gz"
info("Exporting to archive: '{0}'".format(full_tarball_path))
if not upstream_repo.export_repository(tag or "", tarball_path):
error("Failed to create archive of upstream repository at '{0}'".format(show_uri))
if tag and vcs_type == "git": # can only check for git repos
with change_directory(upstream_repo.get_path()):
if not tag_exists(tag):
warning("'{0}' is not a tag in the upstream repository...".format(tag))
if not branch_exists(tag):
warning("'{0}' is not a branch in the upstream repository...".format(tag))
if not os.path.exists(full_tarball_path):
error("Tarball was not created.", exit=True)
info("md5: {0}".format(calculate_file_md5(full_tarball_path)))
def update_track(track_dict):
for key, value in DEFAULT_TEMPLATE.iteritems():
if key in ['actions']:
if track_dict[key] != DEFAULT_TEMPLATE[key]:
warning("""\
Your track's '{0}' configuration is not the same as the default, should it be updated to the default setting?"""
.format(key))
if maybe_continue('n'):
track_dict[key] = DEFAULT_TEMPLATE[key]
elif key not in track_dict:
value = value.default if isinstance(value, PromptEntry) else value
track_dict[key] = value
return track_dict
def check_for_updates():
if sys.argv[0].endswith('bloom-update'):
return
user_bloom = os.path.join(os.path.expanduser('~'), '.bloom')
if os.path.exists(user_bloom):
with open(user_bloom, 'r') as f:
raw = f.read()
if not raw:
return
version_dict = json.loads(raw)
os.remove(user_bloom) # Remove only on successful parse
if type(version_dict) == dict and len(version_dict) == 2 and version_dict['current'] == bloom.__version__:
warning(UPDATE_MSG.format(**version_dict))
def update_track(track_dict):
for key, value in DEFAULT_TEMPLATE.iteritems():
if key in ['actions']:
if track_dict[key] != DEFAULT_TEMPLATE[key]:
warning("""\
Your track's '{0}' configuration is not the same as the default, should it be updated to the default setting?"""
.format(key))
if maybe_continue('n'):
track_dict[key] = DEFAULT_TEMPLATE[key]
elif key not in track_dict:
value = value.default if isinstance(value, PromptEntry) else value
track_dict[key] = value
return track_dict
def check_for_updates():
if sys.argv[0].endswith('bloom-update'):
return
user_bloom = os.path.join(os.path.expanduser('~'), '.bloom')
if os.path.exists(user_bloom):
with open(user_bloom, 'r') as f:
raw = f.read()
if not raw:
return
version_dict = json.loads(raw)
os.remove(user_bloom) # Remove only on successful parse
if type(version_dict) == dict and len(version_dict) == 2 and version_dict['current'] == bloom.__version__:
warning(UPDATE_MSG.format(**version_dict))
def version_check(version):
last_tag = get_last_tag_by_version()
if not last_tag:
return
last_tag_version = last_tag.split('/')[-1]
info(fmt("The latest upstream tag in the release repository is '@!{0}@|'."
.format(last_tag)))
# Ensure the new version is greater than the last tag
if parse_version(version) < parse_version(last_tag_version):
warning("""\
Version discrepancy:
The upstream version '{0}' isn't newer than upstream version '{1}'.
""".format(version, last_tag_version))
def execute_track(track, track_dict, release_inc, pretend=True, debug=False, fast=False):
info("Processing release track settings for '{0}'".format(track))
settings = process_track_settings(track_dict, release_inc)
# setup extra settings
archive_dir_path = tempfile.mkdtemp()
settings['archive_dir_path'] = archive_dir_path
if settings['release_tag'] != ':{none}':
archive_file = '{name}-{release_tag}.tar.gz'.format(**settings)
else:
archive_file = '{name}.tar.gz'.format(**settings)
settings['archive_path'] = os.path.join(archive_dir_path, archive_file)
# execute actions
info("", use_prefix=False)
info("Executing release track '{0}'".format(track))
for action in track_dict['actions']:
if 'bloom-export-upstream' in action and settings['vcs_type'] == 'tar':
warning("Explicitly skipping bloom-export-upstream for tar.")
settings['archive_path'] = settings['vcs_uri']
continue
templated_action = template_str(action, settings)
info(fmt("@{bf}@!==> @|@!" + sanitize(str(templated_action))))
if pretend:
continue
stdout = None
stderr = None
if bloom.util._quiet:
stdout = subprocess.PIPE
stderr = subprocess.STDOUT
if debug and 'DEBUG' not in os.environ:
os.environ['DEBUG'] = '1'
if fast and 'BLOOM_UNSAFE' not in os.environ:
os.environ['BLOOM_UNSAFE'] = '1'
templated_action = templated_action.split()
templated_action[0] = find_full_path(templated_action[0])
p = subprocess.Popen(templated_action, stdout=stdout, stderr=stderr,
shell=False, env=os.environ.copy())
out, err = p.communicate()
if bloom.util._quiet:
info(out, use_prefix=False)
ret = p.returncode
if ret > 0:
error(fmt(_error + "Error running command '@!{0}'@|")
.format(templated_action), exit=True)
info('', use_prefix=False)
if not pretend:
# Update the release_inc
tracks_dict = get_tracks_dict_raw()
tracks_dict['tracks'][track]['release_inc'] = settings['release_inc']
tracks_dict['tracks'][track]['last_version'] = settings['version']
write_tracks_dict_raw(tracks_dict,
'Updating release inc to: ' + str(settings['release_inc']))
def execute_track(track, track_dict, release_inc, pretend=True, debug=False, fast=False):
info("Processing release track settings for '{0}'".format(track))
settings = process_track_settings(track_dict, release_inc)
# setup extra settings
archive_dir_path = tempfile.mkdtemp()
settings['archive_dir_path'] = archive_dir_path
if settings['release_tag'] != ':{none}':
archive_file = '{name}-{release_tag}.tar.gz'.format(**settings)
else:
archive_file = '{name}.tar.gz'.format(**settings)
settings['archive_path'] = os.path.join(archive_dir_path, archive_file)
# execute actions
info("", use_prefix=False)
info("Executing release track '{0}'".format(track))
for action in track_dict['actions']:
if 'bloom-export-upstream' in action and settings['vcs_type'] == 'tar':
warning("Explicitly skipping bloom-export-upstream for tar.")
settings['archive_path'] = settings['vcs_uri']
continue
templated_action = template_str(action, settings)
info(fmt("@{bf}@!==> @|@!" + sanitize(str(templated_action))))
if pretend:
continue
stdout = None
stderr = None
if bloom.util._quiet:
stdout = subprocess.PIPE
stderr = subprocess.STDOUT
if debug and 'DEBUG' not in os.environ:
os.environ['DEBUG'] = '1'
if fast and 'BLOOM_UNSAFE' not in os.environ:
os.environ['BLOOM_UNSAFE'] = '1'
templated_action = templated_action.split()
templated_action[0] = find_full_path(templated_action[0])
p = subprocess.Popen(templated_action, stdout=stdout, stderr=stderr,
shell=False, env=os.environ.copy())
out, err = p.communicate()
if bloom.util._quiet:
info(out, use_prefix=False)
ret = p.returncode
if ret > 0:
error(fmt(_error + "Error running command '@!{0}'@|")
.format(templated_action), exit=True)
info('', use_prefix=False)
if not pretend:
# Update the release_inc
tracks_dict = get_tracks_dict_raw()
tracks_dict['tracks'][track]['release_inc'] = settings['release_inc']
tracks_dict['tracks'][track]['last_version'] = settings['version']
write_tracks_dict_raw(tracks_dict,
'Updating release inc to: ' + str(settings['release_inc']))
def export_upstream(uri, tag, vcs_type, output_dir, show_uri, name):
tag = tag if tag != ':{none}' else None
output_dir = output_dir or os.getcwd()
if uri.startswith('git@'):
uri_is_path = False
else:
uri_parsed = urlparse(uri)
uri = uri if uri_parsed.scheme else uri_parsed.path
uri_is_path = False if uri_parsed.scheme else True
name = name or 'upstream'
with temporary_directory() as tmp_dir:
info("Checking out repository at '{0}'".format(show_uri or uri) +
(" to reference '{0}'.".format(tag) if tag else '.'))
if uri_is_path:
upstream_repo = get_vcs_client(vcs_type, uri)
else:
repo_path = os.path.join(tmp_dir, 'upstream')
upstream_repo = get_vcs_client(vcs_type, repo_path)
if not upstream_repo.checkout(uri, tag or ''):
error("Failed to clone repository at '{0}'".format(uri) +
(" to reference '{0}'.".format(tag) if tag else '.'),
exit=True)
if get_root() is not None and has_submodules(upstream_repo.get_path()):
error("""\
bloom does not support exporting git repositories with submodules, see:
- https://github.com/ros-infrastructure/bloom/issues/202
- https://github.com/ros-infrastructure/bloom/issues/217
- https://github.com/vcstools/vcstools/issues/84
""",
exit=True)
tarball_prefix = '{0}-{1}'.format(name, tag) if tag else name
tarball_path = os.path.join(output_dir, tarball_prefix)
full_tarball_path = tarball_path + '.tar.gz'
info("Exporting to archive: '{0}'".format(full_tarball_path))
if not upstream_repo.export_repository(tag or '', tarball_path):
error("Failed to create archive of upstream repository at '{0}'".
format(show_uri))
if tag and vcs_type == 'git': # can only check for git repos
with change_directory(upstream_repo.get_path()):
if not tag_exists(tag):
warning(
"'{0}' is not a tag in the upstream repository...".
format(tag))
if not branch_exists(tag):
warning(
"'{0}' is not a branch in the upstream repository..."
.format(tag))
if not os.path.exists(full_tarball_path):
error("Tarball was not created.", exit=True)
info("md5: {0}".format(calculate_file_md5(full_tarball_path)))
def place_template_files(self, build_type, debian_dir='debian'):
# Create/Clean the debian folder
if os.path.exists(debian_dir):
if self.interactive:
warning("debian directory exists: " + debian_dir)
warning("Do you wish to overwrite it?")
if not maybe_continue('y'):
error("Answered no to continue, aborting.", exit=True)
elif 'BLOOM_CLEAR_DEBIAN_ON_GENERATION' in os.environ:
warning("Overwriting debian directory: " + debian_dir)
execute_command('git rm -rf ' + debian_dir)
execute_command(
'git commit -m "Clearing previous debian folder"')
if os.path.exists(debian_dir):
shutil.rmtree(debian_dir)
else:
warning("Not overwriting debian directory.")
# Use generic place template files command
place_template_files('.', build_type, gbp=True)
# Commit results
execute_command('git add ' + debian_dir)
_, has_files, _ = execute_command('git diff --cached --name-only',
return_io=True)
if has_files:
execute_command('git commit -m "Placing debian template files"')
def get_package_data(branch_name=None, directory=None, quiet=True, fuerte=False):
"""
Gets package data about the package(s) in the current branch.
:param branch_name: name of the branch you are searching on (log use only)
"""
log = debug if quiet else info
repo_dir = directory if directory else os.getcwd()
stack_path = os.path.join(repo_dir, 'stack.xml')
if os.path.exists(stack_path) and not fuerte:
warning("stack.xml is present but going to be ignored because this is not a release for Fuerte.")
if branch_name:
log("Looking for packages in '{0}' branch... ".format(branch_name), end='')
else:
log("Looking for packages in '{0}'... ".format(directory or os.getcwd()), end='')
## Check for package.xml(s)
if not fuerte:
packages = find_packages(repo_dir)
else:
packages = {}
if type(packages) == dict and packages != {}:
if len(packages) > 1:
log("found " + str(len(packages)) + " packages.",
use_prefix=False)
else:
log("found '" + packages.values()[0].name + "'.",
use_prefix=False)
version = verify_equal_package_versions(packages.values())
return [p.name for p in packages.values()], version, packages
## Check for stack.xml
has_rospkg = False
try:
import rospkg
has_rospkg = True
except ImportError:
log(ansi('redf') + "failed." + ansi('reset'), use_prefix=False)
warning("rospkg was not detected, stack.xml discovery is disabled",
file=sys.stderr)
if not has_rospkg:
error("no package.xml(s) found, and no name specified with "
"'--package-name', aborting.", use_prefix=False, exit=True)
if os.path.exists(stack_path):
log("found stack.xml.", use_prefix=False)
stack = rospkg.stack.parse_stack_file(stack_path)
return stack.name, stack.version, stack
# Otherwise we have a problem
log("failed.", use_prefix=False)
error("no package.xml(s) or stack.xml found, and no name "
"specified with '--package-name', aborting.",
use_prefix=False, exit=True)
def get_github_interface():
# First check to see if the oauth token is stored
oauth_config_path = os.path.join(os.path.expanduser('~'), '.config', 'bloom')
config = {}
if os.path.exists(oauth_config_path):
with open(oauth_config_path, 'r') as f:
config = json.loads(f.read())
token = config.get('oauth_token', None)
username = config.get('github_user', None)
if token and username:
return Github(username, auth=auth_header_from_oauth_token(token), token=token)
if not os.path.isdir(os.path.dirname(oauth_config_path)):
os.makedirs(os.path.dirname(oauth_config_path))
# Ok, now we have to ask for the user name and pass word
info("")
info("Looks like bloom doesn't have an oauth token for you yet.")
info("Therefore bloom will require your Github username and password just this once.")
info("With your Github username and password bloom will create an oauth token on your behalf.")
info("The token will be stored in `~/.config/bloom`.")
info("You can delete the token from that file to have a new token generated.")
info("Guard this token like a password, because it allows someone/something to act on your behalf.")
info("If you need to unauthorize it, remove it from the 'Applications' menu in your Github account page.")
info("")
token = None
while token is None:
try:
username = getpass.getuser()
username = safe_input("Github username [{0}]: ".format(username)) or username
password = getpass.getpass("Github password (never stored): ")
except (KeyboardInterrupt, EOFError):
return None
if not password:
error("No password was given, aborting.")
return None
gh = Github(username, auth=auth_header_from_basic_auth(username, password))
try:
token = gh.create_new_bloom_authorization(update_auth=True)
with open(oauth_config_path, 'a') as f:
config.update({'oauth_token': token, 'github_user': username})
f.write(json.dumps(config))
info("The token '{token}' was created and stored in the bloom config file: '{oauth_config_path}'"
.format(**locals()))
except GithubException as exc:
error("{0}".format(exc))
info("")
warning("This sometimes fails when the username or password are incorrect, try again?")
if not maybe_continue():
return None
return gh
def import_patches(directory=None):
# Get current branch
current_branch = get_current_branch(directory)
# Construct the patches branch name
patches_branch = 'patches/' + current_branch
# Ensure the patches branch exists and is tracked
if branch_exists(patches_branch, False, directory=directory):
if not branch_exists(patches_branch, True, directory=directory):
track_branches(patches_branch, directory)
else:
error("The patches branch ({0}) does not ".format(patches_branch) + \
"exist, did you use git-bloom-branch?")
return code.BRANCH_DOES_NOT_EXIST
# Create a swap space
tmp_dir = tempfile.mkdtemp()
try:
# Get parent branch and base commit from patches branch
config = get_patch_config(patches_branch, directory)
parent_branch, commit = config['parent'], config['base']
if commit != get_commit_hash(current_branch, directory):
warning("The current commit is not the same as the most recent "
"rebase commit. This might mean that you have committed "
"since the last time you did 'git-bloom-patch export'.")
return code.PATCHES_NOT_EXPORTED
# Checkout to the patches branch
checkout(patches_branch, directory=directory)
# Copy the patches to a temp location
patches = list_patches(directory)
if len(patches) == 0:
debug("No patches in the patches branch, nothing to do")
return code.NOTHING_TO_DO
tmp_dir_patches = []
for patch in patches:
tmp_dir_patches.append(os.path.join(tmp_dir, patch))
if directory is not None:
patch = os.path.join(directory, patch)
shutil.copy(patch, tmp_dir)
# Now checkout back to the original branch and import them
checkout(current_branch, directory=directory)
cmd = 'git am {0}*.patch'.format(tmp_dir + os.sep)
execute_command(cmd, cwd=directory)
# Notify the user
info("Applied {0} patches".format(len(patches)))
finally:
if current_branch:
checkout(current_branch, directory=directory)
if os.path.exists(tmp_dir):
shutil.rmtree(tmp_dir)
return 0
def post_patch(self, destination):
# Figure out the version of the given package
if self.name is not None:
warning("Cannot automatically tag the release because this is "
"not a catkin project. Please create a tag manually with:")
warning(" git tag -f release/" + str(self.name) + "/<version>")
return 0
with inbranch(destination):
package_data = get_package_data(destination)
if type(package_data) not in [list, tuple]:
return package_data
name, version, packages = package_data
# Execute git tag
execute_command('git tag -f ' + destination + '/' + version)
return 0
def handle_tree(tree, directory, root_path, version):
for path, kind in tree.iteritems():
if kind == 'directory':
# Path relative to start path
rel_path = os.path.join(directory, path)
# If it is a file, error
if os.path.isfile(rel_path):
error("In patches path '{0}' is a directory".format(rel_path) +
", but it exists in the upstream branch as a file.",
exit=True)
# If it is not already a directory, create it
if not os.path.isdir(rel_path):
info(" Createing directory... '{0}'".format(rel_path))
os.mkdir(rel_path)
# Recurse on the directory
handle_tree(ls_tree(BLOOM_CONFIG_BRANCH, os.path.join(root_path, rel_path)),
rel_path, root_path, version)
if kind == 'file':
# Path relative to start path
rel_path = os.path.join(directory, path)
# If the local version is a directory, error
if os.path.isdir(rel_path):
error("In patches path '{0}' is a file, ".format(rel_path) +
"but it exists in the upstream branch as a directory.",
exit=True)
# If the file already exists, warn
if os.path.isfile(rel_path):
warning(" File '{0}' already exists, overwriting..."
.format(rel_path))
execute_command('git rm {0}'.format(rel_path), shell=True)
# If package.xml tempalte in version, else grab data
if path in ['stack.xml']:
warning(" Skipping '{0}' templating, fuerte not supported"
.format(rel_path))
if path in ['package.xml']:
info(" Templating '{0}' into upstream branch..."
.format(rel_path))
file_data = show(BLOOM_CONFIG_BRANCH, os.path.join(root_path, rel_path))
file_data = file_data.replace(':{version}', version)
else:
info(" Overlaying '{0}' into upstream branch..."
.format(rel_path))
file_data = show(BLOOM_CONFIG_BRANCH, os.path.join(root_path, rel_path))
# Write file
with open(rel_path, 'wb') as f:
f.write(file_data)
# Add it with git
execute_command('git add {0}'.format(rel_path), shell=True)
def handle_arguments(self, args):
self.interactive = args.interactive
self.debian_inc = args.debian_inc
self.os_name = args.os_name
self.distros = args.distros
if self.distros in [None, []]:
index = rosdistro.get_index(rosdistro.get_index_url())
distribution_file = rosdistro.get_distribution_file(
index, self.rosdistro)
if self.os_name not in distribution_file.release_platforms:
if args.os_not_required:
warning(
"No platforms defined for os '{0}' in release file for the "
"'{1}' distro. This os was not required; continuing without error."
.format(self.os_name, self.rosdistro))
sys.exit(0)
error(
"No platforms defined for os '{0}' in release file for the '{1}' distro."
.format(self.os_name, self.rosdistro),
exit=True)
self.distros = distribution_file.release_platforms[self.os_name]
self.install_prefix = args.install_prefix
if args.install_prefix is None:
self.install_prefix = self.default_install_prefix
self.prefix = args.prefix
self.branches = match_branches_with_prefix(self.prefix,
get_branches,
prune=not args.match_all)
if len(self.branches) == 0:
error("No packages found, check your --prefix or --src arguments.",
exit=True)
self.packages = {}
self.tag_names = {}
self.names = []
self.branch_args = []
self.debian_branches = []
for branch in self.branches:
package = get_package_from_branch(branch)
if package is None:
# This is an ignored package
continue
self.packages[package.name] = package
self.names.append(package.name)
args = self.generate_branching_arguments(package, branch)
# First branch is debian/[<rosdistro>/]<package>
self.debian_branches.append(args[0][0])
self.branch_args.extend(args)
def mfa_prompt(oauth_config_path, username):
"""Explain how to create a token for users with Multi-Factor Authentication configured."""
warning(
"Receiving 401 when trying to create an oauth token can be caused by the user "
"having two-factor authentication enabled.")
warning(
"If 2FA is enabled, the user will have to create an oauth token manually."
)
warning("A token can be created at https://github.com/settings/tokens")
warning(
"The resulting token can be placed in the '{oauth_config_path}' file as such:"
.format(**locals()))
info("")
warning(
'{{"github_user": "******", "oauth_token": "TOKEN_GOES_HERE"}}'
.format(**locals()))
info("")
def get_package_data(branch_name=None, directory=None, quiet=True, release_directory=None):
"""
Gets package data about the package(s) in the current branch.
It also ignores the packages in the `packages.ignore` file in the master branch.
:param branch_name: name of the branch you are searching on (log use only)
"""
log = debug if quiet else info
repo_dir = directory or os.getcwd()
if branch_name:
log("Looking for packages in '{0}' branch... ".format(branch_name), end='')
else:
log("Looking for packages in '{0}'... ".format(directory or os.getcwd()), end='')
# Check for package.xml(s)
packages = find_packages(repo_dir)
if type(packages) == dict and packages != {}:
if len(packages) > 1:
log("found " + str(len(packages)) + " packages.",
use_prefix=False)
else:
log("found '" + list(packages.values())[0].name + "'.",
use_prefix=False)
version = verify_equal_package_versions(packages.values())
ignored_packages = get_ignored_packages(release_directory=release_directory)
for k, v in dict(packages).items():
# Check for packages with upper case names
if v.name.lower() != v.name:
error("Cowardly refusing to release packages with uppercase characters in the name: " + v.name)
error("See:")
error(" https://github.com/ros-infrastructure/bloom/issues/191")
error(" https://github.com/ros-infrastructure/bloom/issues/76")
error("Invalid package names, aborting.", exit=True)
# Check for ignored packages
if v.name in ignored_packages:
warning("Explicitly ignoring package '{0}' because it is in the `{1}.ignored` file."
.format(v.name, os.environ.get('BLOOM_TRACK', 'packages')))
del packages[k]
if packages == {}:
error("All packages that were found were also ignored, aborting.",
exit=True)
return [p.name for p in packages.values()], version, packages
# Otherwise we have a problem
log("failed.", use_prefix=False)
error("No package.xml(s) found, and '--package-name' not given, aborting.",
use_prefix=False, exit=True)
def handle_tree(tree, directory, root_path, version):
for path, kind in tree.iteritems():
if kind == "directory":
# Path relative to start path
rel_path = os.path.join(directory, path)
# If it is a file, error
if os.path.isfile(rel_path):
error(
"In patches path '{0}' is a directory".format(rel_path)
+ ", but it exists in the upstream branch as a file.",
exit=True,
)
# If it is not already a directory, create it
if not os.path.isdir(rel_path):
info(" Createing directory... '{0}'".format(rel_path))
os.mkdir(rel_path)
# Recurse on the directory
handle_tree(ls_tree("bloom", os.path.join(root_path, rel_path)), rel_path, root_path, version)
if kind == "file":
# Path relative to start path
rel_path = os.path.join(directory, path)
# If the local version is a directory, error
if os.path.isdir(rel_path):
error(
"In patches path '{0}' is a file, ".format(rel_path)
+ "but it exists in the upstream branch as a directory.",
exit=True,
)
# If the file already exists, warn
if os.path.isfile(rel_path):
warning(" File '{0}' already exists, overwriting...".format(rel_path))
execute_command("git rm {0}".format(rel_path), shell=True)
# If package.xml tempalte in version, else grab data
if path in ["package.xml", "stack.xml"]:
info(" Templating '{0}' into upstream branch...".format(rel_path))
file_data = show("bloom", os.path.join(root_path, rel_path))
file_data = file_data.replace(":{version}", version)
else:
info(" Overlaying '{0}' into upstream branch...".format(rel_path))
file_data = show("bloom", os.path.join(root_path, rel_path))
# Write file
with open(rel_path, "wb") as f:
f.write(file_data)
# Add it with git
execute_command("git add {0}".format(rel_path), shell=True)
def generate_ros_distro_diff(track, repository, distro, distro_file_url=ROS_DISTRO_FILE):
distro_file_url = distro_file_url.format(distro)
distro_file_raw = fetch_distro_file(distro_file_url)
distro_file = yaml.load(distro_file_raw)
with inbranch('upstream'):
# Check for package.xml(s)
try:
from catkin_pkg.packages import find_packages
except ImportError:
error("catkin_pkg was not detected, please install it.",
file=sys.stderr, exit=True)
packages = find_packages(os.getcwd())
if len(packages) == 0:
warning("No packages found, will not generate 'package: path' entries for rosdistro.")
track_dict = get_tracks_dict_raw()['tracks'][track]
last_version = track_dict['last_version']
release_inc = track_dict['release_inc']
distro_file['repositories'][repository]['version'] = '{0}-{1}'.format(last_version, release_inc)
if packages and (len(packages) > 1 or packages.keys()[0] != '.'):
distro_file['repositories'][repository]['packages'] = {}
for path, package in packages.iteritems():
distro_file['repositories'][repository]['packages'][package.name] = path
distro_file_name = distro_file_url.split('/')[-1]
# distro_dump_orig = yaml.dump(distro_file_orig, indent=2, default_flow_style=False)
distro_dump = yaml.dump(distro_file, indent=2, default_flow_style=False)
udiff = difflib.unified_diff(distro_file_raw.splitlines(), distro_dump.splitlines(),
fromfile=distro_file_name, tofile=distro_file_name)
if udiff:
info("Unified diff for the ROS distro file located at '{0}':".format(distro_file_url))
for line in udiff:
if line.startswith('@@'):
line = fmt('@{cf}' + line)
if line.startswith('+'):
if not line.startswith('+++'):
line += '\n'
line = fmt('@{gf}' + line)
if line.startswith('-'):
if not line.startswith('---'):
line += '\n'
line = fmt('@{rf}' + line)
if line.startswith(' '):
line += '\n'
info(line, use_prefix=False, end='')
else:
warning("This release resulted in no changes to the ROS distro file...")
def __init__(self, directory=None, track_all=True):
self.disabled = get_git_clone_state()
if self.disabled:
warning('Skipping transactional safety mechanism, be careful...')
return
self.tmp_dir = None
self.directory = directory if directory is not None else os.getcwd()
if get_root(directory) is None:
raise RuntimeError("Provided directory, '" + str(directory) +
"', is not a git repository")
self.track_all = track_all
if self.track_all:
track_branches(directory=directory)
self.current_branches = get_branches()
self.tmp_dir = tempfile.mkdtemp()
self.clone_dir = os.path.join(self.tmp_dir, 'clone')
self.repo_url = 'file://' + os.path.abspath(self.directory)
execute_command('git clone ' + self.repo_url + ' ' + self.clone_dir)
def handle_arguments(self, args):
self.interactive = args.interactive
self.debian_inc = args.debian_inc
self.os_name = args.os_name
self.distros = args.distros
if self.distros in [None, []]:
index = rosdistro.get_index(rosdistro.get_index_url())
distribution_file = rosdistro.get_distribution_file(index, self.rosdistro)
if self.os_name not in distribution_file.release_platforms:
if args.os_not_required:
warning("No platforms defined for os '{0}' in release file for the "
"'{1}' distro. This os was not required; continuing without error."
.format(self.os_name, self.rosdistro))
sys.exit(0)
error("No platforms defined for os '{0}' in release file for the '{1}' distro."
.format(self.os_name, self.rosdistro), exit=True)
self.distros = distribution_file.release_platforms[self.os_name]
self.install_prefix = args.install_prefix
if args.install_prefix is None:
self.install_prefix = self.default_install_prefix
self.prefix = args.prefix
self.branches = match_branches_with_prefix(self.prefix, get_branches, prune=not args.match_all)
if len(self.branches) == 0:
error(
"No packages found, check your --prefix or --src arguments.",
exit=True
)
self.packages = {}
self.tag_names = {}
self.names = []
self.branch_args = []
self.debian_branches = []
for branch in self.branches:
package = get_package_from_branch(branch)
if package is None:
# This is an ignored package
continue
self.packages[package.name] = package
self.names.append(package.name)
args = self.generate_branching_arguments(package, branch)
# First branch is debian/[<rosdistro>/]<package>
self.debian_branches.append(args[0][0])
self.branch_args.extend(args)
def get_package_data(branch_name, directory=None):
"""
Gets package data about the package(s) in the current branch.
:param branch_name: name of the branch you are searching on (log use only)
"""
debug("Looking for packages in '{0}'... ".format(branch_name), end='')
## Check for package.xml(s)
repo_dir = directory if directory else os.getcwd()
packages = find_packages(repo_dir)
if type(packages) == dict and packages != {}:
if len(packages) > 1:
debug("found " + str(len(packages)) + " packages.",
use_prefix=False)
else:
debug("found '" + packages.values()[0].name + "'.",
use_prefix=False)
version = verify_equal_package_versions(packages.values())
return [p.name for p in packages.values()], version, packages
## Check for stack.xml
has_rospkg = False
try:
import rospkg
has_rospkg = True
except ImportError:
debug(ansi('redf') + "failed." + ansi('reset'), use_prefix=False)
warning("rospkg was not detected, stack.xml discovery is disabled",
file=sys.stderr)
if not has_rospkg:
error("no package.xml(s) found, and no name specified with "
"'--package-name', aborting.", use_prefix=False)
return code.NO_PACKAGE_XML_FOUND
stack_path = os.path.join(repo_dir, 'stack.xml')
if os.path.exists(stack_path):
debug("found stack.xml.", use_prefix=False)
stack = rospkg.stack.parse_stack_file(stack_path)
return stack.name, stack.version, stack
# Otherwise we have a problem
debug("failed.", use_prefix=False)
error("no package.xml(s) or stack.xml found, and not name "
"specified with '--package-name', aborting.", use_prefix=False)
return code.NO_PACKAGE_XML_FOUND
def __init__(self, directory=None, track_all=True):
self.disabled = get_git_clone_state()
self.disabled_quiet = get_git_clone_state_quiet()
if self.disabled:
if not self.disabled_quiet:
warning("Skipping transactional safety mechanism, be careful...")
return
self.tmp_dir = None
self.directory = directory if directory is not None else os.getcwd()
if get_root(directory) is None:
raise RuntimeError("Provided directory, '" + str(directory) + "', is not a git repository")
self.track_all = track_all
if self.track_all:
track_branches(directory=directory)
self.current_branches = get_branches()
self.tmp_dir = tempfile.mkdtemp()
self.clone_dir = os.path.join(self.tmp_dir, "clone")
self.repo_url = "file://" + os.path.abspath(self.directory)
info(fmt("@!@{gf}+++@| Cloning working copy for safety"))
execute_command("git clone " + self.repo_url + " " + self.clone_dir)
def __init__(self, directory=None, track_all=True):
self.disabled = get_git_clone_state()
self.disabled_quiet = get_git_clone_state_quiet()
if self.disabled:
if not self.disabled_quiet:
warning('Skipping transactional safety mechanism, be careful...')
return
self.tmp_dir = None
self.directory = directory if directory is not None else os.getcwd()
if get_root(directory) is None:
raise RuntimeError("Provided directory, '" + str(directory) +
"', is not a git repository")
self.track_all = track_all
if self.track_all:
track_branches(directory=directory)
self.current_branches = get_branches()
self.tmp_dir = tempfile.mkdtemp()
self.clone_dir = os.path.join(self.tmp_dir, 'clone')
self.repo_url = 'file://' + os.path.abspath(self.directory)
info(fmt("@!@{gf}+++@| Cloning working copy for safety"))
execute_command('git clone ' + self.repo_url + ' ' + self.clone_dir)
def _set_trim_sub_dir(sub_dir, force, config, directory):
debug("_set_trim_sub_dir(" + str(sub_dir) + ", " + str(force) + ", " +
str(config) + ", " + str(directory) + ")")
if sub_dir is not None:
if config['trim'] != '' and config['trim'] != sub_dir:
warning("You are trying to set the trim sub directory to " +
sub_dir + ", but it is already set to " + config['trim'] +
".")
if not force:
warning("Changing the sud directory is not advised. "
"If you are sure you want to do this, use "
"'--force'")
return None
else:
warning("Forcing the change of the sub directory.")
# Make the sub_dir absolute
git_root = get_root(directory)
sub_dir_abs = os.path.join(git_root, sub_dir)
# Make sure it is a directory
if not os.path.isdir(sub_dir_abs):
error("The given sub directory, (" + sub_dir + ") does not "
"exist in the git repository at " + git_root)
return None
# Set the trim sub directory
config['trim'] = sub_dir
return config
def post_patch(self, destination, color='bluef'):
if destination in self.debian_branches:
return
# Tag after patches have been applied
with inbranch(destination):
# Tag
tag_name = self.tag_names[destination]
if tag_exists(tag_name):
if self.interactive:
warning("Tag exists: " + tag_name)
warning("Do you wish to overwrite it?")
if not maybe_continue('y'):
error("Answered no to continue, aborting.", exit=True)
else:
warning("Overwriting tag: " + tag_name)
else:
info("Creating tag: " + tag_name)
execute_command('git tag -f ' + tag_name)
# Report of success
name = destination.split('/')[-1]
package = self.packages[name]
distro = destination.split('/')[-2]
info(ansi(color) + "####" + ansi('reset'), use_prefix=False)
info(ansi(color) + "#### " + ansi('greenf') + "Successfully" +
ansi(color) + " generated '" + ansi('boldon') + distro +
ansi('boldoff') + "' debian for package"
" '" + ansi('boldon') + package.name + ansi('boldoff') + "'" +
" at version '" + ansi('boldon') + package.version + "-" +
str(self.debian_inc) + ansi('boldoff') + "'" + ansi('reset'),
use_prefix=False)
info(ansi(color) + "####\n" + ansi('reset'), use_prefix=False)
def export_upstream(uri, tag, vcs_type, output_dir, show_uri, name):
tag = tag if tag != ':{none}' else None
output_dir = output_dir or os.getcwd()
if uri.startswith('git@'):
uri_is_path = False
else:
uri_parsed = urlparse(uri)
uri = uri if uri_parsed.scheme else uri_parsed.path
uri_is_path = False if uri_parsed.scheme else True
name = name or 'upstream'
with temporary_directory() as tmp_dir:
info("Checking out repository at '{0}'".format(show_uri or uri) +
(" to reference '{0}'.".format(tag) if tag else '.'))
if uri_is_path:
upstream_repo = get_vcs_client(vcs_type, uri)
else:
repo_path = os.path.join(tmp_dir, 'upstream')
upstream_repo = get_vcs_client(vcs_type, repo_path)
if not upstream_repo.checkout(uri, tag or ''):
error("Failed to clone repository at '{0}'".format(uri) +
(" to reference '{0}'.".format(tag) if tag else '.'),
exit=True)
tarball_prefix = '{0}-{1}'.format(name, tag) if tag else name
tarball_path = os.path.join(output_dir, tarball_prefix)
full_tarball_path = tarball_path + '.tar.gz'
info("Exporting to archive: '{0}'".format(full_tarball_path))
if not upstream_repo.export_repository(tag or '', tarball_path):
error("Failed to create archive of upstream repository at '{0}'"
.format(show_uri))
if tag:
with change_directory(upstream_repo.get_path()):
if not tag_exists(tag):
warning("'{0}' is not a tag in the upstream repository..."
.format(tag))
if not branch_exists(tag):
warning("'{0}' is not a branch in the upstream repository..."
.format(tag))
if not os.path.exists(full_tarball_path):
error("Tarball was not created.", exit=True)
info("md5: {0}".format(calculate_file_md5(full_tarball_path)))
def post_patch(self, destination, color='bluef'):
# Tag after patches have been applied
with inbranch(destination):
# Tag
tag_name = self.tag_names[destination]
if tag_exists(tag_name):
if self.interactive:
warning("Tag exists: " + tag_name)
warning("Do you wish to overwrite it?")
if not maybe_continue('y'):
error("Answered no to continue, aborting.")
return code.ANSWERED_NO_TO_CONTINUE
else:
warning("Overwriting tag: " + tag_name)
else:
info("Creating tag: " + tag_name)
execute_command('git tag -f ' + tag_name)
# Report of success
name = destination.split('/')[-1]
stackage, kind = self.packages[name]
distro = destination.split('/')[-2]
info(ansi(color) + "####" + ansi('reset'), use_prefix=False)
info(
ansi(color) + "#### " + ansi('greenf') + "Successfully" + \
ansi(color) + " generated '" + ansi('boldon') + distro + \
ansi('boldoff') + "' debian for " + kind + \
" '" + ansi('boldon') + stackage.name + ansi('boldoff') + "'" + \
" at version '" + ansi('boldon') + stackage.version + \
"-" + str(self.debian_inc) + ansi('boldoff') + "'" + \
ansi('reset'),
use_prefix=False
)
info(ansi(color) + "####\n" + ansi('reset'), use_prefix=False)
def post_patch(self, destination, color='bluef'):
if destination in self.rpm_branches:
return
# Tag after patches have been applied
with inbranch(destination):
# Tag
tag_name = self.tag_names[destination]
if tag_exists(tag_name):
if self.interactive:
warning("Tag exists: " + tag_name)
warning("Do you wish to overwrite it?")
if not maybe_continue('y'):
error("Answered no to continue, aborting.", exit=True)
else:
warning("Overwriting tag: " + tag_name)
else:
info("Creating tag: " + tag_name)
execute_command('git tag -f ' + tag_name)
# Report of success
name = destination.split('/')[-1]
package = self.packages[name]
distro = destination.split('/')[-2]
info(ansi(color) + "####" + ansi('reset'), use_prefix=False)
info(
ansi(color) + "#### " + ansi('greenf') + "Successfully" +
ansi(color) + " generated '" + ansi('boldon') + distro +
ansi('boldoff') + "' RPM for package"
" '" + ansi('boldon') + package.name + ansi('boldoff') + "'" +
" at version '" + ansi('boldon') + package.version +
"-" + str(self.rpm_inc) + ansi('boldoff') + "'" +
ansi('reset'),
use_prefix=False
)
info(ansi(color) + "####\n" + ansi('reset'), use_prefix=False)
def post_patch(self, destination):
# Figure out the version of the given package
if self.name is not None:
warning("""\
Cannot automatically tag the release because this is not a catkin project.""")
warning("""\
Please checkout the release branch and then create a tag manually with:""")
warning(" git checkout release/" + str(self.name))
warning(" git tag -f release/" + str(self.name) + "/<version>")
return
with inbranch(destination):
name, version, packages = get_package_data(destination)
# Execute git tag
release_tag = destination + '/' + version + '-' + self.release_inc
execute_command('git tag ' + release_tag)
def ensure_clean_working_env(force=False, git_status=True, directory=None):
"""
Checks the environment to ensure it is clean, raises SystemExit otherwise.
Clean is defined as:
- In a git repository
- In a valid branch (force overrides)
- Does not have local changes (force overrides)
- Does not have untracked files (force overrides)
:param force: If True, overrides a few of the fail conditions
:param directory: directory in which to run this command
:raises: subprocess.CalledProcessError if any git calls fail
:raises: SystemExit if any git calls fail
"""
def ecwe_fail(code, show_git_status):
if not bloom.util._quiet and show_git_status:
info('\n++ git status:\n', use_prefix=False)
os.system('git status')
error(code, exit=True)
# Is it a git repo
if get_root(directory) is None:
error("Not in a valid git repository", exit=True)
# Are we on a branch?
current_branch = get_current_branch(directory)
if current_branch is None:
msg = "Could not determine current branch"
if not force:
return ecwe_fail(msg, git_status)
else:
warning(msg)
# Are there local changes?
if has_changes(directory):
msg = "Current git working branch has local changes"
if not force:
return ecwe_fail(msg, git_status)
else:
warning(msg)
# Are there untracked files or directories?
if has_untracked_files(directory):
msg = "Current git working branch has untracked files/directories"
if not force:
return ecwe_fail(msg, git_status)
else:
warning(msg)
def place_template_files(self, build_type, rpm_dir='rpm'):
# Create/Clean the rpm folder
if os.path.exists(rpm_dir):
if self.interactive:
warning("rpm directory exists: " + rpm_dir)
warning("Do you wish to overwrite it?")
if not maybe_continue('y'):
error("Answered no to continue, aborting.", exit=True)
else:
warning("Overwriting rpm directory: " + rpm_dir)
execute_command('git rm -rf ' + rpm_dir)
execute_command('git commit -m "Clearing previous rpm folder"')
if os.path.exists(rpm_dir):
shutil.rmtree(rpm_dir)
# Use generic place template files command
place_template_files('.', build_type, gbp=True)
# Commit results
execute_command('git add ' + rpm_dir)
execute_command('git commit -m "Placing rpm template files"')
def place_template_files(self, arch_dir='arch'):
# Create/Clean the arch folder
if os.path.exists(arch_dir):
if self.interactive:
warning("arch directory exists: " + arch_dir)
warning("Do you wish to overwrite it?")
if not maybe_continue('y'):
error("Answered no to continue, aborting.", exit=True)
else:
warning("Overwriting arch directory: " + arch_dir)
execute_command('git rm -rf ' + arch_dir)
execute_command('git commit -m "Clearing previous arch folder"')
if os.path.exists(arch_dir):
shutil.rmtree(arch_dir)
# Use generic place template files command
place_template_files('.')
# Commit results
execute_command('git add ' + arch_dir)
execute_command('git commit -m "Placing arch template files"')
def update_track(track_dict):
for key, value in DEFAULT_TEMPLATE.items():
if key in ['actions']:
if track_dict[key] != DEFAULT_TEMPLATE[key]:
warning(
"Your track's '{0}' configuration is not the same as the default."
.format(key))
default = 'n'
if key == 'actions' and track_dict[key] in ACTION_LIST_HISTORY:
default = 'y'
warning("Unless you have manually modified your 'actions' "
"(the commands which get run for a release), "
"you should update to the new default.")
warning("Should it be updated to the default setting?")
if maybe_continue(default):
track_dict[key] = DEFAULT_TEMPLATE[key]
elif key not in track_dict:
value = value.default if isinstance(value, PromptEntry) else value
track_dict[key] = value
return track_dict
def generate_substitutions_from_package(package,
os_name,
os_version,
ros_distro,
installation_prefix='/usr',
deb_inc=0,
peer_packages=None,
releaser_history=None,
fallback_resolver=None,
native=False):
peer_packages = peer_packages or []
data = {}
# Name, Version, Description
data['Name'] = package.name
data['Version'] = package.version
data['Description'] = format_description(package.description)
# Websites
websites = [str(url) for url in package.urls if url.type == 'website']
homepage = websites[0] if websites else ''
if homepage == '':
warning("No homepage set, defaulting to ''")
data['Homepage'] = homepage
# Debian Increment Number
data['DebianInc'] = '' if native else '-{0}'.format(deb_inc)
# Debian Package Format
data['format'] = 'native' if native else 'quilt'
# Package name
data['Package'] = sanitize_package_name(package.name)
# Installation prefix
data['InstallationPrefix'] = installation_prefix
# Resolve dependencies
depends = package.run_depends + package.buildtool_export_depends
build_depends = package.build_depends + package.buildtool_depends + package.test_depends
unresolved_keys = depends + build_depends + package.replaces + package.conflicts
# The installer key is not considered here, but it is checked when the keys are checked before this
resolved_deps = resolve_dependencies(
unresolved_keys, os_name, os_version, ros_distro,
peer_packages + [d.name for d in package.replaces + package.conflicts],
fallback_resolver)
data['Depends'] = sorted(set(format_depends(depends, resolved_deps)))
data['BuildDepends'] = sorted(
set(format_depends(build_depends, resolved_deps)))
data['Replaces'] = sorted(
set(format_depends(package.replaces, resolved_deps)))
data['Conflicts'] = sorted(
set(format_depends(package.conflicts, resolved_deps)))
# Set the distribution
data['Distribution'] = os_version
# Use the time stamp to set the date strings
stamp = datetime.datetime.now(tz.tzlocal())
data['Date'] = stamp.strftime('%a, %d %b %Y %T %z')
data['YYYY'] = stamp.strftime('%Y')
# Maintainers
maintainers = []
for m in package.maintainers:
maintainers.append(str(m))
data['Maintainer'] = maintainers[0]
data['Maintainers'] = ', '.join(maintainers)
# Changelog
changelogs = get_changelogs(package, releaser_history)
if changelogs and package.version not in [x[0] for x in changelogs]:
warning("")
warning(
"A CHANGELOG.rst was found, but no changelog for this version was found."
)
warning(
"You REALLY should have a entry (even a blank one) for each version of your package."
)
warning("")
if not changelogs:
# Ensure at least a minimal changelog
changelogs = []
if package.version not in [x[0] for x in changelogs]:
changelogs.insert(0, (
package.version, get_rfc_2822_date(datetime.datetime.now()),
' * Autogenerated, no changelog for this version found in CHANGELOG.rst.',
package.maintainers[0].name, package.maintainers[0].email))
bad_changelog = False
# Make sure that the first change log is the version being released
if package.version != changelogs[0][0]:
error("")
error("The version of the first changelog entry '{0}' is not the "
"same as the version being currently released '{1}'.".format(
package.version, changelogs[0][0]))
bad_changelog = True
# Make sure that the current version is the latest in the changelog
for changelog in changelogs:
if parse_version(package.version) < parse_version(changelog[0]):
error("")
error(
"There is at least one changelog entry, '{0}', which has a "
"newer version than the version of package '{1}' being released, '{2}'."
.format(changelog[0], package.name, package.version))
bad_changelog = True
if bad_changelog:
error("This is almost certainly by mistake, you should really take a "
"look at the changelogs for the package you are releasing.")
error("")
if not maybe_continue('n', 'Continue anyways'):
sys.exit("User quit.")
data['changelogs'] = changelogs
# Use debhelper version 7 for oneric, otherwise 9
data['debhelper_version'] = 7 if os_version in ['oneiric'] else 9
# Summarize dependencies
summarize_dependency_mapping(data, depends, build_depends, resolved_deps)
def convertToUnicode(obj):
if sys.version_info.major == 2:
if isinstance(obj, str):
return unicode(obj.decode('utf8'))
elif isinstance(obj, unicode):
return obj
else:
if isinstance(obj, bytes):
return str(obj.decode('utf8'))
elif isinstance(obj, str):
return obj
if isinstance(obj, list):
for i, val in enumerate(obj):
obj[i] = convertToUnicode(val)
return obj
elif isinstance(obj, type(None)):
return None
elif isinstance(obj, tuple):
obj_tmp = list(obj)
for i, val in enumerate(obj_tmp):
obj_tmp[i] = convertToUnicode(obj_tmp[i])
return tuple(obj_tmp)
elif isinstance(obj, int):
return obj
raise RuntimeError('need to deal with type %s' % (str(type(obj))))
for item in data.items():
data[item[0]] = convertToUnicode(item[1])
return data
def execute_track(track,
track_dict,
release_inc,
pretend=True,
debug=False,
fast=False):
info("Processing release track settings for '{0}'".format(track))
settings = process_track_settings(track_dict, release_inc)
# setup extra settings
archive_dir_path = tempfile.mkdtemp()
settings['archive_dir_path'] = archive_dir_path
if settings['release_tag'] != ':{none}':
archive_file = '{name}-{release_tag}.tar.gz'.format(**settings)
else:
archive_file = '{name}.tar.gz'.format(**settings)
settings['archive_path'] = os.path.join(archive_dir_path, archive_file)
# execute actions
info("", use_prefix=False)
info("Executing release track '{0}'".format(track))
for action in track_dict['actions']:
if 'bloom-export-upstream' in action and settings['vcs_type'] == 'tar':
warning("Explicitly skipping bloom-export-upstream for tar.")
settings['archive_path'] = settings['vcs_uri']
continue
templated_action = template_str(action, settings)
info(fmt("@{bf}@!==> @|@!" + sanitize(str(templated_action))))
if pretend:
continue
stdout = None
stderr = None
if bloom.util._quiet:
stdout = subprocess.PIPE
stderr = subprocess.STDOUT
if debug and 'DEBUG' not in os.environ:
os.environ['DEBUG'] = '1'
if fast and 'BLOOM_UNSAFE' not in os.environ:
os.environ['BLOOM_UNSAFE'] = '1'
templated_action = templated_action.split()
templated_action[0] = find_full_path(templated_action[0])
p = subprocess.Popen(templated_action,
stdout=stdout,
stderr=stderr,
shell=False,
env=os.environ.copy())
out, err = p.communicate()
if bloom.util._quiet:
info(out, use_prefix=False)
ret = p.returncode
if ret > 0:
if 'bloom-generate' in templated_action[
0] and ret == code.GENERATOR_NO_ROSDEP_KEY_FOR_DISTRO:
error(
fmt(_error +
"The following generator action reported that it is missing one or more"
))
error(
fmt(" @|rosdep keys, but that the key exists in other platforms:"
))
error(fmt("@|'@!{0}'@|").format(templated_action))
info('', use_prefix=False)
error(
fmt("@|If you are @!@_@{rf}absolutely@| sure that this key is unavailable for the platform in"
))
error(
fmt("@|question, the generator can be skipped and you can proceed with the release."
))
if maybe_continue(
'n',
'Skip generator action and continue with release'):
info("\nAction skipped, continuing with release.\n")
continue
info('', use_prefix=False)
error(fmt(_error + "Error running command '@!{0}'@|").format(
templated_action),
exit=True)
info('', use_prefix=False)
if not pretend:
# Update the release_inc
tracks_dict = get_tracks_dict_raw()
tracks_dict['tracks'][track]['release_inc'] = settings['release_inc']
tracks_dict['tracks'][track]['last_version'] = settings['version']
# if release tag is set to ask and a custom value is used
if settings['version'] != settings['release_tag']:
tracks_dict['tracks'][track]['last_release'] = settings[
'release_tag']
write_tracks_dict_raw(
tracks_dict,
'Updating release inc to: ' + str(settings['release_inc']))
