def generate_ros_distro_diff(track, repository, distro):
distribution_dict = get_distribution_file(distro).get_data()
# Get packages
packages = get_packages()
if len(packages) == 0:
warning("No packages found, will not generate 'package: path' entries for rosdistro.")
# Get version
track_dict = get_tracks_dict_raw()['tracks'][track]
last_version = track_dict['last_version']
release_inc = track_dict['release_inc']
version = '{0}-{1}'.format(last_version, release_inc)
# Create a repository if there isn't already one
if repository not in distribution_dict['repositories']:
global _user_provided_release_url
distribution_dict['repositories'][repository] = {}
# Create a release entry if there isn't already one
if 'release' not in distribution_dict['repositories'][repository]:
distribution_dict['repositories'][repository]['release'] = {
'url': _user_provided_release_url
}
# Update the repository
repo = distribution_dict['repositories'][repository]['release']
if 'tags' not in repo:
repo['tags'] = {}
repo['tags']['release'] = generate_release_tag(distro)
repo['version'] = version
if 'packages' not in repo:
repo['packages'] = []
for path, pkg in packages.items():
if pkg.name not in repo['packages']:
repo['packages'].append(pkg.name)
# Remove any missing packages
packages_being_released = [p.name for p in packages.values()]
for pkg_name in list(repo['packages']):
if pkg_name not in packages_being_released:
repo['packages'].remove(pkg_name)
repo['packages'].sort()
# Do the diff
distro_file_name = get_relative_distribution_file_path(distro)
updated_distribution_file = rosdistro.DistributionFile(distro, distribution_dict)
distro_dump = yaml_from_distribution_file(updated_distribution_file)
distro_file_raw = load_url_to_file_handle(get_disitrbution_file_url(distro)).read()
if distro_file_raw != distro_dump:
# Calculate the diff
udiff = difflib.unified_diff(distro_file_raw.splitlines(), distro_dump.splitlines(),
fromfile=distro_file_name, tofile=distro_file_name)
temp_dir = tempfile.mkdtemp()
udiff_file = os.path.join(temp_dir, repository + '-' + version + '.patch')
udiff_raw = ''
info("Unified diff for the ROS distro file located at '{0}':".format(udiff_file))
for line in udiff:
if line.startswith('@@'):
udiff_raw += line
line = fmt('@{cf}' + sanitize(line))
if line.startswith('+'):
if not line.startswith('+++'):
line += '\n'
udiff_raw += line
line = fmt('@{gf}' + sanitize(line))
if line.startswith('-'):
if not line.startswith('---'):
line += '\n'
udiff_raw += line
line = fmt('@{rf}' + sanitize(line))
if line.startswith(' '):
line += '\n'
udiff_raw += line
info(line, use_prefix=False, end='')
# Assert that only this repository is being changed
distro_file_yaml = yaml.load(distro_file_raw)
distro_yaml = yaml.load(distro_dump)
if 'repositories' in distro_file_yaml:
distro_file_repos = distro_file_yaml['repositories']
for repo in distro_yaml['repositories']:
if repo == repository:
continue
if repo not in distro_file_repos or distro_file_repos[repo] != distro_yaml['repositories'][repo]:
error("This generated pull request modifies a repository entry other than the one being released.")
error("This likely occured because the upstream rosdistro changed during this release.")
error("This pull request will abort, please re-run this command with the -p option to try again.",
exit=True)
# Write the diff out to file
with open(udiff_file, 'w+') as f:
f.write(udiff_raw)
# Return the diff
return updated_distribution_file
else:
warning("This release resulted in no changes to the ROS distro file...")
return None
def import_upstream(tarball_path, patches_path, version, name, replace):
# Check for a url and download it
url = urlparse(tarball_path)
if url.scheme: # Some scheme like http, https, or file...
tmp_dir = tempfile.mkdtemp()
try:
info("Fetching file from url: '{0}'".format(tarball_path))
req = load_url_to_file_handle(tarball_path)
tarball_path = os.path.join(tmp_dir, os.path.basename(url.path))
with open(tarball_path, 'wb') as f:
chunk_size = 16 * 1024
while True:
chunk = req.read(chunk_size)
if not chunk:
break
f.write(chunk)
return import_upstream(tarball_path, patches_path, version, name, replace)
finally:
shutil.rmtree(tmp_dir)
# If there is not tarball at the given path, fail
if not os.path.exists(tarball_path):
error("Specified archive does not exists: '{0}'".format(tarball_path),
exit=True)
# If either version or name are not provided, guess from archive name
if not version or not name:
# Parse tarball name
tarball_file = os.path.basename(tarball_path)
ending = None
if tarball_file.endswith('.tar.gz'):
ending = '.tar.gz'
elif tarball_file.endswith('.zip'):
ending = '.zip'
else:
error("Cannot detect type of archive: '{0}'"
.format(tarball_file), exit=True)
tarball_file = tarball_file[:-len(ending)]
split_tarball_file = tarball_file.split('-')
if len(split_tarball_file) < 2 and not version or len(split_tarball_file) < 1:
error("Cannot detect name and/or version from archive: '{0}'"
.format(tarball_file), exit=True)
if not name and len(split_tarball_file) == 1:
name = split_tarball_file[0]
elif not name and len(split_tarball_file) == 1:
name = '-'.join(split_tarball_file[:-1])
if not version and len(split_tarball_file) < 2:
error("Cannot detect version from archive: '{0}'"
.format(tarball_file) + " and the version was not spcified.",
exit=True)
version = version if version else split_tarball_file[-1]
# Check if the patches_path (if given) exists
patches_path_dict = None
if patches_path:
patches_path_dict = ls_tree(BLOOM_CONFIG_BRANCH, patches_path)
if not patches_path_dict:
error("Given patches path '{0}' does not exist in bloom branch."
.format(patches_path), exit=True)
# Do version checking
version_check(version)
# Check for existing tags
upstream_tag = 'upstream/{0}'.format(version)
if tag_exists(upstream_tag):
if not replace:
error("Tag '{0}' already exists, use --replace to override it."
.format(upstream_tag), exit=True)
warning("Removing tag: '{0}'".format(upstream_tag))
delete_tag(upstream_tag)
if not get_git_clone_state():
delete_remote_tag(upstream_tag)
name_tag = '{0}/{1}'.format(name or 'upstream', version)
if name_tag != upstream_tag and tag_exists(name_tag):
if not replace:
error("Tag '{0}' already exists, use --replace to override it."
.format(name_tag), exit=True)
warning("Removing tag: '{0}'".format(name_tag))
delete_tag(name_tag)
if not get_git_clone_state():
delete_remote_tag(name_tag)
# If there is not upstream branch, create one
if not branch_exists('upstream'):
info("Creating upstream branch.")
create_branch('upstream', orphaned=True)
else:
track_branches(['upstream'])
# Import the given tarball
info("Importing archive into upstream branch...")
import_tarball(tarball_path, 'upstream', version, name)
# Handle patches_path
if patches_path:
import_patches(patches_path, patches_path_dict, 'upstream', version)
# Create tags
with inbranch('upstream'):
info("Creating tag: '{0}'".format(upstream_tag))
create_tag(upstream_tag)
if name_tag != upstream_tag:
info("Creating tag: '{0}'".format(name_tag))
create_tag(name_tag)
def generate_ros_distro_diff(track, repository, distro):
release_dict = get_release_file(distro).get_data()
# Get packages
packages = get_packages()
if len(packages) == 0:
warning("No packages found, will not generate 'package: path' entries for rosdistro.")
# Get version
track_dict = get_tracks_dict_raw()["tracks"][track]
last_version = track_dict["last_version"]
release_inc = track_dict["release_inc"]
version = "{0}-{1}".format(last_version, release_inc)
# Create a repository if there isn't already one
if repository not in release_dict["repositories"]:
global _user_provided_release_url
release_dict["repositories"][repository] = {"url": _user_provided_release_url}
# Update the repository
repo = release_dict["repositories"][repository]
if "tags" not in repo:
repo["tags"] = {}
repo["tags"]["release"] = "release/%s/{package}/{version}" % distro
repo["version"] = version
if "packages" not in repo:
repo["packages"] = {}
for path, pkg in packages.items():
if pkg.name not in repo["packages"]:
repo["packages"][pkg.name] = {}
repo["packages"][pkg.name]["subfolder"] = path # This will be shortened
# Remove any missing packages
for pkg_name in dict(repo["packages"]):
if pkg_name not in [p.name for p in packages.values()]:
if pkg_name in repo["packages"]:
del repo["packages"][pkg_name]
# Do the diff
distro_file_name = get_relative_release_file_path(distro)
updated_release_file = rosdistro.ReleaseFile("distro", release_dict)
distro_dump = yaml_from_release_file(updated_release_file)
distro_file_raw = load_url_to_file_handle(get_release_file_url(distro)).read()
if distro_file_raw != distro_dump:
udiff = difflib.unified_diff(
distro_file_raw.splitlines(), distro_dump.splitlines(), fromfile=distro_file_name, tofile=distro_file_name
)
temp_dir = tempfile.mkdtemp()
udiff_file = os.path.join(temp_dir, repository + "-" + version + ".patch")
udiff_raw = ""
info("Unified diff for the ROS distro file located at '{0}':".format(udiff_file))
for line in udiff:
if line.startswith("@@"):
udiff_raw += line
line = fmt("@{cf}" + sanitize(line))
if line.startswith("+"):
if not line.startswith("+++"):
line += "\n"
udiff_raw += line
line = fmt("@{gf}" + sanitize(line))
if line.startswith("-"):
if not line.startswith("---"):
line += "\n"
udiff_raw += line
line = fmt("@{rf}" + sanitize(line))
if line.startswith(" "):
line += "\n"
udiff_raw += line
info(line, use_prefix=False, end="")
with open(udiff_file, "w+") as f:
f.write(udiff_raw)
return updated_release_file
else:
warning("This release resulted in no changes to the ROS distro file...")
return None
def generate_ros_distro_diff(track, repository, distro):
distribution_dict = get_distribution_file(distro).get_data()
# Get packages
packages = get_packages()
if len(packages) == 0:
warning("No packages found, will not generate 'package: path' entries for rosdistro.")
# Get version
track_dict = get_tracks_dict_raw()['tracks'][track]
last_version = track_dict['last_version']
release_inc = track_dict['release_inc']
version = '{0}-{1}'.format(last_version, release_inc).encode('utf-8')
# Create a repository if there isn't already one
if repository not in distribution_dict['repositories']:
global _user_provided_release_url
distribution_dict['repositories'][repository] = {}
# Create a release entry if there isn't already one
if 'release' not in distribution_dict['repositories'][repository]:
distribution_dict['repositories'][repository]['release'.encode('utf-8')] = {
'url'.encode('utf-8'): _user_provided_release_url
}
# Update the repository
repo = distribution_dict['repositories'][repository]['release']
if 'tags' not in repo:
repo['tags'.encode('utf-8')] = {}
repo['tags']['release'.encode('utf-8')] = generate_release_tag(distro)
repo['version'.encode('utf-8')] = version
if 'packages' not in repo:
repo['packages'.encode('utf-8')] = []
for path, pkg in packages.items():
if pkg.name not in repo['packages']:
repo['packages'].append(pkg.name)
# Remove any missing packages
packages_being_released = [p.name for p in packages.values()]
for pkg_name in list(repo['packages']):
if pkg_name not in packages_being_released:
repo['packages'].remove(pkg_name)
repo['packages'].sort()
def get_repository_info_from_user():
data = {}
while True:
vcs_type = safe_input('VCS type [git, svn, hg, bzr]: ')
if vcs_type in ['git', 'svn', 'hg', 'bzr']:
break
error("'{0}' is not a valid vcs type.".format(vcs_type))
if not maybe_continue(msg='Try again'):
return {}
data['type'] = vcs_type
while True:
url = safe_input('VCS url: ')
if url:
break
error("Nothing entered for url.")
if not maybe_continue(msg='Try again'):
return {}
data['url'] = url
while True:
version = safe_input('VCS version [commit, tag, branch, etc]: ')
if version:
break
error("Nothing entered for version.")
if not maybe_continue(msg='Try again'):
return {}
data['version'] = version
return data
# Ask for doc entry
if 'BLOOM_DONT_ASK_FOR_DOCS' not in os.environ:
docs = distribution_dict['repositories'][repository].get('doc', {})
if not docs and maybe_continue(msg='Would you like to add documentation information for this repository?'):
info("Please enter your repository information for the doc generation job.")
info("This information should point to the repository from which documentation should be generated.")
docs = get_repository_info_from_user()
distribution_dict['repositories'][repository]['doc'] = docs
# Ask for source entry
if 'BLOOM_DONT_ASK_FOR_SOURCE' not in os.environ:
source = distribution_dict['repositories'][repository].get('source', {})
if not source and maybe_continue(msg='Would you like to add source information for this repository?'):
info("Please enter information which points ot the active development branch for this repository.")
info("This information is used to run continuous integration jobs and for developers to checkout from.")
source = get_repository_info_from_user()
distribution_dict['repositories'][repository]['source'] = source
# Ask for maintainership information
if 'BLOOM_DONT_ASK_FOR_MAINTENANCE_STATUS' not in os.environ:
status = distribution_dict['repositories'][repository].get('status', None)
description = distribution_dict['repositories'][repository].get('status_description', None)
if status is None and maybe_continue(msg='Would you like to add a maintenance status for this repository?'):
info("Please enter a maintenance status.")
info("Valid maintenance statuses:")
info("- developed: active development is in progress")
info("- maintained: no new development, but bug fixes and pull requests are addressed")
info("- end-of-life: should not be used, will disapear at some point")
while True:
status = safe_input('Status: ')
if status in ['developed', 'maintained', 'end-of-life']:
break
error("'{0}' is not a valid status.".format(status))
if not maybe_continue(msg='Try again'):
status = None
break
if status is not None:
info("You can also enter a status description.")
info("This is usually reserved for giving a reason when a status is 'end-of-life'.")
if description is not None:
info("Current status description: {0}".format(description))
description_in = safe_input('Status Description [press Enter for no change]: ')
if description_in:
description = description_in
if status is not None:
distribution_dict['repositories'][repository]['status'] = status
if description is not None:
distribution_dict['repositories'][repository]['status_description'] = description
# Do the diff
distro_file_name = get_relative_distribution_file_path(distro)
updated_distribution_file = rosdistro.DistributionFile(distro, distribution_dict)
distro_dump = yaml_from_distribution_file(updated_distribution_file)
distro_file_raw = load_url_to_file_handle(get_disitrbution_file_url(distro)).read()
if distro_file_raw != distro_dump:
# Calculate the diff
udiff = difflib.unified_diff(distro_file_raw.splitlines(), distro_dump.splitlines(),
fromfile=distro_file_name, tofile=distro_file_name)
temp_dir = tempfile.mkdtemp()
udiff_file = os.path.join(temp_dir, repository + '-' + version + '.patch')
udiff_raw = ''
info("Unified diff for the ROS distro file located at '{0}':".format(udiff_file))
for line in udiff:
if line.startswith('@@'):
udiff_raw += line
line = fmt('@{cf}' + sanitize(line))
if line.startswith('+'):
if not line.startswith('+++'):
line += '\n'
udiff_raw += line
line = fmt('@{gf}' + sanitize(line))
if line.startswith('-'):
if not line.startswith('---'):
line += '\n'
udiff_raw += line
line = fmt('@{rf}' + sanitize(line))
if line.startswith(' '):
line += '\n'
udiff_raw += line
info(line, use_prefix=False, end='')
# Assert that only this repository is being changed
distro_file_yaml = yaml.load(distro_file_raw)
distro_yaml = yaml.load(distro_dump)
if 'repositories' in distro_file_yaml:
distro_file_repos = distro_file_yaml['repositories']
for repo in distro_yaml['repositories']:
if repo == repository:
continue
if repo not in distro_file_repos or distro_file_repos[repo] != distro_yaml['repositories'][repo]:
error("This generated pull request modifies a repository entry other than the one being released.")
error("This likely occured because the upstream rosdistro changed during this release.")
error("This pull request will abort, please re-run this command with the -p option to try again.",
exit=True)
# Write the diff out to file
with open(udiff_file, 'w+') as f:
f.write(udiff_raw)
# Return the diff
return updated_distribution_file
else:
warning("This release resulted in no changes to the ROS distro file...")
return None
def generate_ros_distro_diff(track, repository, distro):
distribution_dict = get_distribution_file(distro).get_data()
# Get packages
packages = get_packages()
if len(packages) == 0:
warning("No packages found, will not generate 'package: path' entries for rosdistro.")
# Get version
track_dict = get_tracks_dict_raw()['tracks'][track]
last_version = track_dict['last_version']
release_inc = track_dict['release_inc']
version = '{0}-{1}'.format(last_version, release_inc)
# Create a repository if there isn't already one
if repository not in distribution_dict['repositories']:
global _user_provided_release_url
distribution_dict['repositories'][repository] = {}
# Create a release entry if there isn't already one
if 'release' not in distribution_dict['repositories'][repository]:
distribution_dict['repositories'][repository]['release'] = {
'url': _user_provided_release_url
}
# Update the repository
repo = distribution_dict['repositories'][repository]['release']
if 'tags' not in repo:
repo['tags'] = {}
repo['tags']['release'] = 'release/%s/{package}/{version}' % distro
repo['version'] = version
if 'packages' not in repo:
repo['packages'] = []
for path, pkg in packages.items():
if pkg.name not in repo['packages']:
repo['packages'].append(pkg.name)
# Remove any missing packages
packages_being_released = [p.name for p in packages.values()]
for pkg_name in list(repo['packages']):
if pkg_name not in packages_being_released:
repo['packages'].remove(pkg_name)
repo['packages'].sort()
# Do the diff
distro_file_name = get_relative_distribution_file_path(distro)
updated_distribution_file = rosdistro.DistributionFile(distro, distribution_dict)
distro_dump = yaml_from_distribution_file(updated_distribution_file)
distro_file_raw = load_url_to_file_handle(get_disitrbution_file_url(distro)).read()
if distro_file_raw != distro_dump:
udiff = difflib.unified_diff(distro_file_raw.splitlines(), distro_dump.splitlines(),
fromfile=distro_file_name, tofile=distro_file_name)
temp_dir = tempfile.mkdtemp()
udiff_file = os.path.join(temp_dir, repository + '-' + version + '.patch')
udiff_raw = ''
info("Unified diff for the ROS distro file located at '{0}':".format(udiff_file))
for line in udiff:
if line.startswith('@@'):
udiff_raw += line
line = fmt('@{cf}' + sanitize(line))
if line.startswith('+'):
if not line.startswith('+++'):
line += '\n'
udiff_raw += line
line = fmt('@{gf}' + sanitize(line))
if line.startswith('-'):
if not line.startswith('---'):
line += '\n'
udiff_raw += line
line = fmt('@{rf}' + sanitize(line))
if line.startswith(' '):
line += '\n'
udiff_raw += line
info(line, use_prefix=False, end='')
with open(udiff_file, 'w+') as f:
f.write(udiff_raw)
return updated_distribution_file
else:
warning("This release resulted in no changes to the ROS distro file...")
return None
def generate_ros_distro_diff(track, repository, distro):
release_dict = get_release_file(distro).get_data()
# Get packages
packages = get_packages()
if len(packages) == 0:
warning(
"No packages found, will not generate 'package: path' entries for rosdistro."
)
# Get version
track_dict = get_tracks_dict_raw()['tracks'][track]
last_version = track_dict['last_version']
release_inc = track_dict['release_inc']
version = '{0}-{1}'.format(last_version, release_inc)
# Create a repository if there isn't already one
if repository not in release_dict['repositories']:
global _user_provided_release_url
release_dict['repositories'][repository] = {
'url': _user_provided_release_url
}
# Update the repository
repo = release_dict['repositories'][repository]
if 'tags' not in repo:
repo['tags'] = {}
repo['tags']['release'] = 'release/%s/{package}/{version}' % distro
repo['version'] = version
if 'packages' not in repo:
repo['packages'] = {}
for path, pkg in packages.items():
if pkg.name not in repo['packages']:
repo['packages'][pkg.name] = {}
repo['packages'][
pkg.name]['subfolder'] = path # This will be shortened
# Remove any missing packages
for pkg_name in dict(repo['packages']):
if pkg_name not in [p.name for p in packages.values()]:
if pkg_name in repo['packages']:
del repo['packages'][pkg_name]
# Do the diff
distro_file_name = get_relative_release_file_path(distro)
updated_release_file = rosdistro.ReleaseFile('distro', release_dict)
distro_dump = yaml_from_release_file(updated_release_file)
distro_file_raw = load_url_to_file_handle(
get_release_file_url(distro)).read()
if distro_file_raw != distro_dump:
udiff = difflib.unified_diff(distro_file_raw.splitlines(),
distro_dump.splitlines(),
fromfile=distro_file_name,
tofile=distro_file_name)
temp_dir = tempfile.mkdtemp()
udiff_file = os.path.join(temp_dir,
repository + '-' + version + '.patch')
udiff_raw = ''
info("Unified diff for the ROS distro file located at '{0}':".format(
udiff_file))
for line in udiff:
if line.startswith('@@'):
udiff_raw += line
line = fmt('@{cf}' + sanitize(line))
if line.startswith('+'):
if not line.startswith('+++'):
line += '\n'
udiff_raw += line
line = fmt('@{gf}' + sanitize(line))
if line.startswith('-'):
if not line.startswith('---'):
line += '\n'
udiff_raw += line
line = fmt('@{rf}' + sanitize(line))
if line.startswith(' '):
line += '\n'
udiff_raw += line
info(line, use_prefix=False, end='')
with open(udiff_file, 'w+') as f:
f.write(udiff_raw)
return updated_release_file
else:
warning(
"This release resulted in no changes to the ROS distro file...")
return None
