def test_verify_files_parsable():
url = 'file://' + FILES_DIR + '/index.yaml'
index = get_index(url)
release_file = get_release_file(index, 'foo')
data = yaml_from_release_file(release_file)
with open(os.path.join(FILES_DIR, 'foo', 'release.yaml'), 'r') as f:
expected = f.read()
assert data == expected, get_diff(expected, data)
def open_pull_request(track, repository, distro):
# Get the diff
release_file = get_release_file(distro)
if repository in release_file.repositories:
orig_version = release_file.repositories[repository].version
else:
orig_version = None
updated_release_file = generate_ros_distro_diff(track, repository, distro)
if updated_release_file is None:
# There were no changes, no pull request required
return None
version = updated_release_file.repositories[repository].version
updated_distro_file = yaml_from_release_file(updated_release_file)
# Determine if the distro file is hosted on github...
gh_org, gh_repo, gh_branch, gh_path = get_gh_info(get_release_file_url(distro))
if None in [gh_org, gh_repo, gh_branch, gh_path]:
warning("Automated pull request only available via github.com")
return
# Get the github user name
gh_username = None
bloom_user_path = os.path.join(os.path.expanduser("~"), ".bloom_user")
if os.path.exists(bloom_user_path):
with open(bloom_user_path, "r") as f:
gh_username = f.read().strip()
gh_username = gh_username or getpass.getuser()
response = raw_input("github user name [{0}]: ".format(gh_username))
if response:
gh_username = response
info("Would you like bloom to store your github user name (~/.bloom_user)?")
if maybe_continue():
with open(bloom_user_path, "w") as f:
f.write(gh_username)
else:
with open(bloom_user_path, "w") as f:
f.write(" ")
warning("If you want to have bloom store it in the future remove the ~/.bloom_user file.")
# Get the github password
gh_password = getpass.getpass("github password (This is not stored):")
if not gh_password or not gh_username:
error("Either the github username or github password is not set.")
warning("Skipping the pull request...")
return
# Check for fork
info(fmt("@{bf}@!==> @|@!Checking for rosdistro fork on github..."))
gh_user_repos = fetch_github_api("https://api.github.com/users/{0}/repos".format(gh_username), use_pagination=True)
if gh_user_repos is None:
error("Failed to get a list of repositories for user: '******'".format(gh_username))
warning("Skipping the pull request...")
return
if "rosdistro" not in [x["name"] for x in gh_user_repos if "name" in x]:
warning(
"Github user '{0}' does not have a fork ".format(gh_username)
+ "of the {0}:{1} repository, create one?".format(gh_org, gh_repo)
)
if not maybe_continue():
warning("Skipping the pull request...")
return
# Create a fork
create_fork(gh_org, gh_repo, gh_username, gh_password)
# Clone the fork
info(fmt("@{bf}@!==> @|@!" + "Cloning {0}/{1}...".format(gh_username, gh_repo)))
temp_dir = tempfile.mkdtemp()
new_branch = None
title = "{0}: {1} in '{2}' [bloom]".format(repository, version, gh_path)
body = """\
Increasing version of package(s) in repository `{0}`:
- previous version: `{1}`
- new version: `{2}`
- distro file: `{3}`
- bloom version: `{4}`
""".format(
repository, orig_version or "null", version, gh_path, bloom.__version__
)
with change_directory(temp_dir):
def _my_run(cmd):
info(fmt("@{bf}@!==> @|@!" + str(cmd)))
# out = check_output(cmd, stderr=subprocess.STDOUT, shell=True)
out = None
from subprocess import call
call(cmd, shell=True)
if out:
info(out, use_prefix=False)
_my_run("git clone https://github.com/{0}/{1}.git".format(gh_username, gh_repo))
with change_directory(gh_repo):
_my_run("git remote add bloom https://github.com/{0}/{1}.git".format(gh_org, gh_repo))
_my_run("git remote update")
_my_run("git fetch")
track_branches()
branches = get_branches()
new_branch = "bloom-{repository}-{count}"
count = 0
while new_branch.format(repository=repository, count=count) in branches:
count += 1
new_branch = new_branch.format(repository=repository, count=count)
# Final check
info(fmt("@{cf}Pull Request Title: @{yf}" + title))
info(fmt("@{cf}Pull Request Body : \n@{yf}" + body))
msg = fmt(
"@!Open a @|@{cf}pull request@| @!@{kf}from@| @!'@|@!@{bf}"
+ "{gh_username}/{gh_repo}:{new_branch}".format(**locals())
+ "@|@!' @!@{kf}into@| @!'@|@!@{bf}"
+ "{gh_org}/{gh_repo}:{gh_branch}".format(**locals())
+ "@|@!'?"
)
info(msg)
if not maybe_continue():
warning("Skipping the pull request...")
return
_my_run("git checkout -b {0} bloom/{1}".format(new_branch, gh_branch))
with open("{0}".format(gh_path), "w") as f:
info(fmt("@{bf}@!==> @|@!Writing new distribution file: ") + str(gh_path))
f.write(updated_distro_file)
_my_run("git add {0}".format(gh_path))
_my_run('git commit -m "{0}"'.format(title))
_my_run("git push origin {0}".format(new_branch))
# Open the pull request
return create_pull_request(gh_org, gh_repo, gh_username, gh_password, gh_branch, new_branch, title, body)
def generate_ros_distro_diff(track, repository, distro):
release_dict = get_release_file(distro).get_data()
# Get packages
packages = get_packages()
if len(packages) == 0:
warning("No packages found, will not generate 'package: path' entries for rosdistro.")
# Get version
track_dict = get_tracks_dict_raw()["tracks"][track]
last_version = track_dict["last_version"]
release_inc = track_dict["release_inc"]
version = "{0}-{1}".format(last_version, release_inc)
# Create a repository if there isn't already one
if repository not in release_dict["repositories"]:
global _user_provided_release_url
release_dict["repositories"][repository] = {"url": _user_provided_release_url}
# Update the repository
repo = release_dict["repositories"][repository]
if "tags" not in repo:
repo["tags"] = {}
repo["tags"]["release"] = "release/%s/{package}/{version}" % distro
repo["version"] = version
if "packages" not in repo:
repo["packages"] = {}
for path, pkg in packages.items():
if pkg.name not in repo["packages"]:
repo["packages"][pkg.name] = {}
repo["packages"][pkg.name]["subfolder"] = path # This will be shortened
# Remove any missing packages
for pkg_name in dict(repo["packages"]):
if pkg_name not in [p.name for p in packages.values()]:
if pkg_name in repo["packages"]:
del repo["packages"][pkg_name]
# Do the diff
distro_file_name = get_relative_release_file_path(distro)
updated_release_file = rosdistro.ReleaseFile("distro", release_dict)
distro_dump = yaml_from_release_file(updated_release_file)
distro_file_raw = load_url_to_file_handle(get_release_file_url(distro)).read()
if distro_file_raw != distro_dump:
udiff = difflib.unified_diff(
distro_file_raw.splitlines(), distro_dump.splitlines(), fromfile=distro_file_name, tofile=distro_file_name
)
temp_dir = tempfile.mkdtemp()
udiff_file = os.path.join(temp_dir, repository + "-" + version + ".patch")
udiff_raw = ""
info("Unified diff for the ROS distro file located at '{0}':".format(udiff_file))
for line in udiff:
if line.startswith("@@"):
udiff_raw += line
line = fmt("@{cf}" + sanitize(line))
if line.startswith("+"):
if not line.startswith("+++"):
line += "\n"
udiff_raw += line
line = fmt("@{gf}" + sanitize(line))
if line.startswith("-"):
if not line.startswith("---"):
line += "\n"
udiff_raw += line
line = fmt("@{rf}" + sanitize(line))
if line.startswith(" "):
line += "\n"
udiff_raw += line
info(line, use_prefix=False, end="")
with open(udiff_file, "w+") as f:
f.write(udiff_raw)
return updated_release_file
else:
warning("This release resulted in no changes to the ROS distro file...")
return None
def generate_ros_distro_diff(track, repository, distro):
release_dict = get_release_file(distro).get_data()
# Get packages
packages = get_packages()
if len(packages) == 0:
warning("No packages found, will not generate 'package: path' entries for rosdistro.")
# Get version
track_dict = get_tracks_dict_raw()['tracks'][track]
last_version = track_dict['last_version']
release_inc = track_dict['release_inc']
version = '{0}-{1}'.format(last_version, release_inc)
# Create a repository if there isn't already one
if repository not in release_dict['repositories']:
global _user_provided_release_url
release_dict['repositories'][repository] = {
'url': _user_provided_release_url
}
# Update the repository
repo = release_dict['repositories'][repository]
if 'tags' not in repo:
repo['tags'] = {}
repo['tags']['release'] = 'release/%s/{package}/{version}' % distro
repo['version'] = version
if 'packages' not in repo:
repo['packages'] = {}
for path, pkg in packages.items():
if pkg.name not in repo['packages']:
repo['packages'][pkg.name] = {}
repo['packages'][pkg.name]['subfolder'] = path # This will be shortened
# Remove any missing packages
for pkg_name in dict(repo['packages']):
if pkg_name not in [p.name for p in packages.values()]:
if pkg_name in repo['packages']:
del repo['packages'][pkg_name]
# Do the diff
distro_file_name = get_relative_release_file_path(distro)
updated_release_file = rosdistro.ReleaseFile('distro', release_dict)
distro_dump = yaml_from_release_file(updated_release_file)
distro_file_raw = load_url_to_file_handle(get_release_file_url(distro)).read()
if distro_file_raw != distro_dump:
udiff = difflib.unified_diff(distro_file_raw.splitlines(), distro_dump.splitlines(),
fromfile=distro_file_name, tofile=distro_file_name)
temp_dir = tempfile.mkdtemp()
udiff_file = os.path.join(temp_dir, repository + '-' + version + '.patch')
udiff_raw = ''
info("Unified diff for the ROS distro file located at '{0}':".format(udiff_file))
for line in udiff:
if line.startswith('@@'):
udiff_raw += line
line = fmt('@{cf}' + sanitize(line))
if line.startswith('+'):
if not line.startswith('+++'):
line += '\n'
udiff_raw += line
line = fmt('@{gf}' + sanitize(line))
if line.startswith('-'):
if not line.startswith('---'):
line += '\n'
udiff_raw += line
line = fmt('@{rf}' + sanitize(line))
if line.startswith(' '):
line += '\n'
udiff_raw += line
info(line, use_prefix=False, end='')
with open(udiff_file, 'w+') as f:
f.write(udiff_raw)
return updated_release_file
else:
warning("This release resulted in no changes to the ROS distro file...")
return None
def open_pull_request(track, repository, distro):
# Get the diff
release_file = get_release_file(distro)
if repository in release_file.repositories:
orig_version = release_file.repositories[repository].version
else:
orig_version = None
updated_release_file = generate_ros_distro_diff(track, repository, distro)
if updated_release_file is None:
# There were no changes, no pull request required
return None
version = updated_release_file.repositories[repository].version
updated_distro_file = yaml_from_release_file(updated_release_file)
# Determine if the distro file is hosted on github...
gh_org, gh_repo, gh_branch, gh_path = get_gh_info(
get_release_file_url(distro))
if None in [gh_org, gh_repo, gh_branch, gh_path]:
warning("Automated pull request only available via github.com")
return
# Get the github user name
gh_username = None
bloom_user_path = os.path.join(os.path.expanduser('~'), '.bloom_user')
if os.path.exists(bloom_user_path):
with open(bloom_user_path, 'r') as f:
gh_username = f.read().strip()
gh_username = gh_username or getpass.getuser()
response = raw_input("github user name [{0}]: ".format(gh_username))
if response:
gh_username = response
info(
"Would you like bloom to store your github user name (~/.bloom_user)?"
)
if maybe_continue():
with open(bloom_user_path, 'w') as f:
f.write(gh_username)
else:
with open(bloom_user_path, 'w') as f:
f.write(' ')
warning(
"If you want to have bloom store it in the future remove the ~/.bloom_user file."
)
# Get the github password
gh_password = getpass.getpass("github password (This is not stored):")
if not gh_password or not gh_username:
error("Either the github username or github password is not set.")
warning("Skipping the pull request...")
return
# Check for fork
info(fmt("@{bf}@!==> @|@!Checking for rosdistro fork on github..."))
gh_user_repos = fetch_github_api(
'https://api.github.com/users/{0}/repos'.format(gh_username),
use_pagination=True)
if gh_user_repos is None:
error("Failed to get a list of repositories for user: '******'".format(
gh_username))
warning("Skipping the pull request...")
return
if 'rosdistro' not in [x['name'] for x in gh_user_repos if 'name' in x]:
warning(
"Github user '{0}' does not have a fork ".format(gh_username) +
"of the {0}:{1} repository, create one?".format(gh_org, gh_repo))
if not maybe_continue():
warning("Skipping the pull request...")
return
# Create a fork
create_fork(gh_org, gh_repo, gh_username, gh_password)
# Clone the fork
info(
fmt("@{bf}@!==> @|@!" +
"Cloning {0}/{1}...".format(gh_username, gh_repo)))
temp_dir = tempfile.mkdtemp()
new_branch = None
title = "{0}: {1} in '{2}' [bloom]".format(repository, version, gh_path)
body = """\
Increasing version of package(s) in repository `{0}`:
- previous version: `{1}`
- new version: `{2}`
- distro file: `{3}`
- bloom version: `{4}`
""".format(repository, orig_version or 'null', version, gh_path,
bloom.__version__)
with change_directory(temp_dir):
def _my_run(cmd):
info(fmt("@{bf}@!==> @|@!" + str(cmd)))
# out = check_output(cmd, stderr=subprocess.STDOUT, shell=True)
out = None
from subprocess import call
call(cmd, shell=True)
if out:
info(out, use_prefix=False)
_my_run('git clone https://github.com/{0}/{1}.git'.format(
gh_username, gh_repo))
with change_directory(gh_repo):
_my_run(
'git remote add bloom https://github.com/{0}/{1}.git'.format(
gh_org, gh_repo))
_my_run('git remote update')
_my_run('git fetch')
track_branches()
branches = get_branches()
new_branch = 'bloom-{repository}-{count}'
count = 0
while new_branch.format(repository=repository,
count=count) in branches:
count += 1
new_branch = new_branch.format(repository=repository, count=count)
# Final check
info(fmt("@{cf}Pull Request Title: @{yf}" + title))
info(fmt("@{cf}Pull Request Body : \n@{yf}" + body))
msg = fmt(
"@!Open a @|@{cf}pull request@| @!@{kf}from@| @!'@|@!@{bf}" +
"{gh_username}/{gh_repo}:{new_branch}".format(**locals()) +
"@|@!' @!@{kf}into@| @!'@|@!@{bf}" +
"{gh_org}/{gh_repo}:{gh_branch}".format(**locals()) + "@|@!'?")
info(msg)
if not maybe_continue():
warning("Skipping the pull request...")
return
_my_run('git checkout -b {0} bloom/{1}'.format(
new_branch, gh_branch))
with open('{0}'.format(gh_path), 'w') as f:
info(
fmt("@{bf}@!==> @|@!Writing new distribution file: ") +
str(gh_path))
f.write(updated_distro_file)
_my_run('git add {0}'.format(gh_path))
_my_run('git commit -m "{0}"'.format(title))
_my_run('git push origin {0}'.format(new_branch))
# Open the pull request
return create_pull_request(gh_org, gh_repo, gh_username, gh_password,
gh_branch, new_branch, title, body)
def generate_ros_distro_diff(track, repository, distro):
release_dict = get_release_file(distro).get_data()
# Get packages
packages = get_packages()
if len(packages) == 0:
warning(
"No packages found, will not generate 'package: path' entries for rosdistro."
)
# Get version
track_dict = get_tracks_dict_raw()['tracks'][track]
last_version = track_dict['last_version']
release_inc = track_dict['release_inc']
version = '{0}-{1}'.format(last_version, release_inc)
# Create a repository if there isn't already one
if repository not in release_dict['repositories']:
global _user_provided_release_url
release_dict['repositories'][repository] = {
'url': _user_provided_release_url
}
# Update the repository
repo = release_dict['repositories'][repository]
if 'tags' not in repo:
repo['tags'] = {}
repo['tags']['release'] = 'release/%s/{package}/{version}' % distro
repo['version'] = version
if 'packages' not in repo:
repo['packages'] = {}
for path, pkg in packages.items():
if pkg.name not in repo['packages']:
repo['packages'][pkg.name] = {}
repo['packages'][
pkg.name]['subfolder'] = path # This will be shortened
# Remove any missing packages
for pkg_name in dict(repo['packages']):
if pkg_name not in [p.name for p in packages.values()]:
if pkg_name in repo['packages']:
del repo['packages'][pkg_name]
# Do the diff
distro_file_name = get_relative_release_file_path(distro)
updated_release_file = rosdistro.ReleaseFile('distro', release_dict)
distro_dump = yaml_from_release_file(updated_release_file)
distro_file_raw = load_url_to_file_handle(
get_release_file_url(distro)).read()
if distro_file_raw != distro_dump:
udiff = difflib.unified_diff(distro_file_raw.splitlines(),
distro_dump.splitlines(),
fromfile=distro_file_name,
tofile=distro_file_name)
temp_dir = tempfile.mkdtemp()
udiff_file = os.path.join(temp_dir,
repository + '-' + version + '.patch')
udiff_raw = ''
info("Unified diff for the ROS distro file located at '{0}':".format(
udiff_file))
for line in udiff:
if line.startswith('@@'):
udiff_raw += line
line = fmt('@{cf}' + sanitize(line))
if line.startswith('+'):
if not line.startswith('+++'):
line += '\n'
udiff_raw += line
line = fmt('@{gf}' + sanitize(line))
if line.startswith('-'):
if not line.startswith('---'):
line += '\n'
udiff_raw += line
line = fmt('@{rf}' + sanitize(line))
if line.startswith(' '):
line += '\n'
udiff_raw += line
info(line, use_prefix=False, end='')
with open(udiff_file, 'w+') as f:
f.write(udiff_raw)
return updated_release_file
else:
warning(
"This release resulted in no changes to the ROS distro file...")
return None