def test_verify_files_parsable(): url = 'file://' + FILES_DIR + '/index_v2.yaml' index = get_index(url) distribution_file = get_distribution_file(index, 'foo') data = yaml_from_distribution_file(distribution_file) with open(os.path.join(FILES_DIR, 'foo', 'distribution.yaml'), 'r') as f: expected = f.read() assert data == expected, get_diff(expected, data)
def open_pull_request(track, repository, distro): # Get the diff distribution_file = get_distribution_file(distro) if repository in distribution_file.repositories and \ distribution_file.repositories[repository].release_repository is not None: orig_version = distribution_file.repositories[repository].release_repository.version else: orig_version = None updated_distribution_file = generate_ros_distro_diff(track, repository, distro) if updated_distribution_file is None: # There were no changes, no pull request required return None version = updated_distribution_file.repositories[repository].release_repository.version updated_distro_file_yaml = yaml_from_distribution_file(updated_distribution_file) # Determine if the distro file is hosted on github... base_org, base_repo, base_branch, base_path = get_gh_info(get_disitrbution_file_url(distro)) if None in [base_org, base_repo, base_branch, base_path]: warning("Automated pull request only available via github.com") return # Get the github interface gh = get_github_interface() # Determine the head org/repo for the pull request head_org = gh.username # The head org will always be gh user head_repo = None # Check if the github user and the base org are the same if gh.username == base_org: # If it is, then a fork is not necessary head_repo = base_repo else: info(fmt("@{bf}@!==> @|@!Checking on github for a fork to make the pull request from...")) # It is not, so a fork will be required # Check if a fork already exists on the user's account with the same name base_full_name = '{base_org}/{base_repo}'.format(**locals()) try: repo_data = gh.get_repo(gh.username, base_repo) if repo_data.get('fork', False): # Check if it is a fork # If it is, check that it is a fork of the destination parent = repo_data.get('parent', {}).get('full_name', None) if parent == base_full_name: # This is a valid fork head_repo = base_repo except GithubException as exc: debug("Received GithubException while checking for fork: {exc}".format(**locals())) pass # 404 or unauthorized, but unauthorized should have been caught above # If not head_repo, then either the fork has a different name, or there isn't one if head_repo is None: info(fmt("@{bf}@!==> @|@!" + "{head_org}/{base_repo} is not a fork, searching...".format(**locals()))) # First we should look at every repository for the user and see if they are a fork user_repos = gh.list_repos(gh.username) for repo in user_repos: # If it is a fork and the parent is base_org/base_repo if repo.get('fork', False) and repo.get('parent', {}).get('full_name', '') == base_full_name: # Then this is a valid fork head_repo = repo['name'] # If not head_repo still, a fork does not exist and must be created if head_repo is None: warning("Could not find a fork of {base_full_name} on the {gh.username} Github account." .format(**locals())) warning("Would you like to create one now?") if not maybe_continue(): warning("Skipping the pull request...") return # Create a fork try: gh.create_fork(base_org, base_repo) # Will raise if not successful head_repo = base_repo except GithubException as exc: error("Aborting pull request: {0}".format(exc)) return info(fmt("@{bf}@!==> @|@!" + "Using this fork to make a pull request from: {head_org}/{head_repo}".format(**locals()))) # Clone the fork info(fmt("@{bf}@!==> @|@!" + "Cloning {0}/{1}...".format(head_org, head_repo))) new_branch = None title = "{0}: {1} in '{2}' [bloom]".format(repository, version, base_path) body = """\ Increasing version of package(s) in repository `{0}` to `{2}`: - distro file: `{3}` - bloom version: `{4}` - previous version for package: `{1}` """.format(repository, orig_version or 'null', version, base_path, bloom.__version__) body += get_changelog_summary(generate_release_tag(distro)) with temporary_directory() as temp_dir: def _my_run(cmd, msg=None): if msg: info(fmt("@{bf}@!==> @|@!" + sanitize(msg))) else: info(fmt("@{bf}@!==> @|@!" + sanitize(str(cmd)))) from subprocess import check_call check_call(cmd, shell=True) # Use the oauth token to clone rosdistro_url = 'https://{gh.token}:[email protected]/{base_org}/{base_repo}.git'.format(**locals()) rosdistro_fork_url = 'https://{gh.token}:[email protected]/{head_org}/{head_repo}.git'.format(**locals()) _my_run('mkdir -p {base_repo}'.format(**locals())) with change_directory(base_repo): _my_run('git init') branches = [x['name'] for x in gh.list_branches(head_org, head_repo)] new_branch = 'bloom-{repository}-{count}' count = 0 while new_branch.format(repository=repository, count=count) in branches: count += 1 new_branch = new_branch.format(repository=repository, count=count) # Final check info(fmt("@{cf}Pull Request Title: @{yf}" + title)) info(fmt("@{cf}Pull Request Body : \n@{yf}" + body)) msg = fmt("@!Open a @|@{cf}pull request@| @!@{kf}from@| @!'@|@!@{bf}" + "{head_repo}/{head_repo}:{new_branch}".format(**locals()) + "@|@!' @!@{kf}into@| @!'@|@!@{bf}" + "{base_org}/{base_repo}:{base_branch}".format(**locals()) + "@|@!'?") info(msg) if not maybe_continue(): warning("Skipping the pull request...") return _my_run('git checkout -b {new_branch}'.format(**locals())) _my_run('git pull {rosdistro_url} {base_branch}'.format(**locals()), "Pulling latest rosdistro branch") with open('{0}'.format(base_path), 'w') as f: info(fmt("@{bf}@!==> @|@!Writing new distribution file: ") + str(base_path)) f.write(updated_distro_file_yaml) _my_run('git add {0}'.format(base_path)) _my_run('git commit -m "{0}"'.format(title)) _my_run('git push {rosdistro_fork_url} {new_branch}'.format(**locals()), "Pushing changes to fork") # Open the pull request return gh.create_pull_request(base_org, base_repo, base_branch, head_org, new_branch, title, body)
def generate_ros_distro_diff(track, repository, distro): distribution_dict = get_distribution_file(distro).get_data() # Get packages packages = get_packages() if len(packages) == 0: warning("No packages found, will not generate 'package: path' entries for rosdistro.") # Get version track_dict = get_tracks_dict_raw()['tracks'][track] last_version = track_dict['last_version'] release_inc = track_dict['release_inc'] version = '{0}-{1}'.format(last_version, release_inc) # Create a repository if there isn't already one if repository not in distribution_dict['repositories']: global _user_provided_release_url distribution_dict['repositories'][repository] = {} # Create a release entry if there isn't already one if 'release' not in distribution_dict['repositories'][repository]: distribution_dict['repositories'][repository]['release'] = { 'url': _user_provided_release_url } # Update the repository repo = distribution_dict['repositories'][repository]['release'] if 'tags' not in repo: repo['tags'] = {} repo['tags']['release'] = generate_release_tag(distro) repo['version'] = version if 'packages' not in repo: repo['packages'] = [] for path, pkg in packages.items(): if pkg.name not in repo['packages']: repo['packages'].append(pkg.name) # Remove any missing packages packages_being_released = [p.name for p in packages.values()] for pkg_name in list(repo['packages']): if pkg_name not in packages_being_released: repo['packages'].remove(pkg_name) repo['packages'].sort() # Do the diff distro_file_name = get_relative_distribution_file_path(distro) updated_distribution_file = rosdistro.DistributionFile(distro, distribution_dict) distro_dump = yaml_from_distribution_file(updated_distribution_file) distro_file_raw = load_url_to_file_handle(get_disitrbution_file_url(distro)).read() if distro_file_raw != distro_dump: # Calculate the diff udiff = difflib.unified_diff(distro_file_raw.splitlines(), distro_dump.splitlines(), fromfile=distro_file_name, tofile=distro_file_name) temp_dir = tempfile.mkdtemp() udiff_file = os.path.join(temp_dir, repository + '-' + version + '.patch') udiff_raw = '' info("Unified diff for the ROS distro file located at '{0}':".format(udiff_file)) for line in udiff: if line.startswith('@@'): udiff_raw += line line = fmt('@{cf}' + sanitize(line)) if line.startswith('+'): if not line.startswith('+++'): line += '\n' udiff_raw += line line = fmt('@{gf}' + sanitize(line)) if line.startswith('-'): if not line.startswith('---'): line += '\n' udiff_raw += line line = fmt('@{rf}' + sanitize(line)) if line.startswith(' '): line += '\n' udiff_raw += line info(line, use_prefix=False, end='') # Assert that only this repository is being changed distro_file_yaml = yaml.load(distro_file_raw) distro_yaml = yaml.load(distro_dump) if 'repositories' in distro_file_yaml: distro_file_repos = distro_file_yaml['repositories'] for repo in distro_yaml['repositories']: if repo == repository: continue if repo not in distro_file_repos or distro_file_repos[repo] != distro_yaml['repositories'][repo]: error("This generated pull request modifies a repository entry other than the one being released.") error("This likely occured because the upstream rosdistro changed during this release.") error("This pull request will abort, please re-run this command with the -p option to try again.", exit=True) # Write the diff out to file with open(udiff_file, 'w+') as f: f.write(udiff_raw) # Return the diff return updated_distribution_file else: warning("This release resulted in no changes to the ROS distro file...") return None
repositories_to_retry.append(repo_name) else: # Nothing to do if the release is there. pass print(f'Found {len(new_repositories)} new repositories to release:', new_repositories) print(f'Found {len(repositories_to_retry)} repositories to retry:', repositories_to_retry) # Copy out an optimistic destination distribution file to bloom everything # against. This obviates the need to bloom packages in a topological order or # do any special handling for dependency cycles between repositories as are # known to occur in the ros2/launch repository. To allow this we must keep # track of repositories that fail to bloom and pull their release in a cleanup # step. with open(dest_distribution_filename, 'w') as f: f.write(yaml_from_distribution_file(dest_distribution)) repositories_bloomed = [] repositories_with_errors = [] workdir = tempfile.mkdtemp() os.chdir(workdir) os.environ['ROSDISTRO_INDEX_URL'] = rosdistro_index_url for repo_name in sorted(new_repositories + repositories_to_retry): try: release_spec = dest_distribution.repositories[repo_name].release_repository print('Adding repo:', repo_name) if release_spec.type != 'git': raise ValueError('This script can only handle git repositories.') remote_url = release_spec.url
def write_internal_distro(self): distro_file_path = pathlib.Path( self.internal_distro_file[len('file://'):]) distro_file_path.write_text( yaml_from_distribution_file(self.internal_distro))
def generate_ros_distro_diff(track, repository, distro): distribution_dict = get_distribution_file(distro).get_data() # Get packages packages = get_packages() if len(packages) == 0: warning("No packages found, will not generate 'package: path' entries for rosdistro.") # Get version track_dict = get_tracks_dict_raw()['tracks'][track] last_version = track_dict['last_version'] release_inc = track_dict['release_inc'] version = '{0}-{1}'.format(last_version, release_inc).encode('utf-8') # Create a repository if there isn't already one if repository not in distribution_dict['repositories']: global _user_provided_release_url distribution_dict['repositories'][repository] = {} # Create a release entry if there isn't already one if 'release' not in distribution_dict['repositories'][repository]: distribution_dict['repositories'][repository]['release'.encode('utf-8')] = { 'url'.encode('utf-8'): _user_provided_release_url } # Update the repository repo = distribution_dict['repositories'][repository]['release'] if 'tags' not in repo: repo['tags'.encode('utf-8')] = {} repo['tags']['release'.encode('utf-8')] = generate_release_tag(distro) repo['version'.encode('utf-8')] = version if 'packages' not in repo: repo['packages'.encode('utf-8')] = [] for path, pkg in packages.items(): if pkg.name not in repo['packages']: repo['packages'].append(pkg.name) # Remove any missing packages packages_being_released = [p.name for p in packages.values()] for pkg_name in list(repo['packages']): if pkg_name not in packages_being_released: repo['packages'].remove(pkg_name) repo['packages'].sort() def get_repository_info_from_user(): data = {} while True: vcs_type = safe_input('VCS type [git, svn, hg, bzr]: ') if vcs_type in ['git', 'svn', 'hg', 'bzr']: break error("'{0}' is not a valid vcs type.".format(vcs_type)) if not maybe_continue(msg='Try again'): return {} data['type'] = vcs_type while True: url = safe_input('VCS url: ') if url: break error("Nothing entered for url.") if not maybe_continue(msg='Try again'): return {} data['url'] = url while True: version = safe_input('VCS version [commit, tag, branch, etc]: ') if version: break error("Nothing entered for version.") if not maybe_continue(msg='Try again'): return {} data['version'] = version return data # Ask for doc entry if 'BLOOM_DONT_ASK_FOR_DOCS' not in os.environ: docs = distribution_dict['repositories'][repository].get('doc', {}) if not docs and maybe_continue(msg='Would you like to add documentation information for this repository?'): info("Please enter your repository information for the doc generation job.") info("This information should point to the repository from which documentation should be generated.") docs = get_repository_info_from_user() distribution_dict['repositories'][repository]['doc'] = docs # Ask for source entry if 'BLOOM_DONT_ASK_FOR_SOURCE' not in os.environ: source = distribution_dict['repositories'][repository].get('source', {}) if not source and maybe_continue(msg='Would you like to add source information for this repository?'): info("Please enter information which points ot the active development branch for this repository.") info("This information is used to run continuous integration jobs and for developers to checkout from.") source = get_repository_info_from_user() distribution_dict['repositories'][repository]['source'] = source # Ask for maintainership information if 'BLOOM_DONT_ASK_FOR_MAINTENANCE_STATUS' not in os.environ: status = distribution_dict['repositories'][repository].get('status', None) description = distribution_dict['repositories'][repository].get('status_description', None) if status is None and maybe_continue(msg='Would you like to add a maintenance status for this repository?'): info("Please enter a maintenance status.") info("Valid maintenance statuses:") info("- developed: active development is in progress") info("- maintained: no new development, but bug fixes and pull requests are addressed") info("- end-of-life: should not be used, will disapear at some point") while True: status = safe_input('Status: ') if status in ['developed', 'maintained', 'end-of-life']: break error("'{0}' is not a valid status.".format(status)) if not maybe_continue(msg='Try again'): status = None break if status is not None: info("You can also enter a status description.") info("This is usually reserved for giving a reason when a status is 'end-of-life'.") if description is not None: info("Current status description: {0}".format(description)) description_in = safe_input('Status Description [press Enter for no change]: ') if description_in: description = description_in if status is not None: distribution_dict['repositories'][repository]['status'] = status if description is not None: distribution_dict['repositories'][repository]['status_description'] = description # Do the diff distro_file_name = get_relative_distribution_file_path(distro) updated_distribution_file = rosdistro.DistributionFile(distro, distribution_dict) distro_dump = yaml_from_distribution_file(updated_distribution_file) distro_file_raw = load_url_to_file_handle(get_disitrbution_file_url(distro)).read() if distro_file_raw != distro_dump: # Calculate the diff udiff = difflib.unified_diff(distro_file_raw.splitlines(), distro_dump.splitlines(), fromfile=distro_file_name, tofile=distro_file_name) temp_dir = tempfile.mkdtemp() udiff_file = os.path.join(temp_dir, repository + '-' + version + '.patch') udiff_raw = '' info("Unified diff for the ROS distro file located at '{0}':".format(udiff_file)) for line in udiff: if line.startswith('@@'): udiff_raw += line line = fmt('@{cf}' + sanitize(line)) if line.startswith('+'): if not line.startswith('+++'): line += '\n' udiff_raw += line line = fmt('@{gf}' + sanitize(line)) if line.startswith('-'): if not line.startswith('---'): line += '\n' udiff_raw += line line = fmt('@{rf}' + sanitize(line)) if line.startswith(' '): line += '\n' udiff_raw += line info(line, use_prefix=False, end='') # Assert that only this repository is being changed distro_file_yaml = yaml.load(distro_file_raw) distro_yaml = yaml.load(distro_dump) if 'repositories' in distro_file_yaml: distro_file_repos = distro_file_yaml['repositories'] for repo in distro_yaml['repositories']: if repo == repository: continue if repo not in distro_file_repos or distro_file_repos[repo] != distro_yaml['repositories'][repo]: error("This generated pull request modifies a repository entry other than the one being released.") error("This likely occured because the upstream rosdistro changed during this release.") error("This pull request will abort, please re-run this command with the -p option to try again.", exit=True) # Write the diff out to file with open(udiff_file, 'w+') as f: f.write(udiff_raw) # Return the diff return updated_distribution_file else: warning("This release resulted in no changes to the ROS distro file...") return None
def open_pull_request(track, repository, distro, ssh_pull_request): # Get the diff distribution_file = get_distribution_file(distro) if repository in distribution_file.repositories and \ distribution_file.repositories[repository].release_repository is not None: orig_version = distribution_file.repositories[repository].release_repository.version else: orig_version = None updated_distribution_file = generate_ros_distro_diff(track, repository, distro) if updated_distribution_file is None: # There were no changes, no pull request required return None version = updated_distribution_file.repositories[repository].release_repository.version updated_distro_file_yaml = yaml_from_distribution_file(updated_distribution_file) # Determine if the distro file is hosted on github... gh_org, gh_repo, gh_branch, gh_path = get_gh_info(get_disitrbution_file_url(distro)) if None in [gh_org, gh_repo, gh_branch, gh_path]: warning("Automated pull request only available via github.com") return # Get the github user name gh_username = None bloom_user_path = os.path.join(os.path.expanduser('~'), '.bloom_user') if os.path.exists(bloom_user_path): with open(bloom_user_path, 'r') as f: gh_username = f.read().strip() gh_username = gh_username or getpass.getuser() response = safe_input("github user name [{0}]: ".format(gh_username)) if response: gh_username = response info("Would you like bloom to store your github user name (~/.bloom_user)?") if maybe_continue(): with open(bloom_user_path, 'w') as f: f.write(gh_username) else: with open(bloom_user_path, 'w') as f: f.write(' ') warning("If you want to have bloom store it in the future remove the ~/.bloom_user file.") # Get the github password gh_password = getpass.getpass("github password (This is not stored):") if not gh_password or not gh_username: error("Either the github username or github password is not set.") warning("Skipping the pull request...") return # Check for fork info(fmt("@{bf}@!==> @|@!Checking for rosdistro fork on github...")) gh_user_repos = fetch_github_api('https://api.github.com/users/{0}/repos'.format(gh_username), use_pagination=True) if gh_user_repos is None: error("Failed to get a list of repositories for user: '******'".format(gh_username)) warning("Skipping the pull request...") return if 'rosdistro' not in [x['name'] for x in gh_user_repos if 'name' in x]: warning("Github user '{0}' does not have a fork ".format(gh_username) + "of the {0}:{1} repository, create one?".format(gh_org, gh_repo)) if not maybe_continue(): warning("Skipping the pull request...") return # Create a fork create_fork(gh_org, gh_repo, gh_username, gh_password) # Clone the fork info(fmt("@{bf}@!==> @|@!" + "Cloning {0}/{1}...".format(gh_username, gh_repo))) temp_dir = tempfile.mkdtemp() new_branch = None title = "{0}: {1} in '{2}' [bloom]".format(repository, version, gh_path) body = """\ Increasing version of package(s) in repository `{0}`: - previous version: `{1}` - new version: `{2}` - distro file: `{3}` - bloom version: `{4}` """.format(repository, orig_version or 'null', version, gh_path, bloom.__version__) with change_directory(temp_dir): def _my_run(cmd): info(fmt("@{bf}@!==> @|@!" + str(cmd))) # out = check_output(cmd, stderr=subprocess.STDOUT, shell=True) out = None from subprocess import call call(cmd, shell=True) if out: info(out, use_prefix=False) if ssh_pull_request: rosdistro_git_fork = '[email protected]/{0}/{1}.git'.format(gh_username, gh_repo) else: rosdistro_git_fork = 'https://github.com/{0}/{1}.git'.format(gh_username, gh_repo) _my_run('git clone {0}'.format(rosdistro_git_fork)) with change_directory(gh_repo): _my_run('git remote add bloom https://github.com/{0}/{1}.git'.format(gh_org, gh_repo)) _my_run('git remote update') _my_run('git fetch') track_branches() branches = get_branches() new_branch = 'bloom-{repository}-{count}' count = 0 while new_branch.format(repository=repository, count=count) in branches: count += 1 new_branch = new_branch.format(repository=repository, count=count) # Final check info(fmt("@{cf}Pull Request Title: @{yf}" + title)) info(fmt("@{cf}Pull Request Body : \n@{yf}" + body)) msg = fmt("@!Open a @|@{cf}pull request@| @!@{kf}from@| @!'@|@!@{bf}" + "{gh_username}/{gh_repo}:{new_branch}".format(**locals()) + "@|@!' @!@{kf}into@| @!'@|@!@{bf}" + "{gh_org}/{gh_repo}:{gh_branch}".format(**locals()) + "@|@!'?") info(msg) if not maybe_continue(): warning("Skipping the pull request...") return _my_run('git checkout -b {0} bloom/{1}'.format(new_branch, gh_branch)) with open('{0}'.format(gh_path), 'w') as f: info(fmt("@{bf}@!==> @|@!Writing new distribution file: ") + str(gh_path)) f.write(updated_distro_file_yaml) _my_run('git add {0}'.format(gh_path)) _my_run('git commit -m "{0}"'.format(title)) _my_run('git push origin {0}'.format(new_branch)) # Open the pull request return create_pull_request(gh_org, gh_repo, gh_username, gh_password, gh_branch, new_branch, title, body)
def generate_ros_distro_diff(track, repository, distro): distribution_dict = get_distribution_file(distro).get_data() # Get packages packages = get_packages() if len(packages) == 0: warning("No packages found, will not generate 'package: path' entries for rosdistro.") # Get version track_dict = get_tracks_dict_raw()['tracks'][track] last_version = track_dict['last_version'] release_inc = track_dict['release_inc'] version = '{0}-{1}'.format(last_version, release_inc) # Create a repository if there isn't already one if repository not in distribution_dict['repositories']: global _user_provided_release_url distribution_dict['repositories'][repository] = {} # Create a release entry if there isn't already one if 'release' not in distribution_dict['repositories'][repository]: distribution_dict['repositories'][repository]['release'] = { 'url': _user_provided_release_url } # Update the repository repo = distribution_dict['repositories'][repository]['release'] if 'tags' not in repo: repo['tags'] = {} repo['tags']['release'] = 'release/%s/{package}/{version}' % distro repo['version'] = version if 'packages' not in repo: repo['packages'] = [] for path, pkg in packages.items(): if pkg.name not in repo['packages']: repo['packages'].append(pkg.name) # Remove any missing packages packages_being_released = [p.name for p in packages.values()] for pkg_name in list(repo['packages']): if pkg_name not in packages_being_released: repo['packages'].remove(pkg_name) repo['packages'].sort() # Do the diff distro_file_name = get_relative_distribution_file_path(distro) updated_distribution_file = rosdistro.DistributionFile(distro, distribution_dict) distro_dump = yaml_from_distribution_file(updated_distribution_file) distro_file_raw = load_url_to_file_handle(get_disitrbution_file_url(distro)).read() if distro_file_raw != distro_dump: udiff = difflib.unified_diff(distro_file_raw.splitlines(), distro_dump.splitlines(), fromfile=distro_file_name, tofile=distro_file_name) temp_dir = tempfile.mkdtemp() udiff_file = os.path.join(temp_dir, repository + '-' + version + '.patch') udiff_raw = '' info("Unified diff for the ROS distro file located at '{0}':".format(udiff_file)) for line in udiff: if line.startswith('@@'): udiff_raw += line line = fmt('@{cf}' + sanitize(line)) if line.startswith('+'): if not line.startswith('+++'): line += '\n' udiff_raw += line line = fmt('@{gf}' + sanitize(line)) if line.startswith('-'): if not line.startswith('---'): line += '\n' udiff_raw += line line = fmt('@{rf}' + sanitize(line)) if line.startswith(' '): line += '\n' udiff_raw += line info(line, use_prefix=False, end='') with open(udiff_file, 'w+') as f: f.write(udiff_raw) return updated_distribution_file else: warning("This release resulted in no changes to the ROS distro file...") return None
def write_internal_distro(self, message=None): new_contents = yaml_from_distribution_file(self.internal_distro) self.write_file(self.internal_distro_path, new_contents, message)