def __exit__(self, *args): # save the cache, if it exists if self.filename: info("Saving cached file '%s'" % self.filename) self.cache_file = open(self.filename, 'wb') pickle.dump(self.cache, self.cache_file) self.cache_file.close()
def downloadArchive(self): if os.path.exists(self.getArchiveName()): info("using cached archive for package '%s'..." % self.name) else: info("downloading archive version for package '%s' from %s..." % (self.name, self.src_uri)) urlretrieve(self.src_uri, self.getArchiveName())
def get_dependencies( self, internal_depends, external_depends, is_native=False): dependencies = set() system_dependencies = set() union_deps = internal_depends | external_depends if len(union_deps) <= 0: return dependencies, system_dependencies for dep in union_deps: if dep in internal_depends: recipe = self.convert_to_oe_name(dep, is_native) dependencies.add(recipe) info('Internal dependency add: ' + recipe) continue try: results = resolve_dep(dep, 'openembedded', self.distro)[0] if not results: yoctoRecipe.rosdep_cache[dep] = [] continue for res in results: recipe = self.convert_to_oe_name(res, is_native) dependencies.add(recipe) system_dependencies.add(self.convert_to_oe_name(res)) yoctoRecipe.rosdep_cache[dep].add(res) info('External dependency add: ' + recipe) except UnresolvedDependency: info('Unresolved dependency: ' + dep) if dep in yoctoRecipe.rosdep_cache: cached_deps = yoctoRecipe.rosdep_cache[dep] if cached_deps == set(['null']): system_dependencies.add(dep) recipe = dep + self.get_native_suffix(is_native) dependencies.add(recipe) msg = 'Failed to resolve (cached):' warn('{0} {1}: {2}'.format(msg, dep, recipe)) elif cached_deps: system_dependencies |= cached_deps for d in cached_deps: recipe = self.convert_to_oe_name(d, is_native) dependencies.add(recipe) msg = 'Resolved in OpenEmbedded (cached):' info('{0} {1}: {2}'.format(msg, dep, recipe)) continue oe_query = OpenEmbeddedLayersDB() oe_query.query_recipe(self.convert_to_oe_name(dep)) if oe_query.exists(): recipe = self.convert_to_oe_name(oe_query.name, is_native) dependencies.add(recipe) oe_name = self.convert_to_oe_name(oe_query.name) system_dependencies.add(oe_name) yoctoRecipe.rosdep_cache[dep].add(oe_name) info('Resolved in OpenEmbedded: ' + dep + ' as ' + oe_query.name + ' in ' + oe_query.layer + ' as recipe ' + recipe) else: recipe = dep + self.get_native_suffix(is_native) dependencies.add(recipe) system_dependencies.add(dep) yoctoRecipe.rosdep_cache[dep].add('null') warn('Failed to resolve fully: ' + dep) return dependencies, system_dependencies
def clean_ros_recipe_dirs(self, distro=None): if distro: info('Cleaning up recipes-ros-{0} directory...'.format(distro)) self.repo.git.rm('-rf', 'recipes-ros-{0}'.format(distro)) else: info('Cleaning up recipes-ros-* directories...') self.repo.git.rm('-rf', 'recipes-ros-*')
def __init__(self, repo_dir, do_clone, org='ros', repo='ros-overlay'): self.repo = RepoInstance( org, repo, repo_dir=repo_dir, do_clone=do_clone ) self.branch_name = 'gentoo-bot-%s' % rand_ascii_str() info('Creating new branch {0}...'.format(self.branch_name)) self.repo.create_branch(self.branch_name)
def __init__( self, dir, do_clone, branch, org='ros', repo='meta-ros', from_branch='' ): self.repo = RepoInstance( org, repo, dir, do_clone, from_branch=from_branch) self.branch_name = branch info('Creating new branch {0}...'.format(self.branch_name)) self.repo.create_branch(self.branch_name)
def __enter__(self): # load the initial cache, if it exists if self.filename and os.path.isfile(self.filename): info("Loading cached file '%s'" % self.filename) self.cache_file = open(self.filename, 'rb') self.cache = pickle.load(self.cache_file) self.cache_file.close() return self.cache
def clone(self, branch=None): shutil.rmtree(self.repo_dir) msg = 'Cloning repo {0}/{1}'.format(self.repo_owner, self.repo_name) if self.repo_dir != self.repo_name: msg += (' into directory {0}'.format(self.repo_dir)) msg += '...' info(msg) self.repo = Repo.clone_from(self.repo_url, self.repo_dir) if branch: self.git.checkout(branch)
def set_up_container(): d = Docker() curr = os.getcwd() try: d.pull('allenh1', 'asw_test_env') except: d.build(os.path.realpath("../docker/Dockerfile")) info("Using repo in directory '%s'" % os.path.realpath("..")) d.map_directory(os.path.realpath(".."), repo_location) return d
def commit_changes(self, distro): info('Adding changes...') self.repo.git.add(self.repo.repo_dir) info('Committing to branch {0}...'.format(self.branch_name)) if distro == 'all': commit_msg = 'regenerate all distros, {0}' else: commit_msg = 'regenerate ros-{1}, {0}' commit_msg = commit_msg.format(time.ctime(), distro) self.repo.git.commit(m='{0}'.format(commit_msg))
def clean_ros_recipe_dirs(self, distro=None): if distro: info( 'Cleaning up meta-ros{}-{}/generated-recipes directory...' .format(yoctoRecipe._get_ros_version(distro), distro)) self.repo.git.rm('-rf', 'meta-ros{}-{}/generated-recipes'.format( yoctoRecipe._get_ros_version(distro), distro)) else: info('Cleaning up generated-recipes directories...') self.repo.git.rm('-rf', 'generated-recipes')
def commit_changes(self, distro): info('Adding changes...') self.repo.git.add(self.repo.repo_dir) info('Committing to branch {0}...'.format(self.branch_name)) commit_msg = { 'update': 'rosdistro sync, ', 'all': 'regenerate all distros, ', 'lunar': 'regenerate ros-lunar, ', 'indigo': 'regenerate ros-indigo, ', 'kinetic': 'regenerate ros-kinetic, ', }[distro or 'update'] + time.ctime() self.repo.git.commit(m='{0}'.format(commit_msg))
def commit_changes(self, distro, commit_msg): info('Adding changes...') self.repo.git.add('generated-recipes-{0}'.format(distro)) self.repo.git.add('conf/ros-distro/include/{0}/*.inc'.format(distro)) self.repo.git.add('files/{0}/cache.*'.format(distro)) self.repo.git.add('files/{0}/rosdep-resolve.yaml'.format(distro)) self.repo.git.add( 'files/{0}/newer-platform-components.list'.format(distro)) self.repo.git.add( 'files/{0}/superflore-change-summary.txt'.format(distro)) info('Committing to branch {0}...'.format(self.branch_name)) self.repo.git.commit(m=commit_msg)
def __enter__(self): if self.arg_path: try: os.mkdir(self.arg_path) except OSError as ex: if ex.errno != errno.EEXIST: raise return self.arg_path else: self.temp_path = tempfile.mkdtemp() info("Working in temporary directory %s" % self.temp_path) return self.temp_path
def add_generated_files(self, distro): info('Adding changes...') self.repo.git.add('meta-ros{0}-{1}/generated-recipes'.format( yoctoRecipe._get_ros_version(distro), distro)) self.repo.git.add('meta-ros{0}-{1}/conf/ros-distro/include/{1}/' 'generated/*.inc'.format( yoctoRecipe._get_ros_version(distro), distro)) self.repo.git.add('meta-ros{0}-{1}/files/{1}/generated/' 'rosdep-resolve.yaml'.format( yoctoRecipe._get_ros_version(distro), distro)) self.repo.git.add('meta-ros{0}-{1}/files/{1}/generated/' 'newer-platform-components.list'.format( yoctoRecipe._get_ros_version(distro), distro))
def __exit__(self, *args): if self.temp_path: info("Cleaning up temporary directory %s" % self.temp_path) try: shutil.rmtree(self.temp_path) except OSError as ex: if ex.errno == errno.EPERM: err("Failed to rmtree %s" % self.temp_path) err("Escalating to sudo rm -rf %s" % self.temp_path) subprocess.check_call( ('sudo rm -rf %s' % self.temp_path).split()) else: raise self.temp_path = None
def __init__(self, repo_dir, do_clone, org='allenh1', repo='meta-ros', from_branch=''): self.repo = RepoInstance(org, repo, repo_dir, do_clone, from_branch=from_branch) self.branch_name = 'yocto-bot-%s' % rand_ascii_str() info('Creating new branch {0}...'.format(self.branch_name)) self.repo.create_branch(self.branch_name)
def commit_changes(self, distro): info('Adding changes...') if distro == 'all' or distro == 'update': self.repo.git.add('recipes-ros-*') else: self.repo.git.add('recipes-ros-{0}'.format(distro)) commit_msg = { 'update': 'rosdistro sync, {0}', 'all': 'regenerate all distros, {0}', 'lunar': 'regenerate ros-lunar, {0}', 'indigo': 'regenerate ros-indigo, {0}', 'kinetic': 'regenerate ros-kinetic, {0}', }[distro].format(time.ctime()) info('Committing to branch {0}...'.format(self.branch_name)) self.repo.git.commit(m='{0}'.format(commit_msg))
def clean_ros_recipe_dirs(self, distro): # superflore-change-summary.txt is no longer being generated since: # https://github.com/ros-infrastructure/superflore/pull/273 # but remove it here to make sure it gets deleted when new distro # release is being generated files = 'meta-ros{0}-{1}/generated-recipes '\ 'meta-ros{0}-{1}/conf/ros-distro/include/{1}/generated '\ 'meta-ros{0}-{1}/files/{1}/generated/'\ 'newer-platform-components.list '\ 'meta-ros{0}-{1}/files/{1}/generated/rosdep-resolve.yaml '\ 'meta-ros{0}-{1}/files/{1}/generated/'\ 'superflore-change-summary.txt '.format( yoctoRecipe._get_ros_version(distro), distro) info('Cleaning up:\n{0}'.format(files)) self.repo.git.rm('-rf', '--ignore-unmatch', files.split())
def run(self, rm=True, show_cmd=False): cmd_string = "bash -c '" for i, bash_cmd in enumerate(self.bash_cmds): cmd_string += bash_cmd if i != len(self.bash_cmds) - 1: cmd_string += ' && ' cmd_string += "'" msg = "Running container with command string '%s'..." info(msg % cmd_string) self.client.containers.run( image=self.image, remove=rm, command=cmd_string, volumes=self.directory_map, ) ok("Docker container exited.")
def commit_changes(self, distro, commit_msg): info('Commit changes...') if self.repo.git.status('--porcelain') == '': info('Nothing changed; no commit done') else: if self.branch_name: info('Committing to branch {0}...'.format(self.branch_name)) else: info('Committing to current branch') self.repo.git.commit(m=commit_msg)
def __init__(self, repo_dir, do_clone, org='ros', repo='ros-overlay', from_branch='', new_branch=True): self.repo = RepoInstance(org, repo, repo_dir=repo_dir, do_clone=do_clone, from_branch=from_branch) if new_branch: self.branch_name = 'gentoo-bot-%s' % rand_ascii_str() info('Creating new branch {0}...'.format(self.branch_name)) self.repo.create_branch(self.branch_name) else: self.branch_name = None
def commit_changes(self, distro): info('Adding changes...') if not distro or distro == 'all' or distro == 'update': self.repo.git.add('recipes-ros-*') else: self.repo.git.add('recipes-ros-{0}'.format(distro)) values = { 'update': 'rosdistro sync, {0}', 'all': 'regenerate all distros, {0}' } values.update( dict([ (d, 'regenerate ros-%s, {0}' % d) \ for d in ros1_distros + ros2_distros ]) ) commit_msg = values[distro].format(time.ctime()) info('Committing to branch {0}...'.format(self.branch_name)) self.repo.git.commit(m='{0}'.format(commit_msg))
def run(self, verbose=True, log_file=None): # TODO(allenh1): add the ability to check out a non-master # branch of the overlay (for CI). info('testing gentoo package integrity') for pkg in sorted(self.package_list.keys()): self.container.add_bash_command('emaint sync -r ros-overlay') self.container.add_bash_command('emerge %s' % pkg) try: self.container.run(rm=True, show_cmd=True, privileged=True, log_file=log_file) self.package_list[pkg] = 'building' ok(" '%s': building" % pkg) except ContainerError: self.package_list[pkg] = 'failing' err(" '%s': failing" % pkg) if verbose: print(self.container.log) self.container.clear_commands() return self.package_list
def regenerate_manifests( self, regen_dict, image_owner='allenh1', image_name='ros_gentoo_base' ): info( "Pulling docker image '%s/%s:latest'..." % ( image_owner, image_name ) ) dock = Docker() dock.pull(image_owner, image_name) info('Running docker image...') info('Generating manifests...') dock.map_directory( '/home/%s/.gnupg' % os.getenv('USER'), '/root/.gnupg' ) dock.map_directory(self.repo.repo_dir, '/tmp/ros-overlay') for key in regen_dict.keys(): for pkg in regen_dict[key]: pkg_dir = '/tmp/ros-overlay/ros-{0}/{1}'.format(key, pkg) dock.add_bash_command('cd {0}'.format(pkg_dir)) dock.add_bash_command('repoman manifest') try: dock.run(show_cmd=True) except docker.errors.ContainerError: print(dock.log) raise
def pull_request(self, message, title, branch='master', remote='origin'): info('Forking repository if a fork does not exist...') self.github = Github(os.environ['SUPERFLORE_GITHUB_TOKEN']) self.gh_user = self.github.get_user() self.gh_upstream = self.github.get_repo( '%s/%s' % (self.repo_owner, self.repo_name)) # TODO(allenh1): Don't fork if you're authorized for repo forked_repo = self.gh_user.create_fork(self.gh_upstream) info('Pushing changes to fork...') self.git.remote('add', 'github', forked_repo.html_url) retry_on_exception( self.git.push, '-u', 'github', self.branch or branch, retry_msg='Could not push', error_msg='Error during push', sleep_secs=0.0, ) info('Filing pull-request...') pr_head = '%s:%s' % (self.gh_user.login, self.branch) pr = self.gh_upstream.create_pull(title=title, body=message, base=self.from_branch or branch, head=pr_head) ok('Successfully filed a pull request.') ok(' %s' % pr.html_url)
def commit_changes(self, distro): info('Adding changes...') if distro == 'all': commit_msg = 'regenerate all distros, {0}' self.repo.git.add('generated-recipes-*') self.repo.git.add('conf/ros-distro/include/*/*.inc') self.repo.git.add('files/*/cache.*') self.repo.git.add('files/*/rosdep-resolve.yaml') self.repo.git.add('files/*/newer-platform-components.list') self.repo.git.add('files/*/superflore-change-summary.txt') else: commit_msg = 'regenerate ros-{1}, {0}' self.repo.git.add('generated-recipes-{0}'.format(distro)) self.repo.git.add( 'conf/ros-distro/include/{0}/*.inc'.format(distro)) self.repo.git.add('files/{0}/cache.*'.format(distro)) self.repo.git.add('files/{0}/rosdep-resolve.yaml'.format(distro)) self.repo.git.add( 'files/{0}/newer-platform-components.list'.format(distro)) self.repo.git.add( 'files/{0}/superflore-change-summary.txt'.format(distro)) commit_msg = commit_msg.format(time.ctime(), distro) info('Committing to branch {0}...'.format(self.branch_name)) self.repo.git.commit(m='{0}'.format(commit_msg))
def commit_changes(self, distro): info('Adding changes...') self.repo.git.add(self.repo.repo_dir) if self.repo.git.status('--porcelain') == '': info('Nothing changed; no commit done') else: if self.branch_name: info('Committing to branch {0}...'.format(self.branch_name)) else: info('Committing to current branch') if distro == 'all': commit_msg = 'regenerate all distros, {0}' elif distro: commit_msg = 'regenerate ros-{1}, {0}' else: commit_msg = 'rosdistro sync, {0}' timestamp = os.getenv('SUPERFLORE_GENERATION_DATETIME', time.ctime()) commit_msg = commit_msg.format(timestamp, distro) self.repo.git.commit(m='{0}'.format(commit_msg))
def pull_request(self, message, title, branch='master', remote='origin'): info('Forking repository if a fork does not exist...') # TODO(allenh1): Don't fork if you're authorized for repo forked_repo = self.gh_user.create_fork(self.gh_upstream) info('Pushing changes to fork...') self.git.remote('add', 'github', forked_repo.html_url) self.git.push('-u', 'github', self.branch or 'master') info('Filing pull-request...') pr_head = '%s:%s' % (self.gh_user.login, self.branch) pr = self.gh_upstream.create_pull(title=title, body=message, base='master', head=pr_head) ok('Successfully filed a pull request.') ok(' %s' % pr.html_url)
def commit_changes(self, distro, commit_msg): info('Adding changes...') self.repo.git.add('meta-ros{0}-{1}/generated-recipes'.format( yoctoRecipe._get_ros_version(distro), distro)) self.repo.git.add('meta-ros{0}-{1}/conf/ros-distro/include/{1}/' 'generated/*.inc'.format( yoctoRecipe._get_ros_version(distro), distro)) self.repo.git.add('meta-ros{0}-{1}/files/{1}/generated/' 'rosdep-resolve.yaml'.format( yoctoRecipe._get_ros_version(distro), distro)) self.repo.git.add('meta-ros{0}-{1}/files/{1}/generated/' 'newer-platform-components.list'.format( yoctoRecipe._get_ros_version(distro), distro)) self.repo.git.add('meta-ros{0}-{1}/files/{1}/generated/' 'superflore-change-summary.txt'.format( yoctoRecipe._get_ros_version(distro), distro)) if self.repo.git.status('--porcelain') == '': info('Nothing changed; no commit done') else: if self.branch_name: info('Committing to branch {0}...'.format(self.branch_name)) else: info('Committing to current branch') self.repo.git.commit(m=commit_msg)