def __exit__(self, *args):
# save the cache, if it exists
if self.filename:
info("Saving cached file '%s'" % self.filename)
self.cache_file = open(self.filename, 'wb')
pickle.dump(self.cache, self.cache_file)
self.cache_file.close()
def downloadArchive(self):
if os.path.exists(self.getArchiveName()):
info("using cached archive for package '%s'..." % self.name)
else:
info("downloading archive version for package '%s' from %s..." %
(self.name, self.src_uri))
urlretrieve(self.src_uri, self.getArchiveName())
def get_dependencies(
self, internal_depends, external_depends, is_native=False):
dependencies = set()
system_dependencies = set()
union_deps = internal_depends | external_depends
if len(union_deps) <= 0:
return dependencies, system_dependencies
for dep in union_deps:
if dep in internal_depends:
recipe = self.convert_to_oe_name(dep, is_native)
dependencies.add(recipe)
info('Internal dependency add: ' + recipe)
continue
try:
results = resolve_dep(dep, 'openembedded', self.distro)[0]
if not results:
yoctoRecipe.rosdep_cache[dep] = []
continue
for res in results:
recipe = self.convert_to_oe_name(res, is_native)
dependencies.add(recipe)
system_dependencies.add(self.convert_to_oe_name(res))
yoctoRecipe.rosdep_cache[dep].add(res)
info('External dependency add: ' + recipe)
except UnresolvedDependency:
info('Unresolved dependency: ' + dep)
if dep in yoctoRecipe.rosdep_cache:
cached_deps = yoctoRecipe.rosdep_cache[dep]
if cached_deps == set(['null']):
system_dependencies.add(dep)
recipe = dep + self.get_native_suffix(is_native)
dependencies.add(recipe)
msg = 'Failed to resolve (cached):'
warn('{0} {1}: {2}'.format(msg, dep, recipe))
elif cached_deps:
system_dependencies |= cached_deps
for d in cached_deps:
recipe = self.convert_to_oe_name(d, is_native)
dependencies.add(recipe)
msg = 'Resolved in OpenEmbedded (cached):'
info('{0} {1}: {2}'.format(msg, dep, recipe))
continue
oe_query = OpenEmbeddedLayersDB()
oe_query.query_recipe(self.convert_to_oe_name(dep))
if oe_query.exists():
recipe = self.convert_to_oe_name(oe_query.name, is_native)
dependencies.add(recipe)
oe_name = self.convert_to_oe_name(oe_query.name)
system_dependencies.add(oe_name)
yoctoRecipe.rosdep_cache[dep].add(oe_name)
info('Resolved in OpenEmbedded: ' + dep + ' as ' +
oe_query.name + ' in ' + oe_query.layer +
' as recipe ' + recipe)
else:
recipe = dep + self.get_native_suffix(is_native)
dependencies.add(recipe)
system_dependencies.add(dep)
yoctoRecipe.rosdep_cache[dep].add('null')
warn('Failed to resolve fully: ' + dep)
return dependencies, system_dependencies
def clean_ros_recipe_dirs(self, distro=None):
if distro:
info('Cleaning up recipes-ros-{0} directory...'.format(distro))
self.repo.git.rm('-rf', 'recipes-ros-{0}'.format(distro))
else:
info('Cleaning up recipes-ros-* directories...')
self.repo.git.rm('-rf', 'recipes-ros-*')
def __init__(self, repo_dir, do_clone, org='ros', repo='ros-overlay'):
self.repo = RepoInstance(
org, repo, repo_dir=repo_dir, do_clone=do_clone
)
self.branch_name = 'gentoo-bot-%s' % rand_ascii_str()
info('Creating new branch {0}...'.format(self.branch_name))
self.repo.create_branch(self.branch_name)
def __init__(
self, dir, do_clone, branch, org='ros', repo='meta-ros', from_branch=''
):
self.repo = RepoInstance(
org, repo, dir, do_clone, from_branch=from_branch)
self.branch_name = branch
info('Creating new branch {0}...'.format(self.branch_name))
self.repo.create_branch(self.branch_name)
def __enter__(self):
# load the initial cache, if it exists
if self.filename and os.path.isfile(self.filename):
info("Loading cached file '%s'" % self.filename)
self.cache_file = open(self.filename, 'rb')
self.cache = pickle.load(self.cache_file)
self.cache_file.close()
return self.cache
def clone(self, branch=None):
shutil.rmtree(self.repo_dir)
msg = 'Cloning repo {0}/{1}'.format(self.repo_owner, self.repo_name)
if self.repo_dir != self.repo_name:
msg += (' into directory {0}'.format(self.repo_dir))
msg += '...'
info(msg)
self.repo = Repo.clone_from(self.repo_url, self.repo_dir)
if branch:
self.git.checkout(branch)
def set_up_container():
d = Docker()
curr = os.getcwd()
try:
d.pull('allenh1', 'asw_test_env')
except:
d.build(os.path.realpath("../docker/Dockerfile"))
info("Using repo in directory '%s'" % os.path.realpath(".."))
d.map_directory(os.path.realpath(".."), repo_location)
return d
def commit_changes(self, distro):
info('Adding changes...')
self.repo.git.add(self.repo.repo_dir)
info('Committing to branch {0}...'.format(self.branch_name))
if distro == 'all':
commit_msg = 'regenerate all distros, {0}'
else:
commit_msg = 'regenerate ros-{1}, {0}'
commit_msg = commit_msg.format(time.ctime(), distro)
self.repo.git.commit(m='{0}'.format(commit_msg))
def clean_ros_recipe_dirs(self, distro=None):
if distro:
info(
'Cleaning up meta-ros{}-{}/generated-recipes directory...'
.format(yoctoRecipe._get_ros_version(distro), distro))
self.repo.git.rm('-rf', 'meta-ros{}-{}/generated-recipes'.format(
yoctoRecipe._get_ros_version(distro), distro))
else:
info('Cleaning up generated-recipes directories...')
self.repo.git.rm('-rf', 'generated-recipes')
def commit_changes(self, distro):
info('Adding changes...')
self.repo.git.add(self.repo.repo_dir)
info('Committing to branch {0}...'.format(self.branch_name))
commit_msg = {
'update': 'rosdistro sync, ',
'all': 'regenerate all distros, ',
'lunar': 'regenerate ros-lunar, ',
'indigo': 'regenerate ros-indigo, ',
'kinetic': 'regenerate ros-kinetic, ',
}[distro or 'update'] + time.ctime()
self.repo.git.commit(m='{0}'.format(commit_msg))
def commit_changes(self, distro, commit_msg):
info('Adding changes...')
self.repo.git.add('generated-recipes-{0}'.format(distro))
self.repo.git.add('conf/ros-distro/include/{0}/*.inc'.format(distro))
self.repo.git.add('files/{0}/cache.*'.format(distro))
self.repo.git.add('files/{0}/rosdep-resolve.yaml'.format(distro))
self.repo.git.add(
'files/{0}/newer-platform-components.list'.format(distro))
self.repo.git.add(
'files/{0}/superflore-change-summary.txt'.format(distro))
info('Committing to branch {0}...'.format(self.branch_name))
self.repo.git.commit(m=commit_msg)
def __enter__(self):
if self.arg_path:
try:
os.mkdir(self.arg_path)
except OSError as ex:
if ex.errno != errno.EEXIST:
raise
return self.arg_path
else:
self.temp_path = tempfile.mkdtemp()
info("Working in temporary directory %s" % self.temp_path)
return self.temp_path
def add_generated_files(self, distro):
info('Adding changes...')
self.repo.git.add('meta-ros{0}-{1}/generated-recipes'.format(
yoctoRecipe._get_ros_version(distro), distro))
self.repo.git.add('meta-ros{0}-{1}/conf/ros-distro/include/{1}/'
'generated/*.inc'.format(
yoctoRecipe._get_ros_version(distro), distro))
self.repo.git.add('meta-ros{0}-{1}/files/{1}/generated/'
'rosdep-resolve.yaml'.format(
yoctoRecipe._get_ros_version(distro), distro))
self.repo.git.add('meta-ros{0}-{1}/files/{1}/generated/'
'newer-platform-components.list'.format(
yoctoRecipe._get_ros_version(distro), distro))
def __exit__(self, *args):
if self.temp_path:
info("Cleaning up temporary directory %s" % self.temp_path)
try:
shutil.rmtree(self.temp_path)
except OSError as ex:
if ex.errno == errno.EPERM:
err("Failed to rmtree %s" % self.temp_path)
err("Escalating to sudo rm -rf %s" % self.temp_path)
subprocess.check_call(
('sudo rm -rf %s' % self.temp_path).split())
else:
raise
self.temp_path = None
def __init__(self,
repo_dir,
do_clone,
org='allenh1',
repo='meta-ros',
from_branch=''):
self.repo = RepoInstance(org,
repo,
repo_dir,
do_clone,
from_branch=from_branch)
self.branch_name = 'yocto-bot-%s' % rand_ascii_str()
info('Creating new branch {0}...'.format(self.branch_name))
self.repo.create_branch(self.branch_name)
def commit_changes(self, distro):
info('Adding changes...')
if distro == 'all' or distro == 'update':
self.repo.git.add('recipes-ros-*')
else:
self.repo.git.add('recipes-ros-{0}'.format(distro))
commit_msg = {
'update': 'rosdistro sync, {0}',
'all': 'regenerate all distros, {0}',
'lunar': 'regenerate ros-lunar, {0}',
'indigo': 'regenerate ros-indigo, {0}',
'kinetic': 'regenerate ros-kinetic, {0}',
}[distro].format(time.ctime())
info('Committing to branch {0}...'.format(self.branch_name))
self.repo.git.commit(m='{0}'.format(commit_msg))
def clean_ros_recipe_dirs(self, distro):
# superflore-change-summary.txt is no longer being generated since:
# https://github.com/ros-infrastructure/superflore/pull/273
# but remove it here to make sure it gets deleted when new distro
# release is being generated
files = 'meta-ros{0}-{1}/generated-recipes '\
'meta-ros{0}-{1}/conf/ros-distro/include/{1}/generated '\
'meta-ros{0}-{1}/files/{1}/generated/'\
'newer-platform-components.list '\
'meta-ros{0}-{1}/files/{1}/generated/rosdep-resolve.yaml '\
'meta-ros{0}-{1}/files/{1}/generated/'\
'superflore-change-summary.txt '.format(
yoctoRecipe._get_ros_version(distro), distro)
info('Cleaning up:\n{0}'.format(files))
self.repo.git.rm('-rf', '--ignore-unmatch', files.split())
def run(self, rm=True, show_cmd=False):
cmd_string = "bash -c '"
for i, bash_cmd in enumerate(self.bash_cmds):
cmd_string += bash_cmd
if i != len(self.bash_cmds) - 1:
cmd_string += ' && '
cmd_string += "'"
msg = "Running container with command string '%s'..."
info(msg % cmd_string)
self.client.containers.run(
image=self.image,
remove=rm,
command=cmd_string,
volumes=self.directory_map,
)
ok("Docker container exited.")
def commit_changes(self, distro, commit_msg):
info('Commit changes...')
if self.repo.git.status('--porcelain') == '':
info('Nothing changed; no commit done')
else:
if self.branch_name:
info('Committing to branch {0}...'.format(self.branch_name))
else:
info('Committing to current branch')
self.repo.git.commit(m=commit_msg)
def __init__(self,
repo_dir,
do_clone,
org='ros',
repo='ros-overlay',
from_branch='',
new_branch=True):
self.repo = RepoInstance(org,
repo,
repo_dir=repo_dir,
do_clone=do_clone,
from_branch=from_branch)
if new_branch:
self.branch_name = 'gentoo-bot-%s' % rand_ascii_str()
info('Creating new branch {0}...'.format(self.branch_name))
self.repo.create_branch(self.branch_name)
else:
self.branch_name = None
def commit_changes(self, distro):
info('Adding changes...')
if not distro or distro == 'all' or distro == 'update':
self.repo.git.add('recipes-ros-*')
else:
self.repo.git.add('recipes-ros-{0}'.format(distro))
values = {
'update': 'rosdistro sync, {0}',
'all': 'regenerate all distros, {0}'
}
values.update( dict([
(d, 'regenerate ros-%s, {0}' % d) \
for d in ros1_distros + ros2_distros
])
)
commit_msg = values[distro].format(time.ctime())
info('Committing to branch {0}...'.format(self.branch_name))
self.repo.git.commit(m='{0}'.format(commit_msg))
def run(self, verbose=True, log_file=None):
# TODO(allenh1): add the ability to check out a non-master
# branch of the overlay (for CI).
info('testing gentoo package integrity')
for pkg in sorted(self.package_list.keys()):
self.container.add_bash_command('emaint sync -r ros-overlay')
self.container.add_bash_command('emerge %s' % pkg)
try:
self.container.run(rm=True,
show_cmd=True,
privileged=True,
log_file=log_file)
self.package_list[pkg] = 'building'
ok(" '%s': building" % pkg)
except ContainerError:
self.package_list[pkg] = 'failing'
err(" '%s': failing" % pkg)
if verbose:
print(self.container.log)
self.container.clear_commands()
return self.package_list
def regenerate_manifests(
self, regen_dict, image_owner='allenh1', image_name='ros_gentoo_base'
):
info(
"Pulling docker image '%s/%s:latest'..." % (
image_owner, image_name
)
)
dock = Docker()
dock.pull(image_owner, image_name)
info('Running docker image...')
info('Generating manifests...')
dock.map_directory(
'/home/%s/.gnupg' % os.getenv('USER'),
'/root/.gnupg'
)
dock.map_directory(self.repo.repo_dir, '/tmp/ros-overlay')
for key in regen_dict.keys():
for pkg in regen_dict[key]:
pkg_dir = '/tmp/ros-overlay/ros-{0}/{1}'.format(key, pkg)
dock.add_bash_command('cd {0}'.format(pkg_dir))
dock.add_bash_command('repoman manifest')
try:
dock.run(show_cmd=True)
except docker.errors.ContainerError:
print(dock.log)
raise
def pull_request(self, message, title, branch='master', remote='origin'):
info('Forking repository if a fork does not exist...')
self.github = Github(os.environ['SUPERFLORE_GITHUB_TOKEN'])
self.gh_user = self.github.get_user()
self.gh_upstream = self.github.get_repo(
'%s/%s' % (self.repo_owner, self.repo_name))
# TODO(allenh1): Don't fork if you're authorized for repo
forked_repo = self.gh_user.create_fork(self.gh_upstream)
info('Pushing changes to fork...')
self.git.remote('add', 'github', forked_repo.html_url)
retry_on_exception(
self.git.push,
'-u',
'github',
self.branch or branch,
retry_msg='Could not push',
error_msg='Error during push',
sleep_secs=0.0,
)
info('Filing pull-request...')
pr_head = '%s:%s' % (self.gh_user.login, self.branch)
pr = self.gh_upstream.create_pull(title=title,
body=message,
base=self.from_branch or branch,
head=pr_head)
ok('Successfully filed a pull request.')
ok(' %s' % pr.html_url)
def commit_changes(self, distro):
info('Adding changes...')
if distro == 'all':
commit_msg = 'regenerate all distros, {0}'
self.repo.git.add('generated-recipes-*')
self.repo.git.add('conf/ros-distro/include/*/*.inc')
self.repo.git.add('files/*/cache.*')
self.repo.git.add('files/*/rosdep-resolve.yaml')
self.repo.git.add('files/*/newer-platform-components.list')
self.repo.git.add('files/*/superflore-change-summary.txt')
else:
commit_msg = 'regenerate ros-{1}, {0}'
self.repo.git.add('generated-recipes-{0}'.format(distro))
self.repo.git.add(
'conf/ros-distro/include/{0}/*.inc'.format(distro))
self.repo.git.add('files/{0}/cache.*'.format(distro))
self.repo.git.add('files/{0}/rosdep-resolve.yaml'.format(distro))
self.repo.git.add(
'files/{0}/newer-platform-components.list'.format(distro))
self.repo.git.add(
'files/{0}/superflore-change-summary.txt'.format(distro))
commit_msg = commit_msg.format(time.ctime(), distro)
info('Committing to branch {0}...'.format(self.branch_name))
self.repo.git.commit(m='{0}'.format(commit_msg))
def commit_changes(self, distro):
info('Adding changes...')
self.repo.git.add(self.repo.repo_dir)
if self.repo.git.status('--porcelain') == '':
info('Nothing changed; no commit done')
else:
if self.branch_name:
info('Committing to branch {0}...'.format(self.branch_name))
else:
info('Committing to current branch')
if distro == 'all':
commit_msg = 'regenerate all distros, {0}'
elif distro:
commit_msg = 'regenerate ros-{1}, {0}'
else:
commit_msg = 'rosdistro sync, {0}'
timestamp = os.getenv('SUPERFLORE_GENERATION_DATETIME',
time.ctime())
commit_msg = commit_msg.format(timestamp, distro)
self.repo.git.commit(m='{0}'.format(commit_msg))
def pull_request(self, message, title, branch='master', remote='origin'):
info('Forking repository if a fork does not exist...')
# TODO(allenh1): Don't fork if you're authorized for repo
forked_repo = self.gh_user.create_fork(self.gh_upstream)
info('Pushing changes to fork...')
self.git.remote('add', 'github', forked_repo.html_url)
self.git.push('-u', 'github', self.branch or 'master')
info('Filing pull-request...')
pr_head = '%s:%s' % (self.gh_user.login, self.branch)
pr = self.gh_upstream.create_pull(title=title,
body=message,
base='master',
head=pr_head)
ok('Successfully filed a pull request.')
ok(' %s' % pr.html_url)
def commit_changes(self, distro, commit_msg):
info('Adding changes...')
self.repo.git.add('meta-ros{0}-{1}/generated-recipes'.format(
yoctoRecipe._get_ros_version(distro), distro))
self.repo.git.add('meta-ros{0}-{1}/conf/ros-distro/include/{1}/'
'generated/*.inc'.format(
yoctoRecipe._get_ros_version(distro), distro))
self.repo.git.add('meta-ros{0}-{1}/files/{1}/generated/'
'rosdep-resolve.yaml'.format(
yoctoRecipe._get_ros_version(distro), distro))
self.repo.git.add('meta-ros{0}-{1}/files/{1}/generated/'
'newer-platform-components.list'.format(
yoctoRecipe._get_ros_version(distro), distro))
self.repo.git.add('meta-ros{0}-{1}/files/{1}/generated/'
'superflore-change-summary.txt'.format(
yoctoRecipe._get_ros_version(distro), distro))
if self.repo.git.status('--porcelain') == '':
info('Nothing changed; no commit done')
else:
if self.branch_name:
info('Committing to branch {0}...'.format(self.branch_name))
else:
info('Committing to current branch')
self.repo.git.commit(m=commit_msg)
