def make_snapshot_repo(self, repo_dir):
"""Create a snapshot of the current state of the task library"""
# This should only run if we are missing repodata in the rpms path
# since this should normally be updated when new tasks are uploaded
src_meta = os.path.join(self.rpmspath, 'repodata')
if not os.path.isdir(src_meta):
log.info("Task library repodata missing, generating...")
self.update_repo()
dst_meta = os.path.join(repo_dir, 'repodata')
if os.path.isdir(dst_meta):
log.info("Destination repodata already exists, skipping snapshot")
else:
# Copy updated repo to recipe specific repo
log.debug("Generating task library snapshot")
with Flock(self.rpmspath):
self._link_rpms(repo_dir)
shutil.copytree(src_meta, dst_meta)
def unlink_rpm(self, rpm_name):
"""
Ensures an RPM is no longer present in the task library
"""
with Flock(self.rpmspath):
self._unlink_locked_rpm(rpm_name)
def update_repo(self):
"""Update the task library yum repo metadata"""
with Flock(self.rpmspath):
self._update_locked_repo()
class TaskLibrary(object):
@property
def rpmspath(self):
# Lazy lookup so module can be imported prior to configuration
return get("basepath.rpms")
def get_rpm_path(self, rpm_name):
return os.path.join(self.rpmspath, rpm_name)
def _unlink_locked_rpms(self, rpm_names):
# Internal call that assumes the flock is already held
for rpm_name in rpm_names:
unlink_ignore(self.get_rpm_path(rpm_name))
def _unlink_locked_rpm(self, rpm_name):
# Internal call that assumes the flock is already held
self._unlink_locked_rpms([rpm_name])
def unlink_rpm(self, rpm_name):
"""
Ensures an RPM is no longer present in the task library
"""
with Flock(self.rpmspath):
self._unlink_locked_rpm(rpm_name)
def _update_locked_repo(self):
# Internal call that assumes the flock is already held
# If the createrepo command crashes for some reason it may leave behind
# its work directories. createrepo refuses to run if .olddata exists,
# createrepo_c refuses to run if .repodata exists.
workdirs = [
os.path.join(self.rpmspath, dirname)
for dirname in ['.repodata', '.olddata']
]
for workdir in workdirs:
if os.path.exists(workdir):
log.warn('Removing stale createrepo directory %s', workdir)
shutil.rmtree(workdir, ignore_errors=True)
# Removed --baseurl, if upgrading you will need to manually
# delete repodata directory before this will work correctly.
command, returncode, out, err = run_createrepo(cwd=self.rpmspath,
update=True)
if out:
log.debug("stdout from %s: %s", command, out)
if err:
log.warn("stderr from %s: %s", command, err)
if returncode != 0:
if returncode < 0:
msg = '%s killed with signal %s' % (command, -returncode)
else:
msg = '%s failed with exit status %s' % (command, returncode)
if err:
msg = '%s\n%s' % (msg, err)
raise RuntimeError(msg)
def update_repo(self):
"""Update the task library yum repo metadata"""
with Flock(self.rpmspath):
self._update_locked_repo()
def _all_rpms(self):
"""Iterator over the task RPMs currently on disk"""
basepath = self.rpmspath
for name in os.listdir(basepath):
if not name.endswith("rpm"):
continue
srcpath = os.path.join(basepath, name)
if os.path.isdir(srcpath):
continue
yield srcpath, name
def _link_rpms(self, dst):
"""Hardlink the task rpms into dst"""
makedirs_ignore(dst, 0755)
for srcpath, name in self._all_rpms():
dstpath = os.path.join(dst, name)
unlink_ignore(dstpath)
os.link(srcpath, dstpath)
def make_snapshot_repo(self, repo_dir):
"""Create a snapshot of the current state of the task library"""
# This should only run if we are missing repodata in the rpms path
# since this should normally be updated when new tasks are uploaded
src_meta = os.path.join(self.rpmspath, 'repodata')
if not os.path.isdir(src_meta):
log.info("Task library repodata missing, generating...")
self.update_repo()
dst_meta = os.path.join(repo_dir, 'repodata')
if os.path.isdir(dst_meta):
log.info("Destination repodata already exists, skipping snapshot")
else:
# Copy updated repo to recipe specific repo
log.debug("Generating task library snapshot")
with Flock(self.rpmspath):
self._link_rpms(repo_dir)
shutil.copytree(src_meta, dst_meta)
def update_task(self, rpm_name, write_rpm):
tasks = self.update_tasks([(rpm_name, write_rpm)])
return tasks[0]
def update_tasks(self, rpm_names_write_rpm):
"""Updates the the task rpm library
rpm_names_write_rpm is a list of two element tuples,
where the first element is the name of the rpm to be written, and
the second element is callable that takes a file object as its
only arg and writes to that file object.
write_rpm must be a callable that takes a file object as its
sole argument and populates it with the raw task RPM contents
Expects to be called in a transaction, and for that transaction
to be rolled back if an exception is thrown.
"""
# XXX (ncoghlan): How do we get rid of that assumption about the
# transaction handling? Assuming we're *not* already in a transaction
# won't work either.
to_sync = []
try:
for rpm_name, write_rpm in rpm_names_write_rpm:
rpm_path = self.get_rpm_path(rpm_name)
upgrade = AtomicFileReplacement(rpm_path)
to_sync.append((
rpm_name,
upgrade,
))
f = upgrade.create_temp()
write_rpm(f)
f.flush()
except Exception, e:
log.error('Error: Failed to copy task %s, aborting.' % rpm_name)
for __, atomic_file in to_sync:
atomic_file.destroy_temp()
raise
old_rpms = []
new_tasks = []
try:
with Flock(self.rpmspath):
for rpm_name, atomic_file in to_sync:
f = atomic_file.temp_file
f.seek(0)
task, downgrade = Task.create_from_taskinfo(
self.read_taskinfo(f))
old_rpm_name = task.rpm
task.rpm = rpm_name
if old_rpm_name:
old_rpms.append(old_rpm_name)
atomic_file.replace_dest()
new_tasks.append(task)
try:
self._update_locked_repo()
except:
# We assume the current transaction is going to be rolled back,
# so the Task possibly defined above, or changes to an existing
# task, will never by written to the database (even if it was
# the _update_locked_repo() call that failed).
# Accordingly, we also throw away the newly created RPMs.
log.error('Failed to update task library repo, aborting')
self._unlink_locked_rpms([task.rpm for task in new_tasks])
raise
# Since it existed when we called _update_locked_repo()
# metadata, albeit not as the latest version.
# However, it's too expensive (several seconds of IO
# with the task repo locked) to do it twice for every
# task update, so we rely on the fact that tasks are
# referenced by name rather than requesting specific
# versions, and thus will always grab the latest.
self._unlink_locked_rpms(old_rpms)
# if this is a downgrade, we run createrepo once more
# so that the metadata doesn't contain the record for the
# now unlinked newer version of the task
if downgrade:
self._update_locked_repo()
finally:
# Some or all of these may have already been destroyed
for __, atomic_file in to_sync:
atomic_file.destroy_temp()
return new_tasks