def symlink_distribution_unit_files(self, units, symlink_dir, progress_callback=None):
"""
Publishing distriubution unit involves publishing files underneath the unit.
Distribution is an aggregate unit with distribution files. This call
looksup each distribution unit and symlinks the files from the storage location
to working directory.
@param units
@type AssociatedUnit
@param symlink_dir: path of where we want the symlink to reside
@type symlink_dir str
@param progress_callback: callback to report progress info to publish_conduit
@type progress_callback: function
@return tuple of status and list of error messages if any occurred
@rtype (bool, [str])
"""
distro_progress_status = self.init_progress()
self.set_progress("distribution", distro_progress_status, progress_callback)
_LOG.info("Process symlinking distribution files with %s units to %s dir" % (len(units), symlink_dir))
errors = []
for u in units:
source_path_dir = u.storage_path
if not u.metadata.has_key('files'):
msg = "No distribution files found for unit %s" % u
_LOG.error(msg)
distro_files = u.metadata['files']
_LOG.info("Found %s distribution files to symlink" % len(distro_files))
distro_progress_status['items_total'] = len(distro_files)
distro_progress_status['items_left'] = len(distro_files)
for dfile in distro_files:
self.set_progress("distribution", distro_progress_status, progress_callback)
source_path = os.path.join(source_path_dir, dfile['relativepath'])
symlink_path = os.path.join(symlink_dir, dfile['relativepath'])
if not os.path.exists(source_path):
msg = "Source path: %s is missing" % source_path
errors.append((source_path, symlink_path, msg))
distro_progress_status['num_error'] += 1
distro_progress_status["items_left"] -= 1
continue
try:
if not util.create_symlink(source_path, symlink_path):
msg = "Unable to create symlink for: %s pointing to %s" % (symlink_path, source_path)
_LOG.error(msg)
errors.append((source_path, symlink_path, msg))
distro_progress_status['num_error'] += 1
distro_progress_status["items_left"] -= 1
continue
distro_progress_status['num_success'] += 1
except Exception, e:
tb_info = traceback.format_exc()
_LOG.error("%s" % tb_info)
_LOG.critical(e)
errors.append((source_path, symlink_path, str(e)))
distro_progress_status['num_error'] += 1
distro_progress_status["items_left"] -= 1
continue
distro_progress_status["items_left"] -= 1
def handle_symlinks(self, units, symlink_dir, progress_callback=None):
"""
@param units list of units that belong to the repo and should be published
@type units [AssociatedUnit]
@param symlink_dir where to create symlinks
@type symlink_dir str
@param progress_callback: callback to report progress info to publish_conduit
@type progress_callback: function
@return tuple of status and list of error messages if any occurred
@rtype (bool, [str])
"""
packages_progress_status = self.init_progress()
_LOG.info("handle_symlinks invoked with %s units to %s dir" % (len(units), symlink_dir))
self.set_progress("packages", packages_progress_status, progress_callback)
errors = []
packages_progress_status["items_total"] = len(units)
packages_progress_status["items_left"] = len(units)
for u in units:
self.set_progress("packages", packages_progress_status, progress_callback)
relpath = util.get_relpath_from_unit(u)
source_path = u.storage_path
symlink_path = os.path.join(symlink_dir, relpath)
if not os.path.exists(source_path):
msg = "Source path: %s is missing" % (source_path)
errors.append((source_path, symlink_path, msg))
packages_progress_status["num_error"] += 1
packages_progress_status["items_left"] -= 1
continue
_LOG.info("Unit exists at: %s we need to symlink to: %s" % (source_path, symlink_path))
try:
if not util.create_symlink(source_path, symlink_path):
msg = "Unable to create symlink for: %s pointing to %s" % (symlink_path, source_path)
_LOG.error(msg)
errors.append((source_path, symlink_path, msg))
packages_progress_status["num_error"] += 1
packages_progress_status["items_left"] -= 1
continue
packages_progress_status["num_success"] += 1
except Exception, e:
tb_info = traceback.format_exc()
_LOG.error("%s" % (tb_info))
_LOG.critical(e)
errors.append((source_path, symlink_path, str(e)))
packages_progress_status["num_error"] += 1
packages_progress_status["items_left"] -= 1
continue
packages_progress_status["items_left"] -= 1
def test_create_symlink(self):
target_dir = os.path.join(self.temp_dir, "a", "b", "c", "d", "e")
distributor = YumDistributor()
# Create an empty file to serve as the source_path
source_path = os.path.join(self.temp_dir, "some_test_file.txt")
open(source_path, "a")
symlink_path = os.path.join(self.temp_dir, "symlink_dir", "a", "b", "file_path.lnk")
# Confirm subdir of symlink_path doesn't exist
self.assertFalse(os.path.isdir(os.path.dirname(symlink_path)))
self.assertTrue(util.create_symlink(source_path, symlink_path))
# Confirm we created the subdir
self.assertTrue(os.path.isdir(os.path.dirname(symlink_path)))
self.assertTrue(os.path.exists(symlink_path))
self.assertTrue(os.path.islink(symlink_path))
# Verify the symlink points to the source_path
a = os.readlink(symlink_path)
self.assertEqual(a, source_path)
def symlink_distribution_unit_files(self, units, symlink_dir, publish_conduit, progress_callback=None):
"""
Publishing distriubution unit involves publishing files underneath the unit.
Distribution is an aggregate unit with distribution files. This call
looksup each distribution unit and symlinks the files from the storage location
to working directory.
@param units
@type AssociatedUnit
@param symlink_dir: path of where we want the symlink to reside
@type symlink_dir str
@param progress_callback: callback to report progress info to publish_conduit
@type progress_callback: function
@return tuple of status and list of error messages if any occurred
@rtype (bool, [str])
"""
distro_progress_status = self.init_progress()
self.set_progress("distribution", distro_progress_status, progress_callback)
_LOG.debug("Process symlinking distribution files with %s units to %s dir" % (len(units), symlink_dir))
# handle orphaned
existing_scratchpad = publish_conduit.get_scratchpad() or {}
scratchpad = self._handle_orphaned_distributions(units, symlink_dir, existing_scratchpad)
errors = []
for u in units:
source_path_dir = u.storage_path
if not u.metadata.has_key('files'):
msg = "No distribution files found for unit %s" % u
_LOG.error(msg)
distro_files = u.metadata['files']
_LOG.debug("Found %s distribution files to symlink" % len(distro_files))
distro_progress_status['items_total'] = len(distro_files)
distro_progress_status['items_left'] = len(distro_files)
# Lookup treeinfo file in the source location
src_treeinfo_path = None
for treeinfo in constants.TREE_INFO_LIST:
src_treeinfo_path = os.path.join(source_path_dir, treeinfo)
if os.path.exists(src_treeinfo_path):
# we found the treeinfo file
break
if src_treeinfo_path is not None:
# create a symlink from content location to repo location.
symlink_treeinfo_path = os.path.join(symlink_dir, treeinfo)
_LOG.debug("creating treeinfo symlink from %s to %s" % (src_treeinfo_path, symlink_treeinfo_path))
util.create_symlink(src_treeinfo_path, symlink_treeinfo_path)
published_distro_files = []
for dfile in distro_files:
self.set_progress("distribution", distro_progress_status, progress_callback)
source_path = os.path.join(source_path_dir, dfile['relativepath'])
symlink_path = os.path.join(symlink_dir, dfile['relativepath'])
if os.path.exists(symlink_path):
# path already exists, skip symlink
distro_progress_status["items_left"] -= 1
published_distro_files.append(symlink_path)
continue
if not os.path.exists(source_path):
msg = "Source path: %s is missing" % source_path
errors.append((source_path, symlink_path, msg))
distro_progress_status['num_error'] += 1
distro_progress_status["items_left"] -= 1
continue
try:
if not util.create_symlink(source_path, symlink_path):
msg = "Unable to create symlink for: %s pointing to %s" % (symlink_path, source_path)
_LOG.error(msg)
errors.append((source_path, symlink_path, msg))
distro_progress_status['num_error'] += 1
distro_progress_status["items_left"] -= 1
continue
distro_progress_status['num_success'] += 1
published_distro_files.append(symlink_path)
except Exception, e:
tb_info = traceback.format_exc()
_LOG.error("%s" % tb_info)
_LOG.critical(e)
errors.append((source_path, symlink_path, str(e)))
distro_progress_status['num_error'] += 1
distro_progress_status["items_left"] -= 1
continue
distro_progress_status["items_left"] -= 1
scratchpad.update({constants.PUBLISHED_DISTRIBUTION_FILES_KEY : {u.id : published_distro_files}})
def publish_repo(self, repo, publish_conduit, config):
summary = {}
details = {}
progress_status = {
"packages": {"state": "NOT_STARTED"},
"distribution": {"state": "NOT_STARTED"},
"metadata": {"state": "NOT_STARTED"},
"packagegroups": {"state": "NOT_STARTED"},
"publish_http": {"state": "NOT_STARTED"},
"publish_https": {"state": "NOT_STARTED"},
}
def progress_callback(type_id, status):
progress_status[type_id] = status
publish_conduit.set_progress(progress_status)
self.repo_working_dir = repo.working_dir
if self.canceled:
return publish_conduit.build_cancel_report(summary, details)
skip_list = config.get('skip') or []
# Determine Content in this repo
pkg_units = []
pkg_errors = []
if 'rpm' not in skip_list:
for type_id in [TYPE_ID_RPM, TYPE_ID_SRPM]:
criteria = UnitAssociationCriteria(type_ids=type_id,
unit_fields=['id', 'name', 'version', 'release', 'arch', 'epoch', '_storage_path', "checksum", "checksumtype" ])
pkg_units += publish_conduit.get_units(criteria=criteria)
drpm_units = []
if 'drpm' not in skip_list:
criteria = UnitAssociationCriteria(type_ids=TYPE_ID_DRPM)
drpm_units = publish_conduit.get_units(criteria=criteria)
pkg_units += drpm_units
# Create symlinks under repo.working_dir
pkg_status, pkg_errors = self.handle_symlinks(pkg_units, repo.working_dir, progress_callback)
if not pkg_status:
_LOG.error("Unable to publish %s items" % (len(pkg_errors)))
distro_errors = []
distro_units = []
if 'distribution' not in skip_list:
criteria = UnitAssociationCriteria(type_ids=TYPE_ID_DISTRO)
distro_units = publish_conduit.get_units(criteria=criteria)
# symlink distribution files if any under repo.working_dir
distro_status, distro_errors = self.symlink_distribution_unit_files(distro_units, repo.working_dir, publish_conduit, progress_callback)
if not distro_status:
_LOG.error("Unable to publish distribution tree %s items" % (len(distro_errors)))
updateinfo_xml_path = None
if 'erratum' not in skip_list:
criteria = UnitAssociationCriteria(type_ids=TYPE_ID_ERRATA)
errata_units = publish_conduit.get_units(criteria=criteria)
updateinfo_xml_path = updateinfo.updateinfo(errata_units, repo.working_dir)
if self.canceled:
return publish_conduit.build_cancel_report(summary, details)
groups_xml_path = None
existing_cats = []
existing_groups = []
if 'packagegroup' not in skip_list:
criteria = UnitAssociationCriteria(type_ids=[TYPE_ID_PKG_GROUP, TYPE_ID_PKG_CATEGORY])
existing_units = publish_conduit.get_units(criteria)
existing_groups = filter(lambda u : u.type_id in [TYPE_ID_PKG_GROUP], existing_units)
existing_cats = filter(lambda u : u.type_id in [TYPE_ID_PKG_CATEGORY], existing_units)
groups_xml_path = comps_util.write_comps_xml(repo.working_dir, existing_groups, existing_cats)
metadata_start_time = time.time()
# update/generate metadata for the published repo
self.use_createrepo = config.get('use_createrepo')
if self.use_createrepo:
metadata_status, metadata_errors = metadata.generate_metadata(
repo.working_dir, publish_conduit, config, progress_callback, groups_xml_path)
else:
metadata_status, metadata_errors = metadata.generate_yum_metadata(repo.id, repo.working_dir, publish_conduit, config,
progress_callback, is_cancelled=self.canceled, group_xml_path=groups_xml_path, updateinfo_xml_path=updateinfo_xml_path,
repo_scratchpad=publish_conduit.get_repo_scratchpad())
metadata_end_time = time.time()
relpath = self.get_repo_relative_path(repo, config)
if relpath.startswith("/"):
relpath = relpath[1:]
# Build the https and http publishing paths
https_publish_dir = self.get_https_publish_dir(config)
https_repo_publish_dir = os.path.join(https_publish_dir, relpath).rstrip('/')
http_publish_dir = self.get_http_publish_dir(config)
http_repo_publish_dir = os.path.join(http_publish_dir, relpath).rstrip('/')
# Clean up the old publish directories, if they exist.
scratchpad = publish_conduit.get_repo_scratchpad()
if OLD_REL_PATH_KEYWORD in scratchpad:
old_relative_path = scratchpad[OLD_REL_PATH_KEYWORD]
old_https_repo_publish_dir = os.path.join(https_publish_dir, old_relative_path)
if os.path.exists(old_https_repo_publish_dir):
util.remove_repo_publish_dir(https_publish_dir, old_https_repo_publish_dir)
old_http_repo_publish_dir = os.path.join(http_publish_dir, old_relative_path)
if os.path.exists(old_http_repo_publish_dir):
util.remove_repo_publish_dir(http_publish_dir, old_http_repo_publish_dir)
# Now write the current publish relative path to the scratch pad. This way, if the relative path
# changes before the next publish, we can clean up the old path.
scratchpad[OLD_REL_PATH_KEYWORD] = relpath
publish_conduit.set_repo_scratchpad(scratchpad)
# Handle publish link for HTTPS
if config.get("https"):
# Publish for HTTPS
self.set_progress("publish_https", {"state" : "IN_PROGRESS"}, progress_callback)
try:
_LOG.info("HTTPS Publishing repo <%s> to <%s>" % (repo.id, https_repo_publish_dir))
util.create_symlink(repo.working_dir, https_repo_publish_dir)
util.generate_listing_files(https_publish_dir, https_repo_publish_dir)
summary["https_publish_dir"] = https_repo_publish_dir
self.set_progress("publish_https", {"state" : "FINISHED"}, progress_callback)
except:
self.set_progress("publish_https", {"state" : "FAILED"}, progress_callback)
else:
self.set_progress("publish_https", {"state" : "SKIPPED"}, progress_callback)
if os.path.lexists(https_repo_publish_dir):
_LOG.debug("Removing link for %s since https is not set" % https_repo_publish_dir)
util.remove_repo_publish_dir(https_publish_dir, https_repo_publish_dir)
# Handle publish link for HTTP
if config.get("http"):
# Publish for HTTP
self.set_progress("publish_http", {"state" : "IN_PROGRESS"}, progress_callback)
try:
_LOG.info("HTTP Publishing repo <%s> to <%s>" % (repo.id, http_repo_publish_dir))
util.create_symlink(repo.working_dir, http_repo_publish_dir)
util.generate_listing_files(http_publish_dir, http_repo_publish_dir)
summary["http_publish_dir"] = http_repo_publish_dir
self.set_progress("publish_http", {"state" : "FINISHED"}, progress_callback)
except:
self.set_progress("publish_http", {"state" : "FAILED"}, progress_callback)
else:
self.set_progress("publish_http", {"state" : "SKIPPED"}, progress_callback)
if os.path.lexists(http_repo_publish_dir):
_LOG.debug("Removing link for %s since http is not set" % http_repo_publish_dir)
util.remove_repo_publish_dir(http_publish_dir, http_repo_publish_dir)
summary["num_package_units_attempted"] = len(pkg_units)
summary["num_package_units_published"] = len(pkg_units) - len(pkg_errors)
summary["num_package_units_errors"] = len(pkg_errors)
summary["num_distribution_units_attempted"] = len(distro_units)
summary["num_distribution_units_published"] = len(distro_units) - len(distro_errors)
summary["num_distribution_units_errors"] = len(distro_errors)
summary["num_package_groups_published"] = len(existing_groups)
summary["num_package_categories_published"] = len(existing_cats)
summary["relative_path"] = relpath
if metadata_status is False and not len(metadata_errors):
summary["skip_metadata_update"] = True
else:
summary["skip_metadata_update"] = False
details["errors"] = pkg_errors + distro_errors # metadata_errors
details['time_metadata_sec'] = metadata_end_time - metadata_start_time
# metadata generate skipped vs run
_LOG.info("Publish complete: summary = <%s>, details = <%s>" % (summary, details))
if details["errors"]:
return publish_conduit.build_failure_report(summary, details)
return publish_conduit.build_success_report(summary, details)
continue
distro_progress_status['num_success'] += 1
published_distro_files.append(symlink_path)
except Exception, e:
tb_info = traceback.format_exc()
_LOG.error("%s" % tb_info)
_LOG.critical(e)
errors.append((source_path, symlink_path, str(e)))
distro_progress_status['num_error'] += 1
distro_progress_status["items_left"] -= 1
continue
distro_progress_status["items_left"] -= 1
scratchpad.update({constants.PUBLISHED_DISTRIBUTION_FILES_KEY : {u.id : published_distro_files}})
# create the Packages symlink to the content dir, in the content dir
packages_symlink_path = os.path.join(symlink_dir, 'Packages')
if not util.create_symlink(symlink_dir, packages_symlink_path):
msg = 'Unable to create Packages symlink required for RHEL 5 distributions'
_LOG.error(msg)
errors.append((symlink_dir, packages_symlink_path, msg))
publish_conduit.set_scratchpad(scratchpad)
if errors:
distro_progress_status["error_details"] = errors
distro_progress_status["state"] = "FAILED"
self.set_progress("distribution", distro_progress_status, progress_callback)
return False, errors
distro_progress_status["state"] = "FINISHED"
self.set_progress("distribution", distro_progress_status, progress_callback)
return True, []
def _handle_orphaned_distributions(self, units, repo_working_dir, scratchpad):
distro_unit_ids = [u.id for u in units]
def publish_repo(self, repo, publish_conduit, config):
summary = {}
details = {}
progress_status = {
"packages": {"state": "NOT_STARTED"},
"distribution": {"state": "NOT_STARTED"},
"metadata": {"state": "NOT_STARTED"},
"packagegroups": {"state": "NOT_STARTED"},
"publish_http": {"state": "NOT_STARTED"},
"publish_https": {"state": "NOT_STARTED"},
}
def progress_callback(type_id, status):
progress_status[type_id] = status
publish_conduit.set_progress(progress_status)
self.repo_working_dir = repo.working_dir
if self.canceled:
return publish_conduit.build_failure_report(summary, details)
skip_list = config.get('skip') or []
# Determine Content in this repo
unfiltered_units = publish_conduit.get_units()
# filter compatible units
rpm_units = filter(lambda u : u.type_id in [TYPE_ID_RPM, TYPE_ID_SRPM], unfiltered_units)
drpm_units = filter(lambda u : u.type_id == TYPE_ID_DRPM, unfiltered_units)
rpm_errors = []
if 'rpm' not in skip_list:
_LOG.info("Publish on %s invoked. %s existing units, %s of which are supported to be published." \
% (repo.id, len(unfiltered_units), len(rpm_units)))
# Create symlinks under repo.working_dir
rpm_status, rpm_errors = self.handle_symlinks(rpm_units, repo.working_dir, progress_callback)
if not rpm_status:
_LOG.error("Unable to publish %s items" % (len(rpm_errors)))
drpm_errors = []
if 'drpm' not in skip_list:
_LOG.info("Publish on %s invoked. %s existing units, %s of which are supported to be published." \
% (repo.id, len(unfiltered_units), len(drpm_units)))
# Create symlinks under repo.working_dir
drpm_status, drpm_errors = self.handle_symlinks(drpm_units, repo.working_dir, progress_callback)
if not drpm_status:
_LOG.error("Unable to publish %s items" % (len(drpm_errors)))
pkg_errors = rpm_errors + drpm_errors
pkg_units = rpm_units + drpm_units
distro_errors = []
distro_units = filter(lambda u: u.type_id == TYPE_ID_DISTRO, unfiltered_units)
if 'distribution' not in skip_list:
# symlink distribution files if any under repo.working_dir
distro_status, distro_errors = self.symlink_distribution_unit_files(distro_units, repo.working_dir, progress_callback)
if not distro_status:
_LOG.error("Unable to publish distribution tree %s items" % (len(distro_errors)))
# update/generate metadata for the published repo
repo_scratchpad = publish_conduit.get_repo_scratchpad()
src_working_dir = ''
if repo_scratchpad.has_key("importer_working_dir"):
src_working_dir = repo_scratchpad['importer_working_dir']
if self.canceled:
return publish_conduit.build_failure_report(summary, details)
groups_xml_path = None
existing_cats = []
existing_groups = []
if 'packagegroup' not in skip_list:
criteria = UnitAssociationCriteria(type_ids=[TYPE_ID_PKG_GROUP, TYPE_ID_PKG_CATEGORY])
existing_units = publish_conduit.get_units(criteria)
existing_groups = filter(lambda u : u.type_id in [TYPE_ID_PKG_GROUP], existing_units)
existing_cats = filter(lambda u : u.type_id in [TYPE_ID_PKG_CATEGORY], existing_units)
groups_xml_path = comps_util.write_comps_xml(repo, existing_groups, existing_cats)
metadata_start_time = time.time()
self.copy_importer_repodata(src_working_dir, repo.working_dir)
metadata_status, metadata_errors = metadata.generate_metadata(
repo, publish_conduit, config, progress_callback, groups_xml_path)
metadata_end_time = time.time()
relpath = self.get_repo_relative_path(repo, config)
if relpath.startswith("/"):
relpath = relpath[1:]
#
# Handle publish link for HTTPS
#
https_publish_dir = self.get_https_publish_dir(config)
https_repo_publish_dir = os.path.join(https_publish_dir, relpath).rstrip('/')
if config.get("https"):
# Publish for HTTPS
self.set_progress("publish_https", {"state" : "IN_PROGRESS"}, progress_callback)
try:
_LOG.info("HTTPS Publishing repo <%s> to <%s>" % (repo.id, https_repo_publish_dir))
util.create_symlink(repo.working_dir, https_repo_publish_dir)
summary["https_publish_dir"] = https_repo_publish_dir
self.set_progress("publish_https", {"state" : "FINISHED"}, progress_callback)
except:
self.set_progress("publish_https", {"state" : "FAILED"}, progress_callback)
else:
self.set_progress("publish_https", {"state" : "SKIPPED"}, progress_callback)
if os.path.lexists(https_repo_publish_dir):
_LOG.debug("Removing link for %s since https is not set" % https_repo_publish_dir)
util.remove_symlink(https_publish_dir, https_repo_publish_dir)
#
# Handle publish link for HTTP
#
http_publish_dir = self.get_http_publish_dir(config)
http_repo_publish_dir = os.path.join(http_publish_dir, relpath).rstrip('/')
if config.get("http"):
# Publish for HTTP
self.set_progress("publish_http", {"state" : "IN_PROGRESS"}, progress_callback)
try:
_LOG.info("HTTP Publishing repo <%s> to <%s>" % (repo.id, http_repo_publish_dir))
util.create_symlink(repo.working_dir, http_repo_publish_dir)
summary["http_publish_dir"] = http_repo_publish_dir
self.set_progress("publish_http", {"state" : "FINISHED"}, progress_callback)
except:
self.set_progress("publish_http", {"state" : "FAILED"}, progress_callback)
else:
self.set_progress("publish_http", {"state" : "SKIPPED"}, progress_callback)
if os.path.lexists(http_repo_publish_dir):
_LOG.debug("Removing link for %s since http is not set" % http_repo_publish_dir)
util.remove_symlink(http_publish_dir, http_repo_publish_dir)
summary["num_package_units_attempted"] = len(pkg_units)
summary["num_package_units_published"] = len(pkg_units) - len(pkg_errors)
summary["num_package_units_errors"] = len(pkg_errors)
summary["num_distribution_units_attempted"] = len(distro_units)
summary["num_distribution_units_published"] = len(distro_units) - len(distro_errors)
summary["num_distribution_units_errors"] = len(distro_errors)
summary["num_package_groups_published"] = len(existing_groups)
summary["num_package_categories_published"] = len(existing_cats)
summary["relative_path"] = relpath
if metadata_status is False and not len(metadata_errors):
summary["skip_metadata_update"] = True
else:
summary["skip_metadata_update"] = False
details["errors"] = pkg_errors + distro_errors + metadata_errors
details['time_metadata_sec'] = metadata_end_time - metadata_start_time
# metadata generate skipped vs run
_LOG.info("Publish complete: summary = <%s>, details = <%s>" % (summary, details))
if details["errors"]:
return publish_conduit.build_failure_report(summary, details)
return publish_conduit.build_success_report(summary, details)
