def _export_distributions(self, units, symlink_dir, progress_callback=None):
"""
Export distriubution unit involves including files within the unit.
Distribution is an aggregate unit with distribution files. This call
looksup each distribution unit and symlinks the files from the storage location
to working directory.
@param units
@type AssociatedUnit
@param symlink_dir: path of where we want the symlink to reside
@type symlink_dir str
@param progress_callback: callback to report progress info to publish_conduit
@type progress_callback: function
@return tuple of status and list of error messages if any occurred
@rtype (bool, [str])
"""
distro_progress_status = self.init_progress()
self.set_progress("distribution", distro_progress_status, progress_callback)
_LOG.info("Process symlinking distribution files with %s units to %s dir" % (len(units), symlink_dir))
errors = []
for u in units:
source_path_dir = u.storage_path
if not u.metadata.has_key('files'):
msg = "No distribution files found for unit %s" % u
_LOG.error(msg)
distro_files = u.metadata['files']
_LOG.info("Found %s distribution files to symlink" % len(distro_files))
distro_progress_status['items_total'] = len(distro_files)
distro_progress_status['items_left'] = len(distro_files)
for dfile in distro_files:
self.set_progress("distribution", distro_progress_status, progress_callback)
source_path = os.path.join(source_path_dir, dfile['relativepath'])
symlink_path = os.path.join(symlink_dir, dfile['relativepath'])
if not os.path.exists(source_path):
msg = "Source path: %s is missing" % source_path
errors.append((source_path, symlink_path, msg))
distro_progress_status['num_error'] += 1
distro_progress_status["items_left"] -= 1
continue
try:
if not util.create_copy(source_path, symlink_path): #util.create_symlink(source_path, symlink_path):
msg = "Unable to create copy for: %s pointing to %s" % (symlink_path, source_path)
_LOG.error(msg)
errors.append((source_path, symlink_path, msg))
distro_progress_status['num_error'] += 1
distro_progress_status["items_left"] -= 1
continue
distro_progress_status['num_success'] += 1
except Exception, e:
tb_info = traceback.format_exc()
_LOG.error("%s" % tb_info)
_LOG.critical(e)
errors.append((source_path, symlink_path, str(e)))
distro_progress_status['num_error'] += 1
distro_progress_status["items_left"] -= 1
continue
distro_progress_status["items_left"] -= 1
def export_rpm(working_dir, rpm_units, progress_callback=None):
"""
This method takes a list of rpm units and exports them to the given working directory
:param working_dir: The full path to the directory to export the content to
:type working_dir: str
:param rpm_units: the list of rpm units to export
:type rpm_units: list of AssociatedUnit
:param progress_callback: callback to report progress info to publish_conduit
:type progress_callback: function
:return: tuple of status and list of error messages if any occurred
:rtype: ({}, [str])
"""
# get rpm units
progress_status = init_progress_report(len(rpm_units))
summary = {
'num_package_units_attempted': 0,
'num_package_units_exported': 0,
'num_package_units_errors': 0,
}
details = {'errors': {}}
errors = []
for unit in rpm_units:
set_progress(ids.TYPE_ID_RPM, progress_status, progress_callback)
source_path = unit.storage_path
destination_path = os.path.join(working_dir, yum_utils.get_relpath_from_unit(unit))
if not yum_utils.create_copy(source_path, destination_path):
msg = "Unable to copy %s to %s" % (source_path, destination_path)
_logger.error(msg)
errors.append(msg)
progress_status[constants.PROGRESS_NUM_ERROR_KEY] += 1
progress_status[constants.PROGRESS_ITEMS_LEFT_KEY] -= 1
continue
progress_status[constants.PROGRESS_NUM_SUCCESS_KEY] += 1
progress_status[constants.PROGRESS_ITEMS_LEFT_KEY] -= 1
summary["num_package_units_attempted"] += len(rpm_units)
summary["num_package_units_exported"] += len(rpm_units) - len(errors)
summary["num_package_units_errors"] += len(errors)
# If errors occurred, write them to details and set the state to failed.
if errors:
details['errors']['rpm_export'] = errors
progress_status[constants.PROGRESS_STATE_KEY] = constants.STATE_FAILED
progress_status[constants.PROGRESS_ERROR_DETAILS_KEY] = errors
set_progress(models.RPM.TYPE, progress_status, progress_callback)
return summary, details
progress_status[constants.PROGRESS_STATE_KEY] = constants.STATE_COMPLETE
set_progress(models.RPM.TYPE, progress_status, progress_callback)
return summary, details
def export_rpms(self, rpm_units, progress_callback=None):
"""
This call looksup each rpm units and exports to the working directory.
@param rpm_units
@type rpm_units list of AssociatedUnit to be exported
@param progress_callback: callback to report progress info to publish_conduit
@type progress_callback: function
@return tuple of status and list of error messages if any occurred
@rtype ({}, [str])
"""
# get rpm units
summary = {}
symlink_dir = self.repo_working_dir
packages_progress_status = self.init_progress()
if 'rpm' in self.skip:
packages_progress_status["state"] = "SKIPPED"
self.set_progress("rpms", packages_progress_status, progress_callback)
_LOG.info("rpm unit type in skip list [%s]; skipping export" % self.skip)
return summary, []
packages_progress_status["num_success"] = 0
packages_progress_status["items_left"] = len(rpm_units)
packages_progress_status["items_total"] = len(rpm_units)
errors = []
for u in rpm_units:
self.set_progress("rpms", packages_progress_status, progress_callback)
relpath = util.get_relpath_from_unit(u)
source_path = u.storage_path
symlink_path = os.path.join(symlink_dir, relpath)
if not os.path.exists(source_path):
msg = "Source path: %s is missing" % (source_path)
errors.append((source_path, symlink_path, msg))
packages_progress_status["num_error"] += 1
packages_progress_status["items_left"] -= 1
continue
_LOG.info("Unit exists at: %s we need to copy to: %s" % (source_path, symlink_path))
try:
if not util.create_copy(source_path, symlink_path):
msg = "Unable to create copy for: %s pointing to %s" % (symlink_path, source_path)
_LOG.error(msg)
errors.append((source_path, symlink_path, msg))
packages_progress_status["num_error"] += 1
packages_progress_status["items_left"] -= 1
continue
packages_progress_status["num_success"] += 1
except Exception, e:
tb_info = traceback.format_exc()
_LOG.error("%s" % (tb_info))
_LOG.critical(e)
errors.append((source_path, symlink_path, str(e)))
packages_progress_status["num_error"] += 1
packages_progress_status["items_left"] -= 1
continue
packages_progress_status["items_left"] -= 1
def export_distribution(working_dir, distribution_units, progress_callback=None):
"""
Export distribution unit involves including files within the unit.
Distribution is an aggregate unit with distribution files. This call
looks up each distribution unit and copies the files from the storage location
to working directory.
:param working_dir: The full path to the directory to export the content to
:type working_dir: str
:param distribution_units: The distribution units to export. These should be retrieved from the
publish conduit using a criteria.
:type distribution_units: list of AssociatedUnit
:param progress_callback: callback to report progress info to publish_conduit
:type progress_callback: function
:return: A tuple of the summary and the details, in that order
:rtype: (dict, dict)
"""
progress_status = init_progress_report()
set_progress(ids.TYPE_ID_DISTRO, progress_status, progress_callback)
summary = {}
details = {"errors": {}}
_logger.debug("exporting distribution files to %s dir" % working_dir)
errors = []
for unit in distribution_units:
source_path_dir = unit.storage_path
if "files" not in unit.metadata:
msg = "No distribution files found for unit %s" % unit
_logger.error(msg)
errors.append(msg)
continue
distro_files = unit.metadata["files"]
_logger.debug("Found %s distribution files to symlink" % len(distro_files))
progress_status[constants.PROGRESS_ITEMS_TOTAL_KEY] = len(distro_files)
progress_status[constants.PROGRESS_ITEMS_LEFT_KEY] = len(distro_files)
for dfile in distro_files:
set_progress(ids.TYPE_ID_DISTRO, progress_status, progress_callback)
source_path = os.path.join(source_path_dir, dfile["relativepath"])
destination_path = os.path.join(working_dir, dfile["relativepath"])
if not yum_utils.create_copy(source_path, destination_path):
msg = "Unable to copy %s to %s" % (source_path, destination_path)
_logger.error(msg)
errors.append(msg)
progress_status[constants.PROGRESS_NUM_ERROR_KEY] += 1
progress_status[constants.PROGRESS_ITEMS_LEFT_KEY] -= 1
continue
progress_status[constants.PROGRESS_NUM_SUCCESS_KEY] += 1
progress_status[constants.PROGRESS_ITEMS_LEFT_KEY] -= 1
if errors:
progress_status[constants.PROGRESS_STATE_KEY] = constants.STATE_FAILED
progress_status[constants.PROGRESS_ERROR_DETAILS_KEY] = errors
details["errors"]["distribution_errors"] = errors
else:
progress_status[constants.PROGRESS_STATE_KEY] = constants.STATE_COMPLETE
summary["num_distribution_units_attempted"] = len(distribution_units)
summary["num_distribution_units_exported"] = len(distribution_units) - len(errors)
summary["num_distribution_units_errors"] = len(errors)
set_progress(ids.TYPE_ID_DISTRO, progress_status, progress_callback)
return summary, details