def process_unit(self, unit):
"""
Link the unit to the content directory and the package_dir
:param unit: The unit to process
:type unit: pulp.plugins.model.Unit
"""
source_path = unit.storage_path
relative_path = util.get_relpath_from_unit(unit)
destination_path = os.path.join(self.get_working_dir(), relative_path)
self._create_symlink(source_path, destination_path)
filename = unit.unit_key['name'] + '-' + unit.unit_key['version'] + '-' + \
unit.unit_key['release'] + '.' + unit.unit_key['arch'] + '.json'
path = os.path.join(self.get_working_dir(), filename)
# Remove all keys that start with an underscore, like _id and _ns
for key_to_remove in filter(lambda key: key[0] == '_', unit.metadata.keys()):
unit.metadata.pop(key_to_remove)
# repodata will be regenerated on import, so remove it as well
if 'repodata' in unit.metadata:
unit.metadata.pop('repodata')
dict_to_write = {'unit_key': unit.unit_key, 'unit_metadata': unit.metadata}
with open(path, 'w') as f:
json.dump(dict_to_write, f)
def check_all_and_associate(wanted, sync_conduit):
"""
Given a set of unit keys as namedtuples, this function checks if a unit
already exists in Pulp and returns the set of tuples that were not
found. This checks for the unit in the db as well as for the actual file
on the filesystem. If a unit exists in the db and the filesystem, this function
also associates the unit to the given repo. Note that the check for the actual file
is performed only for the supported unit types.
:param wanted: iterable of units as namedtuples
:type wanted: iterable
:param sync_conduit: repo sync conduit
:type sync_conduit: pulp.plugins.conduits.repo_sync.RepoSync
:return: set of unit keys as namedtuples, identifying which of the
named tuples received as input were not found on the server.
:rtype: set
"""
sorted_units = _sort_by_type(wanted)
for unit_type, values in sorted_units.iteritems():
model = models.TYPE_MAP[unit_type]
unit_fields = model.UNIT_KEY_NAMES + ('_storage_path', 'filename')
rpm_srpm_drpm = unit_type in (models.RPM.TYPE, models.SRPM.TYPE, models.DRPM.TYPE)
rpm_or_srpm = unit_type in (models.RPM.TYPE, models.SRPM.TYPE)
unit_keys_generator = (unit._asdict() for unit in values.copy())
for unit in get_all_existing_units(unit_keys_generator, unit_fields, unit_type,
sync_conduit.search_all_units):
# For RPMs, SRPMs and DRPMs, also check if the file exists on the filesystem.
# If not, we do not want to skip downloading the unit.
if rpm_srpm_drpm:
if unit.storage_path is None or not os.path.isfile(unit.storage_path):
continue
# Since the unit is already downloaded, call respective sync_conduit calls to import
# the unit in given repository.
if rpm_or_srpm:
unit_key = unit.unit_key
rpm_or_srpm_unit = model(unit_key['name'], unit_key['epoch'], unit_key['version'],
unit_key['release'], unit_key['arch'],
unit_key['checksumtype'], unit_key['checksum'],
unit.metadata)
relative_path = rpm_or_srpm_unit.relative_path
else:
relative_path = get_relpath_from_unit(unit)
downloaded_unit = sync_conduit.init_unit(unit_type, unit.unit_key,
unit.metadata, relative_path)
# 1125388 - make sure we keep storage_path on the new unit model obj
downloaded_unit.storage_path = unit.storage_path
sync_conduit.save_unit(downloaded_unit)
# Discard already downloaded unit from the return value.
named_tuple = model(metadata=unit.metadata, **unit.unit_key).as_named_tuple
values.discard(named_tuple)
ret = set()
ret.update(*sorted_units.values())
return ret
def export_rpm(working_dir, rpm_units, progress_callback=None):
"""
This method takes a list of rpm units and exports them to the given working directory
:param working_dir: The full path to the directory to export the content to
:type working_dir: str
:param rpm_units: the list of rpm units to export
:type rpm_units: list of AssociatedUnit
:param progress_callback: callback to report progress info to publish_conduit
:type progress_callback: function
:return: tuple of status and list of error messages if any occurred
:rtype: ({}, [str])
"""
# get rpm units
progress_status = init_progress_report(len(rpm_units))
summary = {
'num_package_units_attempted': 0,
'num_package_units_exported': 0,
'num_package_units_errors': 0,
}
details = {'errors': {}}
errors = []
for unit in rpm_units:
set_progress(ids.TYPE_ID_RPM, progress_status, progress_callback)
source_path = unit.storage_path
destination_path = os.path.join(working_dir, yum_utils.get_relpath_from_unit(unit))
if not yum_utils.create_copy(source_path, destination_path):
msg = "Unable to copy %s to %s" % (source_path, destination_path)
_logger.error(msg)
errors.append(msg)
progress_status[constants.PROGRESS_NUM_ERROR_KEY] += 1
progress_status[constants.PROGRESS_ITEMS_LEFT_KEY] -= 1
continue
progress_status[constants.PROGRESS_NUM_SUCCESS_KEY] += 1
progress_status[constants.PROGRESS_ITEMS_LEFT_KEY] -= 1
summary["num_package_units_attempted"] += len(rpm_units)
summary["num_package_units_exported"] += len(rpm_units) - len(errors)
summary["num_package_units_errors"] += len(errors)
# If errors occurred, write them to details and set the state to failed.
if errors:
details['errors']['rpm_export'] = errors
progress_status[constants.PROGRESS_STATE_KEY] = constants.STATE_FAILED
progress_status[constants.PROGRESS_ERROR_DETAILS_KEY] = errors
set_progress(models.RPM.TYPE, progress_status, progress_callback)
return summary, details
progress_status[constants.PROGRESS_STATE_KEY] = constants.STATE_COMPLETE
set_progress(models.RPM.TYPE, progress_status, progress_callback)
return summary, details
def export_rpms(self, rpm_units, progress_callback=None):
"""
This call looksup each rpm units and exports to the working directory.
@param rpm_units
@type rpm_units list of AssociatedUnit to be exported
@param progress_callback: callback to report progress info to publish_conduit
@type progress_callback: function
@return tuple of status and list of error messages if any occurred
@rtype ({}, [str])
"""
# get rpm units
summary = {}
symlink_dir = self.repo_working_dir
packages_progress_status = self.init_progress()
if 'rpm' in self.skip:
packages_progress_status["state"] = "SKIPPED"
self.set_progress("rpms", packages_progress_status, progress_callback)
_LOG.info("rpm unit type in skip list [%s]; skipping export" % self.skip)
return summary, []
packages_progress_status["num_success"] = 0
packages_progress_status["items_left"] = len(rpm_units)
packages_progress_status["items_total"] = len(rpm_units)
errors = []
for u in rpm_units:
self.set_progress("rpms", packages_progress_status, progress_callback)
relpath = util.get_relpath_from_unit(u)
source_path = u.storage_path
symlink_path = os.path.join(symlink_dir, relpath)
if not os.path.exists(source_path):
msg = "Source path: %s is missing" % (source_path)
errors.append((source_path, symlink_path, msg))
packages_progress_status["num_error"] += 1
packages_progress_status["items_left"] -= 1
continue
_LOG.info("Unit exists at: %s we need to copy to: %s" % (source_path, symlink_path))
try:
if not util.create_copy(source_path, symlink_path):
msg = "Unable to create copy for: %s pointing to %s" % (symlink_path, source_path)
_LOG.error(msg)
errors.append((source_path, symlink_path, msg))
packages_progress_status["num_error"] += 1
packages_progress_status["items_left"] -= 1
continue
packages_progress_status["num_success"] += 1
except Exception, e:
tb_info = traceback.format_exc()
_LOG.error("%s" % (tb_info))
_LOG.critical(e)
errors.append((source_path, symlink_path, str(e)))
packages_progress_status["num_error"] += 1
packages_progress_status["items_left"] -= 1
continue
packages_progress_status["items_left"] -= 1
def test_get_relpath_from_unit(self):
distributor = YumDistributor()
test_unit = Unit("rpm", "unit_key", {}, "")
test_unit.unit_key = {"fileName" : "test_1"}
rel_path = util.get_relpath_from_unit(test_unit)
self.assertEqual(rel_path, "test_1")
test_unit.unit_key = {}
test_unit.storage_path = "test_0"
rel_path = util.get_relpath_from_unit(test_unit)
self.assertEqual(rel_path, "test_0")
test_unit.metadata["filename"] = "test_2"
rel_path = util.get_relpath_from_unit(test_unit)
self.assertEqual(rel_path, "test_2")
test_unit.metadata["relativepath"] = "test_3"
rel_path = util.get_relpath_from_unit(test_unit)
self.assertEqual(rel_path, "test_3")
def test_get_relpath_from_unit(self):
unit = Unit(type_id='random',
unit_key={"filename": "test_unit_key_filename"},
metadata={"filename": "test_metadata_filename"},
storage_path="/test/storage/path")
self.assertEqual(util.get_relpath_from_unit(unit), "test_metadata_filename")
unit = Unit(type_id='random',
unit_key={"filename": "test_unit_key_filename"},
metadata={},
storage_path="/test/storage/path")
self.assertEqual(util.get_relpath_from_unit(unit), "test_unit_key_filename")
unit = Unit(type_id='random',
unit_key={"fileName": "test_unit_key_fileName"},
metadata={},
storage_path="/test/storage/path")
self.assertEqual(util.get_relpath_from_unit(unit), "test_unit_key_fileName")
unit = Unit(type_id='random',
unit_key={},
metadata={},
storage_path="/test/storage/path")
self.assertEqual(util.get_relpath_from_unit(unit), "path")
def handle_symlinks(self, units, symlink_dir, progress_callback=None):
"""
@param units list of units that belong to the repo and should be published
@type units [AssociatedUnit]
@param symlink_dir where to create symlinks
@type symlink_dir str
@param progress_callback: callback to report progress info to publish_conduit
@type progress_callback: function
@return tuple of status and list of error messages if any occurred
@rtype (bool, [str])
"""
packages_progress_status = self.init_progress()
_LOG.info("handle_symlinks invoked with %s units to %s dir" % (len(units), symlink_dir))
self.set_progress("packages", packages_progress_status, progress_callback)
errors = []
packages_progress_status["items_total"] = len(units)
packages_progress_status["items_left"] = len(units)
for u in units:
self.set_progress("packages", packages_progress_status, progress_callback)
relpath = util.get_relpath_from_unit(u)
source_path = u.storage_path
symlink_path = os.path.join(symlink_dir, relpath)
if not os.path.exists(source_path):
msg = "Source path: %s is missing" % (source_path)
errors.append((source_path, symlink_path, msg))
packages_progress_status["num_error"] += 1
packages_progress_status["items_left"] -= 1
continue
_LOG.info("Unit exists at: %s we need to symlink to: %s" % (source_path, symlink_path))
try:
if not util.create_symlink(source_path, symlink_path):
msg = "Unable to create symlink for: %s pointing to %s" % (symlink_path, source_path)
_LOG.error(msg)
errors.append((source_path, symlink_path, msg))
packages_progress_status["num_error"] += 1
packages_progress_status["items_left"] -= 1
continue
packages_progress_status["num_success"] += 1
except Exception, e:
tb_info = traceback.format_exc()
_LOG.error("%s" % (tb_info))
_LOG.critical(e)
errors.append((source_path, symlink_path, str(e)))
packages_progress_status["num_error"] += 1
packages_progress_status["items_left"] -= 1
continue
packages_progress_status["items_left"] -= 1
def test_get_relpath_from_unit(self):
unit = Unit(type_id='random',
unit_key={"filename": "test_unit_key_filename"},
metadata={"filename": "test_metadata_filename"},
storage_path="/test/storage/path")
self.assertEqual(util.get_relpath_from_unit(unit),
"test_metadata_filename")
unit = Unit(type_id='random',
unit_key={"filename": "test_unit_key_filename"},
metadata={},
storage_path="/test/storage/path")
self.assertEqual(util.get_relpath_from_unit(unit),
"test_unit_key_filename")
unit = Unit(type_id='random',
unit_key={"fileName": "test_unit_key_fileName"},
metadata={},
storage_path="/test/storage/path")
self.assertEqual(util.get_relpath_from_unit(unit),
"test_unit_key_fileName")
unit = Unit(type_id='random',
unit_key={},
metadata={},
storage_path="/test/storage/path")
self.assertEqual(util.get_relpath_from_unit(unit), "path")
def process_unit(self, unit):
"""
Link the unit to the drpm content directory and the package_dir
:param unit: The unit to process
:type unit: pulp.plugins.model.Unit
"""
source_path = unit.storage_path
relative_path = os.path.join('drpms', util.get_relpath_from_unit(unit))
destination_path = os.path.join(self.get_working_dir(), relative_path)
self._create_symlink(source_path, destination_path)
for package_dir in self.dist_step.package_dirs:
destination_path = os.path.join(package_dir, relative_path)
self._create_symlink(source_path, destination_path)
self.context.add_unit_metadata(unit)
def _symlink_content(self, unit, working_sub_dir):
"""
Create a symlink to a unit's storage path in the given working subdirectory.
:param unit: unit to create symlink to
:type unit: pulp.plugins.model.Unit
:param working_sub_dir: working subdirectory to create symlink in
:type working_sub_dir: str
"""
_LOG.debug('Creating symbolic link to content: %s' % unit.unit_key.get('name', 'unknown'))
source_path = unit.storage_path
relative_path = util.get_relpath_from_unit(unit)
destination_path = os.path.join(working_sub_dir, relative_path)
self._create_symlink(source_path, destination_path)
def process_unit(self, unit):
"""
Link the unit to the content directory and the package_dir
:param unit: The unit to process
:type unit: pulp.plugins.model.Unit
"""
source_path = unit.storage_path
relative_path = util.get_relpath_from_unit(unit)
destination_path = os.path.join(self.get_working_dir(), relative_path)
self._create_symlink(source_path, destination_path)
for package_dir in self.dist_step.package_dirs:
destination_path = os.path.join(package_dir, relative_path)
self._create_symlink(source_path, destination_path)
for context in (self.file_lists_context, self.other_context, self.primary_context):
context.add_unit_metadata(unit)
def process_unit(self, unit):
"""
Link the unit to the content directory and the package_dir
:param unit: The unit to process
:type unit: pulp.plugins.model.Unit
"""
source_path = unit.storage_path
relative_path = util.get_relpath_from_unit(unit)
destination_path = os.path.join(self.get_working_dir(), relative_path)
self._create_symlink(source_path, destination_path)
package_dir = self.get_step(constants.PUBLISH_DISTRIBUTION_STEP).package_dir
if package_dir:
destination_path = os.path.join(package_dir, relative_path)
self._create_symlink(source_path, destination_path)
for context in (self.file_lists_context, self.other_context, self.primary_context):
context.add_unit_metadata(unit)