def process_unit(self, unit): """ Link the unit to the content directory and the package_dir :param unit: The unit to process :type unit: pulp.plugins.model.Unit """ source_path = unit.storage_path relative_path = util.get_relpath_from_unit(unit) destination_path = os.path.join(self.get_working_dir(), relative_path) self._create_symlink(source_path, destination_path) filename = unit.unit_key['name'] + '-' + unit.unit_key['version'] + '-' + \ unit.unit_key['release'] + '.' + unit.unit_key['arch'] + '.json' path = os.path.join(self.get_working_dir(), filename) # Remove all keys that start with an underscore, like _id and _ns for key_to_remove in filter(lambda key: key[0] == '_', unit.metadata.keys()): unit.metadata.pop(key_to_remove) # repodata will be regenerated on import, so remove it as well if 'repodata' in unit.metadata: unit.metadata.pop('repodata') dict_to_write = {'unit_key': unit.unit_key, 'unit_metadata': unit.metadata} with open(path, 'w') as f: json.dump(dict_to_write, f)
def check_all_and_associate(wanted, sync_conduit): """ Given a set of unit keys as namedtuples, this function checks if a unit already exists in Pulp and returns the set of tuples that were not found. This checks for the unit in the db as well as for the actual file on the filesystem. If a unit exists in the db and the filesystem, this function also associates the unit to the given repo. Note that the check for the actual file is performed only for the supported unit types. :param wanted: iterable of units as namedtuples :type wanted: iterable :param sync_conduit: repo sync conduit :type sync_conduit: pulp.plugins.conduits.repo_sync.RepoSync :return: set of unit keys as namedtuples, identifying which of the named tuples received as input were not found on the server. :rtype: set """ sorted_units = _sort_by_type(wanted) for unit_type, values in sorted_units.iteritems(): model = models.TYPE_MAP[unit_type] unit_fields = model.UNIT_KEY_NAMES + ('_storage_path', 'filename') rpm_srpm_drpm = unit_type in (models.RPM.TYPE, models.SRPM.TYPE, models.DRPM.TYPE) rpm_or_srpm = unit_type in (models.RPM.TYPE, models.SRPM.TYPE) unit_keys_generator = (unit._asdict() for unit in values.copy()) for unit in get_all_existing_units(unit_keys_generator, unit_fields, unit_type, sync_conduit.search_all_units): # For RPMs, SRPMs and DRPMs, also check if the file exists on the filesystem. # If not, we do not want to skip downloading the unit. if rpm_srpm_drpm: if unit.storage_path is None or not os.path.isfile(unit.storage_path): continue # Since the unit is already downloaded, call respective sync_conduit calls to import # the unit in given repository. if rpm_or_srpm: unit_key = unit.unit_key rpm_or_srpm_unit = model(unit_key['name'], unit_key['epoch'], unit_key['version'], unit_key['release'], unit_key['arch'], unit_key['checksumtype'], unit_key['checksum'], unit.metadata) relative_path = rpm_or_srpm_unit.relative_path else: relative_path = get_relpath_from_unit(unit) downloaded_unit = sync_conduit.init_unit(unit_type, unit.unit_key, unit.metadata, relative_path) # 1125388 - make sure we keep storage_path on the new unit model obj downloaded_unit.storage_path = unit.storage_path sync_conduit.save_unit(downloaded_unit) # Discard already downloaded unit from the return value. named_tuple = model(metadata=unit.metadata, **unit.unit_key).as_named_tuple values.discard(named_tuple) ret = set() ret.update(*sorted_units.values()) return ret
def export_rpm(working_dir, rpm_units, progress_callback=None): """ This method takes a list of rpm units and exports them to the given working directory :param working_dir: The full path to the directory to export the content to :type working_dir: str :param rpm_units: the list of rpm units to export :type rpm_units: list of AssociatedUnit :param progress_callback: callback to report progress info to publish_conduit :type progress_callback: function :return: tuple of status and list of error messages if any occurred :rtype: ({}, [str]) """ # get rpm units progress_status = init_progress_report(len(rpm_units)) summary = { 'num_package_units_attempted': 0, 'num_package_units_exported': 0, 'num_package_units_errors': 0, } details = {'errors': {}} errors = [] for unit in rpm_units: set_progress(ids.TYPE_ID_RPM, progress_status, progress_callback) source_path = unit.storage_path destination_path = os.path.join(working_dir, yum_utils.get_relpath_from_unit(unit)) if not yum_utils.create_copy(source_path, destination_path): msg = "Unable to copy %s to %s" % (source_path, destination_path) _logger.error(msg) errors.append(msg) progress_status[constants.PROGRESS_NUM_ERROR_KEY] += 1 progress_status[constants.PROGRESS_ITEMS_LEFT_KEY] -= 1 continue progress_status[constants.PROGRESS_NUM_SUCCESS_KEY] += 1 progress_status[constants.PROGRESS_ITEMS_LEFT_KEY] -= 1 summary["num_package_units_attempted"] += len(rpm_units) summary["num_package_units_exported"] += len(rpm_units) - len(errors) summary["num_package_units_errors"] += len(errors) # If errors occurred, write them to details and set the state to failed. if errors: details['errors']['rpm_export'] = errors progress_status[constants.PROGRESS_STATE_KEY] = constants.STATE_FAILED progress_status[constants.PROGRESS_ERROR_DETAILS_KEY] = errors set_progress(models.RPM.TYPE, progress_status, progress_callback) return summary, details progress_status[constants.PROGRESS_STATE_KEY] = constants.STATE_COMPLETE set_progress(models.RPM.TYPE, progress_status, progress_callback) return summary, details
def export_rpms(self, rpm_units, progress_callback=None): """ This call looksup each rpm units and exports to the working directory. @param rpm_units @type rpm_units list of AssociatedUnit to be exported @param progress_callback: callback to report progress info to publish_conduit @type progress_callback: function @return tuple of status and list of error messages if any occurred @rtype ({}, [str]) """ # get rpm units summary = {} symlink_dir = self.repo_working_dir packages_progress_status = self.init_progress() if 'rpm' in self.skip: packages_progress_status["state"] = "SKIPPED" self.set_progress("rpms", packages_progress_status, progress_callback) _LOG.info("rpm unit type in skip list [%s]; skipping export" % self.skip) return summary, [] packages_progress_status["num_success"] = 0 packages_progress_status["items_left"] = len(rpm_units) packages_progress_status["items_total"] = len(rpm_units) errors = [] for u in rpm_units: self.set_progress("rpms", packages_progress_status, progress_callback) relpath = util.get_relpath_from_unit(u) source_path = u.storage_path symlink_path = os.path.join(symlink_dir, relpath) if not os.path.exists(source_path): msg = "Source path: %s is missing" % (source_path) errors.append((source_path, symlink_path, msg)) packages_progress_status["num_error"] += 1 packages_progress_status["items_left"] -= 1 continue _LOG.info("Unit exists at: %s we need to copy to: %s" % (source_path, symlink_path)) try: if not util.create_copy(source_path, symlink_path): msg = "Unable to create copy for: %s pointing to %s" % (symlink_path, source_path) _LOG.error(msg) errors.append((source_path, symlink_path, msg)) packages_progress_status["num_error"] += 1 packages_progress_status["items_left"] -= 1 continue packages_progress_status["num_success"] += 1 except Exception, e: tb_info = traceback.format_exc() _LOG.error("%s" % (tb_info)) _LOG.critical(e) errors.append((source_path, symlink_path, str(e))) packages_progress_status["num_error"] += 1 packages_progress_status["items_left"] -= 1 continue packages_progress_status["items_left"] -= 1
def test_get_relpath_from_unit(self): distributor = YumDistributor() test_unit = Unit("rpm", "unit_key", {}, "") test_unit.unit_key = {"fileName" : "test_1"} rel_path = util.get_relpath_from_unit(test_unit) self.assertEqual(rel_path, "test_1") test_unit.unit_key = {} test_unit.storage_path = "test_0" rel_path = util.get_relpath_from_unit(test_unit) self.assertEqual(rel_path, "test_0") test_unit.metadata["filename"] = "test_2" rel_path = util.get_relpath_from_unit(test_unit) self.assertEqual(rel_path, "test_2") test_unit.metadata["relativepath"] = "test_3" rel_path = util.get_relpath_from_unit(test_unit) self.assertEqual(rel_path, "test_3")
def test_get_relpath_from_unit(self): unit = Unit(type_id='random', unit_key={"filename": "test_unit_key_filename"}, metadata={"filename": "test_metadata_filename"}, storage_path="/test/storage/path") self.assertEqual(util.get_relpath_from_unit(unit), "test_metadata_filename") unit = Unit(type_id='random', unit_key={"filename": "test_unit_key_filename"}, metadata={}, storage_path="/test/storage/path") self.assertEqual(util.get_relpath_from_unit(unit), "test_unit_key_filename") unit = Unit(type_id='random', unit_key={"fileName": "test_unit_key_fileName"}, metadata={}, storage_path="/test/storage/path") self.assertEqual(util.get_relpath_from_unit(unit), "test_unit_key_fileName") unit = Unit(type_id='random', unit_key={}, metadata={}, storage_path="/test/storage/path") self.assertEqual(util.get_relpath_from_unit(unit), "path")
def handle_symlinks(self, units, symlink_dir, progress_callback=None): """ @param units list of units that belong to the repo and should be published @type units [AssociatedUnit] @param symlink_dir where to create symlinks @type symlink_dir str @param progress_callback: callback to report progress info to publish_conduit @type progress_callback: function @return tuple of status and list of error messages if any occurred @rtype (bool, [str]) """ packages_progress_status = self.init_progress() _LOG.info("handle_symlinks invoked with %s units to %s dir" % (len(units), symlink_dir)) self.set_progress("packages", packages_progress_status, progress_callback) errors = [] packages_progress_status["items_total"] = len(units) packages_progress_status["items_left"] = len(units) for u in units: self.set_progress("packages", packages_progress_status, progress_callback) relpath = util.get_relpath_from_unit(u) source_path = u.storage_path symlink_path = os.path.join(symlink_dir, relpath) if not os.path.exists(source_path): msg = "Source path: %s is missing" % (source_path) errors.append((source_path, symlink_path, msg)) packages_progress_status["num_error"] += 1 packages_progress_status["items_left"] -= 1 continue _LOG.info("Unit exists at: %s we need to symlink to: %s" % (source_path, symlink_path)) try: if not util.create_symlink(source_path, symlink_path): msg = "Unable to create symlink for: %s pointing to %s" % (symlink_path, source_path) _LOG.error(msg) errors.append((source_path, symlink_path, msg)) packages_progress_status["num_error"] += 1 packages_progress_status["items_left"] -= 1 continue packages_progress_status["num_success"] += 1 except Exception, e: tb_info = traceback.format_exc() _LOG.error("%s" % (tb_info)) _LOG.critical(e) errors.append((source_path, symlink_path, str(e))) packages_progress_status["num_error"] += 1 packages_progress_status["items_left"] -= 1 continue packages_progress_status["items_left"] -= 1
def process_unit(self, unit): """ Link the unit to the drpm content directory and the package_dir :param unit: The unit to process :type unit: pulp.plugins.model.Unit """ source_path = unit.storage_path relative_path = os.path.join('drpms', util.get_relpath_from_unit(unit)) destination_path = os.path.join(self.get_working_dir(), relative_path) self._create_symlink(source_path, destination_path) for package_dir in self.dist_step.package_dirs: destination_path = os.path.join(package_dir, relative_path) self._create_symlink(source_path, destination_path) self.context.add_unit_metadata(unit)
def _symlink_content(self, unit, working_sub_dir): """ Create a symlink to a unit's storage path in the given working subdirectory. :param unit: unit to create symlink to :type unit: pulp.plugins.model.Unit :param working_sub_dir: working subdirectory to create symlink in :type working_sub_dir: str """ _LOG.debug('Creating symbolic link to content: %s' % unit.unit_key.get('name', 'unknown')) source_path = unit.storage_path relative_path = util.get_relpath_from_unit(unit) destination_path = os.path.join(working_sub_dir, relative_path) self._create_symlink(source_path, destination_path)
def process_unit(self, unit): """ Link the unit to the content directory and the package_dir :param unit: The unit to process :type unit: pulp.plugins.model.Unit """ source_path = unit.storage_path relative_path = util.get_relpath_from_unit(unit) destination_path = os.path.join(self.get_working_dir(), relative_path) self._create_symlink(source_path, destination_path) for package_dir in self.dist_step.package_dirs: destination_path = os.path.join(package_dir, relative_path) self._create_symlink(source_path, destination_path) for context in (self.file_lists_context, self.other_context, self.primary_context): context.add_unit_metadata(unit)
def process_unit(self, unit): """ Link the unit to the content directory and the package_dir :param unit: The unit to process :type unit: pulp.plugins.model.Unit """ source_path = unit.storage_path relative_path = util.get_relpath_from_unit(unit) destination_path = os.path.join(self.get_working_dir(), relative_path) self._create_symlink(source_path, destination_path) package_dir = self.get_step(constants.PUBLISH_DISTRIBUTION_STEP).package_dir if package_dir: destination_path = os.path.join(package_dir, relative_path) self._create_symlink(source_path, destination_path) for context in (self.file_lists_context, self.other_context, self.primary_context): context.add_unit_metadata(unit)