def POST(self, repo_id): # Params params = self.params() query = params.get('criteria', None) repo_query_manager = manager_factory.repo_query_manager() repo = repo_query_manager.find_by_id(repo_id) if repo is None: raise exceptions.MissingResource(repo_id=repo_id) if query is None: raise exceptions.MissingValue(['criteria']) try: criteria = UnitAssociationCriteria.from_client_input(query) except: _LOG.exception('Error parsing association criteria [%s]' % query) raise exceptions.PulpDataException(), None, sys.exc_info()[2] # Data lookup manager = manager_factory.repo_unit_association_query_manager() if criteria.type_ids is not None and len(criteria.type_ids) == 1: type_id = criteria.type_ids[0] units = manager.get_units_by_type(repo_id, type_id, criteria=criteria) else: units = manager.get_units_across_types(repo_id, criteria=criteria) return self.ok(units)
def test_migrate_category(self): # Setup orig_cat_id = add_unit('c1', self.source_repo_id, ids.TYPE_ID_PKG_CATEGORY) associate_unit(orig_cat_id, self.source_repo_id, ids.TYPE_ID_PKG_CATEGORY) associate_unit(orig_cat_id, self.dest_repo_id, ids.TYPE_ID_PKG_CATEGORY) # Test self.migration.migrate() group_coll = types_db.type_units_collection(ids.TYPE_ID_PKG_CATEGORY) all_cats = group_coll.find({}).sort('repo_id', 1) self.assertEqual(2, all_cats.count()) dest_cat = all_cats[0] # ordered by ID, this will be first self.assertEqual(dest_cat['id'], 'c1') self.assertEqual(dest_cat['repo_id'], self.dest_repo_id) source_cat = all_cats[1] self.assertEqual(source_cat['id'], 'c1') self.assertEqual(source_cat['repo_id'], self.source_repo_id) # Verify the associations query_manager = factory.repo_unit_association_query_manager() source_units = query_manager.get_units(self.source_repo_id) self.assertEqual(1, len(source_units)) self.assertEqual(source_units[0]['unit_type_id'], ids.TYPE_ID_PKG_CATEGORY) self.assertEqual(source_units[0]['unit_id'], source_cat['_id']) dest_units = query_manager.get_units(self.dest_repo_id) self.assertEqual(1, len(dest_units)) self.assertEqual(dest_units[0]['unit_type_id'], ids.TYPE_ID_PKG_CATEGORY) self.assertEqual(dest_units[0]['unit_id'], dest_cat['_id'])
def __init__(self, source_repo_id, dest_repo_id, source_importer_id, dest_importer_id): """ :param source_repo_id: ID of the repository from which units are being copied :type source_repo_id: str :param dest_repo_id: ID of the repository into which units are being copied :type dest_repo_id: str :param source_importer_id: ID of the importer on the source repository :type source_importer_id: str :param dest_importer_id: ID of the importer on the destination repository :type dest_importer_id: str """ ImporterScratchPadMixin.__init__(self, dest_repo_id, dest_importer_id) RepoScratchPadMixin.__init__(self, dest_repo_id, ImporterConduitException) SearchUnitsMixin.__init__(self, ImporterConduitException) AddUnitMixin.__init__(self, dest_repo_id, dest_importer_id) self.source_repo_id = source_repo_id self.dest_repo_id = dest_repo_id self.source_importer_id = source_importer_id self.dest_importer_id = dest_importer_id self.__association_manager = manager_factory.repo_unit_association_manager( ) self.__association_query_manager = manager_factory.repo_unit_association_query_manager( ) self.__importer_manager = manager_factory.repo_importer_manager()
def _add_repo_memberships(units, type_id): """ For a list of units, find what repos each is a member of and add a list of repo_ids to each unit. :param units: list of unit documents :type units: list of dicts :param type_id: content type id :type type_id: str :return: same list of units that was passed in, only for convenience. units are modified in-place """ # quick return if there is nothing to do if not units: return units unit_ids = [unit["_id"] for unit in units] criteria = Criteria( filters={"unit_id": {"$in": unit_ids}, "unit_type_id": type_id}, fields=("repo_id", "unit_id") ) associations = factory.repo_unit_association_query_manager().find_by_criteria(criteria) unit_ids = None criteria = None association_map = {} for association in associations: association_map.setdefault(association["unit_id"], set()).add(association["repo_id"]) for unit in units: unit["repository_memberships"] = list(association_map.get(unit["_id"], [])) return units
def do_get_repo_units(repo_id, criteria, exception_class, as_generator=False): """ Performs a repo unit association query. This is split apart so we can have custom mixins with different signatures. """ try: association_query_manager = manager_factory.repo_unit_association_query_manager( ) # Use a get_units as_generator here and cast to a list later, if necessary. units = association_query_manager.get_units(repo_id, criteria=criteria, as_generator=True) # Load all type definitions so we don't hammer the database. type_defs = dict((t['id'], t) for t in types_db.all_type_definitions()) # Transfer object generator. def _transfer_object_generator(): for u in units: yield common_utils.to_plugin_associated_unit( u, type_defs[u['unit_type_id']]) if as_generator: return _transfer_object_generator() # Maintain legacy behavior by default. return list(_transfer_object_generator()) except Exception, e: logger.exception( 'Exception from server requesting all content units for repository [%s]' % repo_id) raise exception_class(e), None, sys.exc_info()[2]
def calculate_associated_type_ids(source_repo_id, associated_units): if associated_units is not None: associated_unit_type_ids = set([u['unit_type_id'] for u in associated_units]) else: association_query_manager = manager_factory.repo_unit_association_query_manager() associated_unit_type_ids = association_query_manager.unit_type_ids_for_repo(source_repo_id) return associated_unit_type_ids
def _add_repo_memberships(units, type_id): """ For a list of units, find what repos each is a member of, and add a list of repo_ids to each unit. :param units: list of unit documents :type units: list of dicts :param type_id: content type id :type type_id: str :return: same list of units that was passed in, only for convenience. units are modified in-place """ # quick return if there is nothing to do if not units: return units unit_ids = [unit['_id'] for unit in units] criteria = Criteria( filters={'unit_id': {'$in': unit_ids}, 'unit_type_id': type_id}, fields=('repo_id', 'unit_id') ) associations = factory.repo_unit_association_query_manager().find_by_criteria(criteria) unit_ids = None criteria = None association_map = {} for association in associations: association_map.setdefault(association['unit_id'], set()).add( association['repo_id']) for unit in units: unit['repository_memberships'] = list(association_map.get(unit['_id'], [])) return units
def __init__(self, source_repo_id, dest_repo_id, source_importer_id, dest_importer_id, association_owner_type, association_owner_id): """ :param source_repo_id: ID of the repository from which units are being copied :type source_repo_id: str :param dest_repo_id: ID of the repository into which units are being copied :type dest_repo_id: str :param source_importer_id: ID of the importer on the source repository :type source_importer_id: str :param dest_importer_id: ID of the importer on the destination repository :type dest_importer_id: str :param association_owner_type: distinguishes the owner when creating an association through this conduit :type association_owner_type: str :param association_owner_id: specific ID of the owner when creating an association through this conduit :type association_owner_id: str """ ImporterScratchPadMixin.__init__(self, dest_repo_id, dest_importer_id) RepoScratchPadMixin.__init__(self, dest_repo_id, ImporterConduitException) SearchUnitsMixin.__init__(self, ImporterConduitException) AddUnitMixin.__init__(self, dest_repo_id, dest_importer_id, association_owner_type, association_owner_id) self.source_repo_id = source_repo_id self.dest_repo_id = dest_repo_id self.source_importer_id = source_importer_id self.dest_importer_id = dest_importer_id self.association_owner_type = association_owner_type self.association_owner_id = association_owner_id self.__association_manager = manager_factory.repo_unit_association_manager() self.__association_query_manager = manager_factory.repo_unit_association_query_manager() self.__importer_manager = manager_factory.repo_importer_manager()
def _generate_response(cls, query, options, *args, **kwargs): """ Perform the database query using the given search data, and return the resuls as a JSON serialized HttpReponse object. This overrides the base class so we can validate repo existance and to choose the search method depending on how many unit types we are dealing with. :param query: The criteria that should be used to search for objects :type query: dict :param options: additional options for including extra data :type options: dict :return: The serialized search results in an HttpReponse :rtype: django.http.HttpResponse """ repo_id = kwargs.get('repo_id') model.Repository.objects.get_repo_or_missing_resource(repo_id) criteria = UnitAssociationCriteria.from_client_input(query) manager = manager_factory.repo_unit_association_query_manager() if criteria.type_ids is not None and len(criteria.type_ids) == 1: type_id = criteria.type_ids[0] units = manager.get_units_by_type(repo_id, type_id, criteria=criteria) else: units = manager.get_units(repo_id, criteria=criteria) for unit in units: content.remap_fields_with_serializer(unit['metadata']) return generate_json_response_with_pulp_encoder(units)
def _generate_response(cls, query, options, *args, **kwargs): """ Perform the database query using the given search data, and return the resuls as a JSON serialized HttpReponse object. This overrides the base class so we can validate repo existance and to choose the search method depending on how many unit types we are dealing with. :param query: The criteria that should be used to search for objects :type query: dict :param options: additional options for including extra data :type options: dict :return: The serialized search results in an HttpReponse :rtype: django.http.HttpResponse """ repo_id = kwargs.get('repo_id') model.Repository.objects.get_repo_or_missing_resource(repo_id) criteria = UnitAssociationCriteria.from_client_input(query) manager = manager_factory.repo_unit_association_query_manager() if criteria.type_ids is not None and len(criteria.type_ids) == 1: type_id = criteria.type_ids[0] units = manager.get_units_by_type(repo_id, type_id, criteria=criteria) else: units = manager.get_units(repo_id, criteria=criteria) return generate_json_response_with_pulp_encoder(units)
def __init__(self, source_repo_id, dest_repo_id, source_importer_id, dest_importer_id): """ :param source_repo_id: ID of the repository from which units are being copied :type source_repo_id: str :param dest_repo_id: ID of the repository into which units are being copied :type dest_repo_id: str :param source_importer_id: ID of the importer on the source repository :type source_importer_id: str :param dest_importer_id: ID of the importer on the destination repository :type dest_importer_id: str """ ImporterScratchPadMixin.__init__(self, dest_repo_id, dest_importer_id) RepoScratchPadMixin.__init__(self, dest_repo_id, ImporterConduitException) SearchUnitsMixin.__init__(self, ImporterConduitException) AddUnitMixin.__init__(self, dest_repo_id, dest_importer_id) self.source_repo_id = source_repo_id self.dest_repo_id = dest_repo_id self.source_importer_id = source_importer_id self.dest_importer_id = dest_importer_id self.__association_manager = manager_factory.repo_unit_association_manager() self.__association_query_manager = manager_factory.repo_unit_association_query_manager() self.__importer_manager = manager_factory.repo_importer_manager()
def test_migrate_duplicates_doesnt_delete_from_source_repo(self): """ This tests the correct behavior when we try to change the repo_id on an object, and end up causing a duplicate error due to our uniqueness constraint. It also makes sure the units are not deleted from the source repository if they are in the source repository. """ # Let's put two units here with the same IDs with two different repo_ids, and the run the # migration. source_repo_group_id = add_unit('group', self.source_repo_id, ids.TYPE_ID_PKG_GROUP) dest_repo_group_id = add_unit('group', self.dest_repo_id, ids.TYPE_ID_PKG_GROUP) # Associate the source_repo_group_id with both source and destination repos associate_unit(source_repo_group_id, self.source_repo_id, ids.TYPE_ID_PKG_GROUP) associate_unit(source_repo_group_id, self.dest_repo_id, ids.TYPE_ID_PKG_GROUP) associate_unit(dest_repo_group_id, self.dest_repo_id, ids.TYPE_ID_PKG_GROUP) # Migrate should not cause a DuplicateKeyError self.migration.migrate() # Verify that both groups remain, because the migration should not have removed either group_collection = types_db.type_units_collection( ids.TYPE_ID_PKG_GROUP) all_groups = list(group_collection.find()) self.assertEqual(len(all_groups), 2) self.assertEqual( group_collection.find({ 'id': 'group', 'repo_id': self.dest_repo_id }).count(), 1) self.assertEqual( group_collection.find({ 'id': 'group', 'repo_id': self.source_repo_id }).count(), 1) # Let's make sure that there are two associations, and that they are correct. query_manager = factory.repo_unit_association_query_manager() dest_units = query_manager.get_units(self.dest_repo_id) self.assertEqual(len(dest_units), 1) dest_unit = dest_units[0] self.assertEqual(dest_unit['unit_type_id'], ids.TYPE_ID_PKG_GROUP) self.assertEqual(dest_unit['unit_id'], dest_repo_group_id) source_units = query_manager.get_units(self.source_repo_id) self.assertEqual(len(source_units), 1) source_unit = source_units[0] self.assertEqual(source_unit['unit_type_id'], ids.TYPE_ID_PKG_GROUP) self.assertEqual(source_unit['unit_id'], source_repo_group_id) # Verify the repo counts source = model.Repository.objects.get(repo_id='source-repo') self.assertEqual(source.content_unit_counts, {'package_group': 1}) dest = model.Repository.objects.get(repo_id='dest-repo') self.assertEqual(dest.content_unit_counts, {'package_group': 1})
def do_get_repo_units(repo_id, criteria, exception_class, as_generator=False): """ Performs a repo unit association query. This is split apart so we can have custom mixins with different signatures. """ try: association_query_manager = manager_factory.repo_unit_association_query_manager() # Use a get_units as_generator here and cast to a list later, if necessary. units = association_query_manager.get_units(repo_id, criteria=criteria, as_generator=True) # Transfer object generator. def _transfer_object_generator(): unit_key_fields_cache = {} for u in units: type_id = u['unit_type_id'] if type_id not in unit_key_fields_cache: fields = units_controller.get_unit_key_fields_for_type(type_id) unit_key_fields_cache[type_id] = fields yield common_utils.to_plugin_associated_unit(u, type_id, unit_key_fields_cache[type_id]) if as_generator: return _transfer_object_generator() # Maintain legacy behavior by default. return list(_transfer_object_generator()) except Exception, e: _logger.exception( 'Exception from server requesting all content units for repository [%s]' % repo_id) raise exception_class(e), None, sys.exc_info()[2]
def do_get_repo_units(repo_id, criteria, exception_class): """ Performs a repo unit association query. This is split apart so we can have custom mixins with different signatures. """ try: association_query_manager = manager_factory.repo_unit_association_query_manager( ) units = association_query_manager.get_units(repo_id, criteria=criteria) all_units = [] # Load all type definitions in use so we don't hammer the database unique_type_defs = set([u['unit_type_id'] for u in units]) type_defs = {} for def_id in unique_type_defs: type_def = types_db.type_definition(def_id) type_defs[def_id] = type_def # Convert to transfer object for unit in units: type_id = unit['unit_type_id'] u = common_utils.to_plugin_associated_unit(unit, type_defs[type_id]) all_units.append(u) return all_units except Exception, e: _LOG.exception( 'Exception from server requesting all content units for repository [%s]' % repo_id) raise exception_class(e), None, sys.exc_info()[2]
def do_get_repo_units(repo_id, criteria, exception_class, as_generator=False): """ Performs a repo unit association query. This is split apart so we can have custom mixins with different signatures. """ try: association_query_manager = manager_factory.repo_unit_association_query_manager() # Use a get_units as_generator here and cast to a list later, if necessary. units = association_query_manager.get_units(repo_id, criteria=criteria, as_generator=True) # Load all type definitions so we don't hammer the database. type_defs = dict((t['id'], t) for t in types_db.all_type_definitions()) # Transfer object generator. def _transfer_object_generator(): for u in units: yield common_utils.to_plugin_associated_unit(u, type_defs[u['unit_type_id']]) if as_generator: return _transfer_object_generator() # Maintain legacy behavior by default. return list(_transfer_object_generator()) except Exception, e: _LOG.exception('Exception from server requesting all content units for repository [%s]' % repo_id) raise exception_class(e), None, sys.exc_info()[2]
def _get_total(self, id_list=None, ignore_filter=False): """ Return the total number of units that are processed by this step. This is used generally for progress reporting. The value returned should not change during the processing of the step. :param id_list: List of type ids to get the total count of :type id_list: list of str :param ignore_filter: Ignore the association filter and get all units of the given types :type ignore_filter: bool """ if id_list is None: id_list = self.unit_type total = 0 types_to_query = set(id_list).difference(self.skip_list) if not ignore_filter and self.association_filters: # We are copying using a filter so we have to get everything new_filter = copy.deepcopy(self.association_filters) new_filter['unit_type_id'] = {'$in': list(types_to_query)} criteria = Criteria(filters=new_filter) association_query_manager = manager_factory.repo_unit_association_query_manager() units_cursor = association_query_manager.find_by_criteria(criteria) total = units_cursor.count() else: for type_id in types_to_query: total += self.parent.repo.content_unit_counts.get(type_id, 0) return total
def verify(self, num_units=PluginTestBase.NUM_UNITS): # repository manager = managers.repo_query_manager() manager.get_repository(self.REPO_ID) # importer manager = managers.repo_importer_manager() importer = manager.get_importer(self.REPO_ID) manifest_url = importer['config'][constants.MANIFEST_URL_KEYWORD] self.assertTrue(manifest_url.endswith('%s/manifest.json.gz' % self.REPO_ID)) # distributor manager = managers.repo_distributor_manager() manager.get_distributor(self.REPO_ID, FAKE_DISTRIBUTOR) self.assertRaises(MissingResource, manager.get_distributor, self.REPO_ID, constants.HTTP_DISTRIBUTOR) # check units manager = managers.repo_unit_association_query_manager() units = manager.get_units(self.REPO_ID) units = dict([(u['metadata']['N'], u) for u in units]) self.assertEqual(len(units), num_units) for n in range(0, num_units): unit = units[n] unit_id = self.UNIT_ID % n metadata = unit['metadata'] storage_path = metadata['_storage_path'].replace('//', '/') self.assertEqual(unit['unit_type_id'], self.UNIT_TYPE_ID) self.assertEqual(unit['repo_id'], self.REPO_ID) self.assertEqual(unit['owner_id'], constants.HTTP_IMPORTER) file_path = '.'.join((unit_id, self.UNIT_TYPE_ID)) self.assertEqual(storage_path, os.path.join(self.childfs, 'content', file_path)) self.assertTrue(os.path.exists(storage_path)) fp = open(storage_path) content = fp.read() fp.close() self.assertEqual(content, unit_id)
def POST(self, repo_id): # Params params = self.params() query = params.get('criteria', None) repo_query_manager = manager_factory.repo_query_manager() repo = repo_query_manager.find_by_id(repo_id) if repo is None: raise exceptions.MissingResource(repo_id=repo_id) if query is None: raise exceptions.MissingValue(['criteria']) try: criteria = UnitAssociationCriteria.from_client_input(query) except: _logger.error('Error parsing association criteria [%s]' % query) raise exceptions.PulpDataException(), None, sys.exc_info()[2] # Data lookup manager = manager_factory.repo_unit_association_query_manager() if criteria.type_ids is not None and len(criteria.type_ids) == 1: type_id = criteria.type_ids[0] units = manager.get_units_by_type(repo_id, type_id, criteria=criteria) else: units = manager.get_units_across_types(repo_id, criteria=criteria) return self.ok(units)
def resolve_dependencies_by_criteria(self, repo_id, criteria, options): """ Calculates dependencies for units in the given repositories. The repository's importer is used to perform the calculation. The units to resolve dependencies for are calculated by applying the given criteria against the repository. @param repo_id: identifies the repository @type repo_id: str @param criteria: used to determine which units to resolve dependencies for @type criteria: UnitAssociationCriteria @param options: dict of options to pass the importer to drive the resolution @type options: dict @return: report from the plugin @rtype: object """ association_query_manager = manager_factory.repo_unit_association_query_manager() units = association_query_manager.get_units(repo_id, criteria=criteria) # The bulk of the validation will be done in the chained call below return self.resolve_dependencies_by_units(repo_id, units, options)
def _get_total(self, id_list=None, ignore_filter=False): """ Return the total number of units that are processed by this step. This is used generally for progress reporting. The value returned should not change during the processing of the step. :param id_list: List of type ids to get the total count of :type id_list: list of str :param ignore_filter: Ignore the association filter and get all units of the given types :type ignore_filter: bool """ if id_list is None: id_list = self.unit_type total = 0 types_to_query = set(id_list).difference(self.skip_list) if not ignore_filter and self.association_filters: # We are copying using a filter so we have to get everything new_filter = copy.deepcopy(self.association_filters) new_filter['unit_type_id'] = {'$in': list(types_to_query)} criteria = Criteria(filters=new_filter) association_query_manager = manager_factory.repo_unit_association_query_manager( ) units_cursor = association_query_manager.find_by_criteria(criteria) total = units_cursor.count() else: for type_id in types_to_query: total += self.parent.repo.content_unit_counts.get(type_id, 0) return total
def unassociate_by_criteria(repo_id, criteria, notify_plugins=True): """ Unassociate units that are matched by the given criteria. :param repo_id: identifies the repo :type repo_id: str :param criteria: :param notify_plugins: if true, relevant plugins will be informed of the removal :type notify_plugins: bool """ criteria = UnitAssociationCriteria.from_dict(criteria) association_query_manager = manager_factory.repo_unit_association_query_manager( ) unassociate_units = association_query_manager.get_units( repo_id, criteria=criteria) if len(unassociate_units) == 0: return {} unit_map = {} # maps unit_type_id to a list of unit_ids for unit in unassociate_units: id_list = unit_map.setdefault(unit['unit_type_id'], []) id_list.append(unit['unit_id']) collection = RepoContentUnit.get_collection() for unit_type_id, unit_ids in unit_map.items(): spec = { 'repo_id': repo_id, 'unit_type_id': unit_type_id, 'unit_id': { '$in': unit_ids } } collection.remove(spec) unique_count = sum( 1 for unit_id in unit_ids if not RepoUnitAssociationManager.association_exists( repo_id, unit_id, unit_type_id)) if not unique_count: continue repo_controller.update_unit_count(repo_id, unit_type_id, -unique_count) repo_controller.update_last_unit_removed(repo_id) # Convert the units into transfer units. This happens regardless of whether or not # the plugin will be notified as it's used to generate the return result, transfer_units = create_transfer_units(unassociate_units) if notify_plugins: remove_from_importer(repo_id, transfer_units) # Match the return type/format as copy serializable_units = [u.to_id_dict() for u in transfer_units] return {'units_successful': serializable_units}
def resolve_dependencies_by_criteria(self, repo_id, criteria, options): """ Calculates dependencies for units in the given repositories. The repository's importer is used to perform the calculation. The units to resolve dependencies for are calculated by applying the given criteria against the repository. @param repo_id: identifies the repository @type repo_id: str @param criteria: used to determine which units to resolve dependencies for @type criteria: UnitAssociationCriteria @param options: dict of options to pass the importer to drive the resolution @type options: dict @return: report from the plugin @rtype: object """ association_query_manager = manager_factory.repo_unit_association_query_manager( ) units = association_query_manager.get_units(repo_id, criteria=criteria) # The bulk of the validation will be done in the chained call below return self.resolve_dependencies_by_units(repo_id, units, options)
def do_get_repo_units(repo_id, criteria, exception_class): """ Performs a repo unit association query. This is split apart so we can have custom mixins with different signatures. """ try: association_query_manager = manager_factory.repo_unit_association_query_manager() units = association_query_manager.get_units(repo_id, criteria=criteria) all_units = [] # Load all type definitions in use so we don't hammer the database unique_type_defs = set([u['unit_type_id'] for u in units]) type_defs = {} for def_id in unique_type_defs: type_def = types_db.type_definition(def_id) type_defs[def_id] = type_def # Convert to transfer object for unit in units: type_id = unit['unit_type_id'] u = common_utils.to_plugin_unit(unit, type_defs[type_id]) all_units.append(u) return all_units except Exception, e: _LOG.exception('Exception from server requesting all content units for repository [%s]' % repo_id) raise exception_class(e), None, sys.exc_info()[2]
def get_repo_units(self, repo_id, content_type_id, additional_unit_fields=None): """ Searches for units in the given repository with given content type and returns a plugin unit containing unit id, unit key and any additional fields requested. :param repo_id: repo id :type repo_id: str :param content_type_id: content type id of the units :type content_type_id: str :param additional_unit_fields: additional fields from the unit metadata to be added in the result :type additional_unit_fields: list of str :return: list of unit instances :rtype: list of pulp.plugins.model.Unit """ additional_unit_fields = additional_unit_fields or [] try: unit_key_fields = units_controller.get_unit_key_fields_for_type( content_type_id) # Query repo association manager to get all units of given type # associated with given repo. Limit data by requesting only the fields # that are needed. query_manager = managers.repo_unit_association_query_manager() unit_fields = list(unit_key_fields) + list(additional_unit_fields) criteria = UnitAssociationCriteria(association_fields=['unit_id'], unit_fields=unit_fields) units = query_manager.get_units_by_type(repo_id, content_type_id, criteria) # Convert units to plugin units with unit_key and required metadata values for each unit all_units = [] for unit in units: unit_key = {} metadata = {} for k in unit_key_fields: unit_key[k] = unit['metadata'].pop(k) # Add unit_id and any additional unit fields requested by plugins metadata['unit_id'] = unit.pop('unit_id') for field in additional_unit_fields: metadata[field] = unit['metadata'].pop(field, None) u = Unit(content_type_id, unit_key, metadata, None) all_units.append(u) return all_units except Exception, e: _logger.exception( _('Exception from server getting units from repo [%s]' % repo_id)) raise self.exception_class(e), None, sys.exc_info()[2]
def load_associated_units(source_repo_id, criteria): criteria.association_fields = None # Retrieve the units to be associated association_query_manager = manager_factory.repo_unit_association_query_manager() associate_us = association_query_manager.get_units(source_repo_id, criteria=criteria) return associate_us
def unassociate_by_criteria(repo_id, criteria, owner_type, owner_id, notify_plugins=True): """ Unassociate units that are matched by the given criteria. :param repo_id: identifies the repo :type repo_id: str :param criteria: :param owner_type: category of the caller who created the association :type owner_type: str :param owner_id: identifies the call who created the association :type owner_id: str :param notify_plugins: if true, relevant plugins will be informed of the removal :type notify_plugins: bool """ association_query_manager = manager_factory.repo_unit_association_query_manager() unassociate_units = association_query_manager.get_units(repo_id, criteria=criteria) if len(unassociate_units) == 0: return {} unit_map = {} # maps unit_type_id to a list of unit_ids for unit in unassociate_units: id_list = unit_map.setdefault(unit['unit_type_id'], []) id_list.append(unit['unit_id']) collection = RepoContentUnit.get_collection() repo_manager = manager_factory.repo_manager() for unit_type_id, unit_ids in unit_map.items(): spec = {'repo_id': repo_id, 'unit_type_id': unit_type_id, 'unit_id': {'$in': unit_ids} } collection.remove(spec, safe=True) unique_count = sum( 1 for unit_id in unit_ids if not RepoUnitAssociationManager.association_exists( repo_id, unit_id, unit_type_id)) if not unique_count: continue repo_manager.update_unit_count(repo_id, unit_type_id, -unique_count) repo_manager.update_last_unit_removed(repo_id) # Convert the units into transfer units. This happens regardless of whether or not # the plugin will be notified as it's used to generate the return result, unit_type_ids = calculate_associated_type_ids(repo_id, unassociate_units) transfer_units = create_transfer_units(unassociate_units, unit_type_ids) if notify_plugins: remove_from_importer(repo_id, transfer_units) # Match the return type/format as copy serializable_units = [u.to_id_dict() for u in transfer_units] return {'units_successful': serializable_units}
def __unit_ids_to_plugin_unit_keys(self, unit_ids_by_type, repo_ids): """ Parse a dictionary of unit ids keyed by content type id and return a dictionary of corresponding plugin unit keys keyed by content type id. :param unit_ids_by_type: dictionary of <content type id> : <list of unit ids> :type unit_ids_by_type: dict :return: if units are specified, return the corresponding plugin unit_keys. If unit_ids_by_type dict is empty, return plugin unit keys corresponging to all units in given repo ids. If unit ids list for a particular unit type is empty, return all plugin unit_keys in given repo ids with that unit type. :rtype: dict """ repo_unit_association_query_manager = managers.repo_unit_association_query_manager() content_query_manager = managers.content_query_manager() result_unit_keys = {} if unit_ids_by_type is not None: for unit_type_id, unit_ids in unit_ids_by_type.items(): # Get unit type specific collection collection = content_query_manager.get_content_unit_collection(type_id=unit_type_id) type_def = content_types_db.type_definition(unit_type_id) if not unit_ids: # If unit_list is empty for a unit_type, consider all units of specific type criteria = UnitAssociationCriteria(unit_fields = ['unit_id']) for repo_id in repo_ids: repo_units = repo_unit_association_query_manager.get_units_by_type(repo_id, unit_type_id, criteria) # Get metadata for each unit from type specific collection pulp_units = [collection.find_one({'_id': u['unit_id']}) for u in repo_units] # Convert pulp units to plugin unit keys plugin_unit_keys = [common_utils.to_plugin_unit(u, type_def).unit_key for u in pulp_units] result_unit_keys.setdefault(unit_type_id, []).extend(plugin_unit_keys) else: # Get metadata for each unit from type specific collection pulp_units = [collection.find_one({'_id': unit_id}) for unit_id in unit_ids] # Convert pulp units to plugin unit keys plugin_unit_keys = [common_utils.to_plugin_unit(u, type_def).unit_key for u in pulp_units] result_unit_keys.setdefault(unit_type_id, []).extend(plugin_unit_keys) else: # If units are not specified, consider all units in given repos. for repo_id in repo_ids: all_unit_type_ids = content_types_db.all_type_ids() for unit_type_id in all_unit_type_ids: criteria = UnitAssociationCriteria(type_ids=[unit_type_id], unit_fields = ['unit_id', 'unit_type_id']) repo_units = repo_unit_association_query_manager.get_units(repo_id, criteria) # Get unit metadata for each unit from type specific collection collection = content_query_manager.get_content_unit_collection(type_id=unit_type_id) pulp_units = [collection.find_one({'_id': u['unit_id']}) for u in repo_units] # Convert pulp units to plugin unit keys type_def = content_types_db.type_definition(unit_type_id) plugin_unit_keys = [common_utils.to_plugin_unit(u, type_def).unit_key for u in pulp_units] result_unit_keys.setdefault(unit_type_id, []).extend(plugin_unit_keys) return result_unit_keys
def __parse_units(self, user_units, repo_ids): """ Parse units specified by user and return a dictionary of all plugin unit_keys to be considered for applicability keyed by unit_type_id. :param user_units: dictionary of unit metadata filters keyed by unit-type-id specified by user :type user_units: dict :return: if specific units are specified, return the corresponding plugin unit_keys. If units dict is empty, return all plugin unit_keys corresponging to units in given repo ids keyed by unit_type_id. If units list for a particular unit type in units is empty, return all plugin unit_keys in given repo ids with that unit type keyed by unit_type_id. :rtype: dict """ repo_unit_association_query_manager = managers.repo_unit_association_query_manager() content_query_manager = managers.content_query_manager() result_unit_keys = {} if user_units is not None: for unit_type_id, unit_list in user_units.items(): # Get unit type specific collection collection = content_query_manager.get_content_unit_collection(type_id=unit_type_id) type_def = content_types_db.type_definition(unit_type_id) if not unit_list: # If unit_list is empty for a unit_type, consider all units of specific type criteria = UnitAssociationCriteria(unit_fields = ['unit_id']) for repo_id in repo_ids: repo_units = repo_unit_association_query_manager.get_units_by_type(repo_id, unit_type_id, criteria) # Get unit metadata for each unit from type specific collection pulp_units = [collection.find_one({'_id': u['unit_id']}) for u in repo_units] plugin_units = [common_utils.to_plugin_unit(u, type_def) for u in pulp_units] plugin_unit_keys = [u.unit_key for u in plugin_units] result_unit_keys.setdefault(unit_type_id, []).extend(plugin_unit_keys) else: for unit in unit_list: criteria = UnitAssociationCriteria(unit_filters=unit, unit_fields = ['unit_id']) for repo_id in repo_ids: repo_units = repo_unit_association_query_manager.get_units_by_type(repo_id, unit_type_id, criteria) # Get unit metadata for each unit from type specific collection pulp_units = [collection.find_one({'_id': u['unit_id']}) for u in repo_units] plugin_units = [common_utils.to_plugin_unit(u, type_def) for u in pulp_units] plugin_unit_keys = [u.unit_key for u in plugin_units] result_unit_keys.setdefault(unit_type_id, []).extend(plugin_unit_keys) else: # If units are not specified, consider all units in repo_ids list. for repo_id in repo_ids: criteria = UnitAssociationCriteria(unit_fields = ['unit_id','unit_type_id']) repo_units = repo_unit_association_query_manager.get_units(repo_id, criteria) # Get unit metadata for each unit from type specific collection for repo_unit in repo_units: collection = content_query_manager.get_content_unit_collection(type_id=repo_unit['unit_type_id']) type_def = content_types_db.type_definition(repo_unit['unit_type_id']) pulp_unit = collection.find_one({'_id': repo_unit['unit_id']}) plugin_unit = common_utils.to_plugin_unit(pulp_unit, type_def) result_unit_keys.setdefault(repo_unit['unit_type_id'], []).append(plugin_unit.unit_key) return result_unit_keys
def _purge_orphaned_blobs(repo, units): """ Purge blobs associated with removed manifests when no longer referenced by any remaining manifests. :param repo: The affected repository. :type repo: pulp.plugins.model.Repository :param units: List of removed units. :type units: list of: pulp.plugins.model.AssociatedUnit """ # Find blob digests referenced by removed manifests (orphaned) orphaned = set() for unit in units: if unit.type_id != constants.MANIFEST_TYPE_ID: continue manifest = unit for layer in manifest.metadata['fs_layers']: digest = layer['blobSum'] orphaned.add(digest) # Find blob digests still referenced by other manifests (adopted) if not orphaned: # nothing orphaned return adopted = set() manager = manager_factory.repo_unit_association_query_manager() for manifest in manager.get_units_by_type(repo.id, constants.MANIFEST_TYPE_ID): for layer in manifest.metadata['fs_layers']: digest = layer['blobSum'] adopted.add(digest) # Remove unreferenced blobs orphaned = orphaned.difference(adopted) if not orphaned: # all adopted return unit_filter = { 'digest': { '$in': sorted(orphaned) } } criteria = UnitAssociationCriteria( type_ids=[constants.BLOB_TYPE_ID], unit_filters=unit_filter) manager = manager_factory.repo_unit_association_manager() manager.unassociate_by_criteria( repo_id=repo.id, criteria=criteria, owner_type='', # unused owner_id='', # unused notify_plugins=False)
def get_repo_units(self, repo_id, content_type_id, additional_unit_fields=None): """ Searches for units in the given repository with given content type and returns a plugin unit containing unit id, unit key and any additional fields requested. :param repo_id: repo id :type repo_id: str :param content_type_id: content type id of the units :type content_type_id: str :param additional_unit_fields: additional fields from the unit metadata to be added in the result :type additional_unit_fields: list of str :return: list of unit instances :rtype: list of pulp.plugins.model.Unit """ additional_unit_fields = additional_unit_fields or [] try: unit_key_fields = units_controller.get_unit_key_fields_for_type(content_type_id) serializer = units_controller.get_model_serializer_for_type(content_type_id) # Query repo association manager to get all units of given type # associated with given repo. Limit data by requesting only the fields # that are needed. query_manager = managers.repo_unit_association_query_manager() unit_fields = list(unit_key_fields) + list(additional_unit_fields) criteria = UnitAssociationCriteria(association_fields=['unit_id'], unit_fields=unit_fields) units = query_manager.get_units_by_type(repo_id, content_type_id, criteria) # Convert units to plugin units with unit_key and required metadata values for each unit all_units = [] for unit in units: if serializer: serializer.serialize(unit['metadata']) unit_key = {} metadata = {} for k in unit_key_fields: unit_key[k] = unit['metadata'].pop(k) # Add unit_id and any additional unit fields requested by plugins metadata['unit_id'] = unit.pop('unit_id') for field in additional_unit_fields: metadata[field] = unit['metadata'].pop(field, None) u = Unit(content_type_id, unit_key, metadata, None) all_units.append(u) return all_units except Exception, e: _logger.exception(_('Exception from server getting units from repo [%s]' % repo_id)) raise self.exception_class(e), None, sys.exc_info()[2]
def unassociate_by_criteria(self, repo_id, criteria, owner_type, owner_id, notify_plugins=True): """ Unassociate units that are matched by the given criteria. @param repo_id: identifies the repo @type repo_id: str @param criteria: @param owner_type: category of the caller who created the association @type owner_type: str @param owner_id: identifies the call who created the association @type owner_id: str @param notify_plugins: if true, relevant plugins will be informed of the removal @type notify_plugins: bool """ association_query_manager = manager_factory.repo_unit_association_query_manager() unassociate_units = association_query_manager.get_units(repo_id, criteria=criteria) if len(unassociate_units) is 0: return [] unit_map = {} # maps unit_type_id to a list of unit_ids for unit in unassociate_units: id_list = unit_map.setdefault(unit['unit_type_id'], []) id_list.append(unit['unit_id']) collection = RepoContentUnit.get_collection() repo_manager = manager_factory.repo_manager() for unit_type_id, unit_ids in unit_map.items(): spec = {'repo_id': repo_id, 'unit_type_id': unit_type_id, 'unit_id': {'$in': unit_ids}, 'owner_type': owner_type, 'owner_id': owner_id} collection.remove(spec, safe=True) unique_count = sum(1 for unit_id in unit_ids if not self.association_exists(repo_id, unit_id, unit_type_id)) if not unique_count: continue repo_manager.update_unit_count(repo_id, unit_type_id, -unique_count) # Convert the units into transfer units. This happens regardless of whether or not # the plugin will be notified as it's used to generate the return result, unit_type_ids = calculate_associated_type_ids(repo_id, unassociate_units) transfer_units = create_transfer_units(unassociate_units, unit_type_ids) if notify_plugins: remove_from_importer(repo_id, transfer_units) # Match the return type/format as copy serializable_units = [u.to_id_dict() for u in transfer_units] return serializable_units
def __populate_units(self, unit_ids_by_type, repo_ids): """ Parse a dictionary of unit ids keyed by content type id and populate units for each type if the criteria is empty. :param unit_ids_by_type: dictionary of <content type id> : <list of unit ids> :type unit_ids_by_type: dict :return: if units are specified, return the corresponding units. If unit_ids_by_type dict is empty, return unit ids corresponging to all units in given repo ids. If unit ids list for a particular unit type is empty, return all unit ids in given repo ids with that unit type. :rtype: dict """ repo_unit_association_query_manager = managers.repo_unit_association_query_manager( ) result_units = {} if unit_ids_by_type is not None: for unit_type_id, unit_ids in unit_ids_by_type.items(): # Get unit type specific collection if not unit_ids: # If unit_list is empty for a unit_type, consider all units of specific type criteria = Criteria(filters={ "repo_id": { "$in": repo_ids }, "unit_type_id": unit_type_id }, fields=['unit_id']) repo_units = repo_unit_association_query_manager.find_by_criteria( criteria) pulp_unit_ids = [u['unit_id'] for u in repo_units] result_units.setdefault(unit_type_id, []).extend( list(set(pulp_unit_ids))) else: result_units.setdefault(unit_type_id, []).extend(unit_ids) else: # If units are not specified, consider all units in given repos. all_unit_type_ids = content_types_db.all_type_ids() for unit_type_id in all_unit_type_ids: criteria = Criteria(filters={ "repo_id": { "$in": repo_ids }, "unit_type_id": unit_type_id }, fields=['unit_id']) repo_units = repo_unit_association_query_manager.find_by_criteria( criteria) pulp_unit_ids = [u['unit_id'] for u in repo_units] result_units.setdefault(unit_type_id, []).extend(list(set(pulp_unit_ids))) return result_units
def test_migrate_duplicates(self): """ This tests the correct behavior when we try to change the repo_id on an object, and end up causing a duplicate error due to our uniqueness constraint. """ # Let's put two units here with the same IDs with two different repo_ids, and the run the # migration. source_repo_group_id = add_unit('group', self.source_repo_id, ids.TYPE_ID_PKG_GROUP) dest_repo_group_id = add_unit('group', self.dest_repo_id, ids.TYPE_ID_PKG_GROUP) associate_unit(source_repo_group_id, self.dest_repo_id, ids.TYPE_ID_PKG_GROUP) associate_unit(dest_repo_group_id, self.dest_repo_id, ids.TYPE_ID_PKG_GROUP) # Migrate should not cause a DuplicateKeyError self.migration.migrate() # Verify that both groups remain. group_collection = types_db.type_units_collection( ids.TYPE_ID_PKG_GROUP) all_groups = list(group_collection.find()) self.assertEqual(len(all_groups), 2) self.assertEqual( group_collection.find({ 'id': 'group', 'repo_id': self.dest_repo_id }).count(), 1) self.assertEqual( group_collection.find({ 'id': 'group', 'repo_id': self.source_repo_id }).count(), 1) # Let's make sure that the dest group is associated, but not the source one query_manager = factory.repo_unit_association_query_manager() dest_units = query_manager.get_units(self.dest_repo_id) self.assertEqual(len(dest_units), 1) dest_unit = dest_units[0] self.assertEqual(dest_unit['unit_type_id'], ids.TYPE_ID_PKG_GROUP) self.assertEqual(dest_unit['unit_id'], dest_repo_group_id) self.assertEqual(query_manager.get_units(self.source_repo_id), []) # Verify the repo counts self.assertEqual( Repo.get_collection().find({'id': 'source-repo' })[0]['content_unit_counts'], {}) self.assertEqual( Repo.get_collection().find({'id': 'dest-repo' })[0]['content_unit_counts'], {'package_group': 1})
def get_repo_units(self, repo_id, content_type_id, additional_unit_fields=[]): """ Searches for units in the given repository with given content type and returns a plugin unit containing unit id, unit key and any additional fields requested. :param repo_id: repo id :type repo_id: str :param content_type_id: content type id of the units :type content_type_id: str :param additional_unit_fields: additional fields from the unit metadata to be added in the result :type additional_unit_fields: list of str :return: list of unit instances :rtype: list of pulp.plugins.model.Unit """ try: # Get type definition and unit_key for given content type type_def = types_db.type_definition(content_type_id) unit_key_fields = type_def["unit_key"] # Query repo association manager to get all units of given type # associated with given repo. Limit data by requesting only the fields # that are needed. query_manager = managers.repo_unit_association_query_manager() unit_fields = list(set(unit_key_fields + additional_unit_fields)) criteria = UnitAssociationCriteria(association_fields=["unit_id"], unit_fields=unit_fields) units = query_manager.get_units_by_type(repo_id, content_type_id, criteria) # Convert units to plugin units with unit_key and required metadata values for each unit all_units = [] for unit in units: unit_key = {} metadata = {} for k in unit_key_fields: unit_key[k] = unit["metadata"].pop(k) # Add unit_id and any additional unit fields requested by plugins metadata["unit_id"] = unit.pop("unit_id") for field in additional_unit_fields: metadata[field] = unit["metadata"].pop(field, None) u = Unit(content_type_id, unit_key, metadata, None) all_units.append(u) return all_units except Exception, e: _logger.exception(_("Exception from server getting units from repo [%s]" % repo_id)) raise self.exception_class(e), None, sys.exc_info()[2]
def __init__(self, source_repo_id, dest_repo_id, source_importer_id, dest_importer_id): ImporterScratchPadMixin.__init__(self, dest_repo_id, dest_importer_id) RepoScratchPadMixin.__init__(self, dest_repo_id, ImporterConduitException) self.source_repo_id = source_repo_id self.dest_repo_id = dest_repo_id self.source_importer_id = source_importer_id self.dest_importer_id = dest_importer_id self.__association_manager = manager_factory.repo_unit_association_manager() self.__association_query_manager = manager_factory.repo_unit_association_query_manager() self.__importer_manager = manager_factory.repo_importer_manager()
def calculate_associated_type_ids(source_repo_id, associated_units): if associated_units is not None: associated_unit_type_ids = set([u['unit_type_id'] for u in associated_units]) else: association_query_manager = manager_factory.repo_unit_association_query_manager() # We may want to make a call here that only retrieves the unique # type IDs instead of all of the units, but for now it doesn't exist # and I'm not entirely sure this will be a huge problem. all_units = association_query_manager.get_units(source_repo_id) associated_unit_type_ids = set(u['unit_type_id'] for u in all_units) return associated_unit_type_ids
def unassociate_by_criteria(self, repo_id, criteria, owner_type, owner_id): """ Unassociate units that are matched by the given criteria. @param repo_id: identifies the repo @type repo_id: str @param criteria: @param owner_type: category of the caller who created the association @type owner_type: str @param owner_id: identifies the call who created the association @type owner_id: str """ association_query_manager = manager_factory.repo_unit_association_query_manager() unassociate_units = association_query_manager.get_units(repo_id, criteria=criteria) if len(unassociate_units) is 0: return unit_map = {} # maps unit_type_id to a list of unit_ids for unit in unassociate_units: id_list = unit_map.setdefault(unit['unit_type_id'], []) id_list.append(unit['unit_id']) collection = RepoContentUnit.get_collection() repo_manager = manager_factory.repo_manager() for unit_type_id, unit_ids in unit_map.items(): spec = {'repo_id': repo_id, 'unit_type_id': unit_type_id, 'unit_id': {'$in': unit_ids}, 'owner_type': owner_type, 'owner_id': owner_id} collection.remove(spec, safe=True) unique_count = sum(1 for unit_id in unit_ids if not self.association_exists(repo_id, unit_id, unit_type_id)) if not unique_count: continue repo_manager.update_unit_count(repo_id, -unique_count) try: repo_query_manager = manager_factory.repo_query_manager() repo = repo_query_manager.get_repository(repo_id) importer_manager = manager_factory.repo_importer_manager() importer = importer_manager.get_importer(repo_id) importer.remove_units(repo, unassociate_units) except: _LOG.exception('Exception informing importer for [%s] of unassociation' % repo_id)
def test_migrate_duplicates_doesnt_delete_from_source_repo(self): """ This tests the correct behavior when we try to change the repo_id on an object, and end up causing a duplicate error due to our uniqueness constraint. It also makes sure the units are not deleted from the source repository if they are in the source repository. """ # Let's put two units here with the same IDs with two different repo_ids, and the run the # migration. source_repo_group_id = add_unit('group', self.source_repo_id, ids.TYPE_ID_PKG_GROUP) dest_repo_group_id = add_unit('group', self.dest_repo_id, ids.TYPE_ID_PKG_GROUP) # Associate the source_repo_group_id with both source and destination repos associate_unit(source_repo_group_id, self.source_repo_id, ids.TYPE_ID_PKG_GROUP) associate_unit(source_repo_group_id, self.dest_repo_id, ids.TYPE_ID_PKG_GROUP) associate_unit(dest_repo_group_id, self.dest_repo_id, ids.TYPE_ID_PKG_GROUP) # Migrate should not cause a DuplicateKeyError self.migration.migrate() # Verify that both groups remain, because the migration should not have removed either group_collection = types_db.type_units_collection(ids.TYPE_ID_PKG_GROUP) all_groups = list(group_collection.find()) self.assertEqual(len(all_groups), 2) self.assertEqual( group_collection.find({'id': 'group', 'repo_id': self.dest_repo_id}).count(), 1) self.assertEqual( group_collection.find({'id': 'group', 'repo_id': self.source_repo_id}).count(), 1) # Let's make sure that there are two associations, and that they are correct. query_manager = factory.repo_unit_association_query_manager() dest_units = query_manager.get_units(self.dest_repo_id) self.assertEqual(len(dest_units), 1) dest_unit = dest_units[0] self.assertEqual(dest_unit['unit_type_id'], ids.TYPE_ID_PKG_GROUP) self.assertEqual(dest_unit['unit_id'], dest_repo_group_id) source_units = query_manager.get_units(self.source_repo_id) self.assertEqual(len(source_units), 1) source_unit = source_units[0] self.assertEqual(source_unit['unit_type_id'], ids.TYPE_ID_PKG_GROUP) self.assertEqual(source_unit['unit_id'], source_repo_group_id) # Verify the repo counts self.assertEqual( Repo.get_collection().find({'id': 'source-repo'})[0]['content_unit_counts'], {'package_group': 1}) self.assertEqual(Repo.get_collection().find({'id': 'dest-repo'})[0]['content_unit_counts'], {'package_group': 1})
def get_repo_units(self, repo_id, content_type_id, additional_unit_fields=[]): """ Searches for units in the given repository with given content type and returns a plugin unit with containing unit key and any additional fields requested. :param repo_id: repo id :type repo_id: str :param content_type_id: content type id of the units :type content_type_id: str :param additional_unit_fields: additional fields from the unit metadata to be added in the result :type additional_unit_fields: list of str :return: list of unit instances :rtype: list of pulp.plugins.model.Unit """ try: query_manager = managers.repo_unit_association_query_manager() criteria = UnitAssociationCriteria(type_ids=[content_type_id]) units = query_manager.get_units(repo_id, criteria) # Get type definition and unit_key for given content type type_def = types_db.type_definition(content_type_id) key_list = type_def['unit_key'] # Return plugin units with unit_key and required metadata values for each unit all_units = [] for unit in units: unit_key = {} metadata = {} for k in key_list: unit_key[k] = unit['metadata'].pop(k) # Add unit_id and any additional unit fields requested by plugins metadata['unit_id'] = unit.pop('unit_id', None) for field in additional_unit_fields: metadata[field] = unit['metadata'].pop(field, None) u = Unit(content_type_id, unit_key, metadata, None) all_units.append(u) return all_units except Exception, e: _LOG.exception(_('Exception from server getting units from repo [%s]' % repo_id)) raise self.exception_class(e), None, sys.exc_info()[2]
def unassociate_by_criteria(self, repo_id, criteria, owner_type, owner_id, notify_plugins=True): """ Unassociate units that are matched by the given criteria. @param repo_id: identifies the repo @type repo_id: str @param criteria: @param owner_type: category of the caller who created the association @type owner_type: str @param owner_id: identifies the call who created the association @type owner_id: str @param notify_plugins: if true, relevant plugins will be informed of the removal @type notify_plugins: bool """ association_query_manager = manager_factory.repo_unit_association_query_manager() unassociate_units = association_query_manager.get_units(repo_id, criteria=criteria) if len(unassociate_units) is 0: return unit_map = {} # maps unit_type_id to a list of unit_ids for unit in unassociate_units: id_list = unit_map.setdefault(unit['unit_type_id'], []) id_list.append(unit['unit_id']) collection = RepoContentUnit.get_collection() repo_manager = manager_factory.repo_manager() for unit_type_id, unit_ids in unit_map.items(): spec = {'repo_id': repo_id, 'unit_type_id': unit_type_id, 'unit_id': {'$in': unit_ids}, 'owner_type': owner_type, 'owner_id': owner_id} collection.remove(spec, safe=True) unique_count = sum(1 for unit_id in unit_ids if not self.association_exists(repo_id, unit_id, unit_type_id)) if not unique_count: continue repo_manager.update_unit_count(repo_id, unit_type_id, -unique_count) if notify_plugins: remove_from_importer(repo_id, unassociate_units)
def test_migrate_groups(self): # Setup orig_group_id = add_unit('g1', self.source_repo_id, ids.TYPE_ID_PKG_GROUP) associate_unit(orig_group_id, self.source_repo_id, ids.TYPE_ID_PKG_GROUP) associate_unit(orig_group_id, self.dest_repo_id, ids.TYPE_ID_PKG_GROUP) # Test self.migration.migrate() # Verify # Verify a new group was created with the correct metadata group_coll = types_db.type_units_collection(ids.TYPE_ID_PKG_GROUP) all_groups = group_coll.find({}).sort('repo_id', 1) self.assertEqual(2, all_groups.count()) dest_group = all_groups[0] # ordered by ID, this will be first self.assertEqual(dest_group['id'], 'g1') self.assertEqual(dest_group['repo_id'], self.dest_repo_id) source_group = all_groups[1] self.assertEqual(source_group['id'], 'g1') self.assertEqual(source_group['repo_id'], self.source_repo_id) # Verify the associations query_manager = factory.repo_unit_association_query_manager() source_units = query_manager.get_units(self.source_repo_id) self.assertEqual(1, len(source_units)) self.assertEqual(source_units[0]['unit_type_id'], ids.TYPE_ID_PKG_GROUP) self.assertEqual(source_units[0]['unit_id'], source_group['_id']) dest_units = query_manager.get_units(self.dest_repo_id) self.assertEqual(1, len(dest_units)) self.assertEqual(dest_units[0]['unit_type_id'], ids.TYPE_ID_PKG_GROUP) self.assertEqual(dest_units[0]['unit_id'], dest_group['_id'])
def test_migrate_duplicates(self): """ This tests the correct behavior when we try to change the repo_id on an object, and end up causing a duplicate error due to our uniqueness constraint. """ # Let's put two units here with the same IDs with two different repo_ids, and the run the # migration. source_repo_group_id = add_unit('group', self.source_repo_id, ids.TYPE_ID_PKG_GROUP) dest_repo_group_id = add_unit('group', self.dest_repo_id, ids.TYPE_ID_PKG_GROUP) associate_unit(source_repo_group_id, self.dest_repo_id, ids.TYPE_ID_PKG_GROUP) associate_unit(dest_repo_group_id, self.dest_repo_id, ids.TYPE_ID_PKG_GROUP) # Migrate should not cause a DuplicateKeyError self.migration.migrate() # Verify that both groups remain. group_collection = types_db.type_units_collection(ids.TYPE_ID_PKG_GROUP) all_groups = list(group_collection.find()) self.assertEqual(len(all_groups), 2) self.assertEqual( group_collection.find({'id': 'group', 'repo_id': self.dest_repo_id}).count(), 1) self.assertEqual( group_collection.find({'id': 'group', 'repo_id': self.source_repo_id}).count(), 1) # Let's make sure that the dest group is associated, but not the source one query_manager = factory.repo_unit_association_query_manager() dest_units = query_manager.get_units(self.dest_repo_id) self.assertEqual(len(dest_units), 1) dest_unit = dest_units[0] self.assertEqual(dest_unit['unit_type_id'], ids.TYPE_ID_PKG_GROUP) self.assertEqual(dest_unit['unit_id'], dest_repo_group_id) self.assertEqual(query_manager.get_units(self.source_repo_id), []) # Verify the repo counts source = model.Repository.objects.get(repo_id='source-repo') self.assertDictEqual(source.content_unit_counts, {}) dest = model.Repository.objects.get(repo_id='dest-repo') self.assertDictEqual(dest.content_unit_counts, {'package_group': 1})
def __init__(self, source_repo_id, dest_repo_id, source_importer_id, dest_importer_id, association_owner_type, association_owner_id): """ :param source_repo_id: ID of the repository from which units are being copied :type source_repo_id: str :param dest_repo_id: ID of the repository into which units are being copied :type dest_repo_id: str :param source_importer_id: ID of the importer on the source repository :type source_importer_id: str :param dest_importer_id: ID of the importer on the destination repository :type dest_importer_id: str :param association_owner_type: distinguishes the owner when creating an association through this conduit :type association_owner_type: str :param association_owner_id: specific ID of the owner when creating an association through this conduit :type association_owner_id: str """ ImporterScratchPadMixin.__init__(self, dest_repo_id, dest_importer_id) RepoScratchPadMixin.__init__(self, dest_repo_id, ImporterConduitException) SearchUnitsMixin.__init__(self, ImporterConduitException) AddUnitMixin.__init__(self, dest_repo_id, dest_importer_id, association_owner_type, association_owner_id) self.source_repo_id = source_repo_id self.dest_repo_id = dest_repo_id self.source_importer_id = source_importer_id self.dest_importer_id = dest_importer_id self.association_owner_type = association_owner_type self.association_owner_id = association_owner_id self.__association_manager = manager_factory.repo_unit_association_manager( ) self.__association_query_manager = manager_factory.repo_unit_association_query_manager( ) self.__importer_manager = manager_factory.repo_importer_manager()
def __populate_units(self, unit_ids_by_type, repo_ids): """ Parse a dictionary of unit ids keyed by content type id and populate units for each type if the criteria is empty. :param unit_ids_by_type: dictionary of <content type id> : <list of unit ids> :type unit_ids_by_type: dict :return: if units are specified, return the corresponding units. If unit_ids_by_type dict is empty, return unit ids corresponging to all units in given repo ids. If unit ids list for a particular unit type is empty, return all unit ids in given repo ids with that unit type. :rtype: dict """ repo_unit_association_query_manager = managers.repo_unit_association_query_manager() result_units = {} if unit_ids_by_type is not None: for unit_type_id, unit_ids in unit_ids_by_type.items(): # Get unit type specific collection if not unit_ids: # If unit_list is empty for a unit_type, consider all units of specific type criteria = Criteria(filters={"repo_id": {"$in": repo_ids}, "unit_type_id":unit_type_id}, fields=['unit_id']) repo_units = repo_unit_association_query_manager.find_by_criteria(criteria) pulp_unit_ids = [u['unit_id'] for u in repo_units] result_units.setdefault(unit_type_id, []).extend(list(set(pulp_unit_ids))) else: result_units.setdefault(unit_type_id, []).extend(unit_ids) else: # If units are not specified, consider all units in given repos. all_unit_type_ids = content_types_db.all_type_ids() for unit_type_id in all_unit_type_ids: criteria = Criteria(filters={"repo_id": {"$in": repo_ids}, "unit_type_id":unit_type_id}, fields=['unit_id']) repo_units = repo_unit_association_query_manager.find_by_criteria(criteria) pulp_unit_ids = [u['unit_id'] for u in repo_units] result_units.setdefault(unit_type_id, []).extend(list(set(pulp_unit_ids))) return result_units
def verify(self, num_units=PluginTestBase.NUM_UNITS): # repository manager = managers.repo_query_manager() manager.get_repository(self.REPO_ID) # importer manager = managers.repo_importer_manager() importer = manager.get_importer(self.REPO_ID) manifest_url = importer['config']['manifest_url'] self.assertTrue(manifest_url.endswith('%s/units.json.gz' % self.REPO_ID)) # distributor manager = managers.repo_distributor_manager() distributor = manager.get_distributor(self.REPO_ID, CITRUS_DISTRUBUTOR) protocol = distributor['config']['protocol'] self.assertEqual(protocol, 'file') alias = distributor['config'][protocol]['alias'] self.assertEqual(alias[0], self.upfs) self.assertEqual(alias[1], self.upfs) # check units manager = managers.repo_unit_association_query_manager() units = manager.get_units(self.REPO_ID) units = dict([(u['metadata']['N'], u) for u in units]) self.assertEqual(len(units), num_units) for n in range(0, num_units): unit = units[n] unit_id = self.UNIT_ID % n metadata = unit['metadata'] storage_path = metadata['_storage_path'].replace('//', '/') self.assertEqual(unit['unit_type_id'], self.UNIT_TYPE_ID) self.assertEqual(unit['repo_id'], self.REPO_ID) self.assertEqual(unit['owner_id'], CITRUS_IMPORTER) file = '.'.join((unit_id, self.UNIT_TYPE_ID)) self.assertEqual(storage_path, os.path.join(self.downfs, 'content', file)) self.assertTrue(os.path.exists(storage_path)) fp = open(storage_path) content = fp.read() fp.close() self.assertEqual(content, unit_id)
import copy import logging import pymongo.errors from pulp.server.managers import factory from pulp.server.managers.repo.cud import RepoContentUnit from pulp.server.managers.repo.unit_association_query import UnitAssociationCriteria from pulp_rpm.common import ids _log = logging.getLogger('pulp') # Initialize plugin loader api and other managers factory.initialize() ass_query_mgr = factory.repo_unit_association_query_manager() ass_mgr = factory.repo_unit_association_manager() content_mgr = factory.content_manager() repo_mgr = factory.repo_manager() def _get_repos(): """ Lookups all the yum based repos in pulp. @return a list of repoids """ repos = factory.repo_query_manager().find_with_importer_type( "yum_importer") if not repos: _log.debug("No repos found to perform db migrate") return []