def delete_orphan_content_units_by_type(type_id, content_unit_ids=None):
"""
Delete the orphaned content units for the given content type.
This method only applies to new style content units that are loaded via entry points
NOTE: this method deletes the content unit's bits from disk, if applicable.
:param type_id: id of the content type
:type type_id: basestring
:param content_unit_ids: list of content unit ids to delete; None means delete them all
:type content_unit_ids: iterable or None
:return: count of units deleted
:rtype: int
"""
# get the model matching the type
content_model = plugin_api.get_unit_model_by_id(type_id)
if content_unit_ids:
query_sets = []
for page in plugin_misc.paginate(content_unit_ids):
qs = content_model.objects(id__in=page).only(
'id', '_storage_path')
query_sets.append(qs)
content_units = itertools.chain(*query_sets)
else:
content_units = content_model.objects.only('id', '_storage_path')
count = 0
# Paginate the content units
for units_group in plugin_misc.paginate(content_units):
# Build the list of ids to search for an easier way to access units in the group by id
unit_dict = dict()
for unit in units_group:
unit_dict[unit.id] = unit
id_list = list(unit_dict.iterkeys())
# Clear the units that are currently associated from unit_dict
non_orphan = model.RepositoryContentUnit.objects(unit_id__in=id_list)\
.distinct('unit_id')
for non_orphan_id in non_orphan:
unit_dict.pop(non_orphan_id)
# Remove the unit, lazy catalog entries, and any content in storage.
for unit_to_delete in unit_dict.itervalues():
model.LazyCatalogEntry.objects(
unit_id=str(unit_to_delete.id),
unit_type_id=str(type_id)).delete()
unit_to_delete.delete()
if unit_to_delete._storage_path:
OrphanManager.delete_orphaned_file(
unit_to_delete._storage_path)
count += 1
return count
def delete_orphan_content_units_by_type(type_id, content_unit_ids=None):
"""
Delete the orphaned content units for the given content type.
This method only applies to new style content units that are loaded via entry points
NOTE: this method deletes the content unit's bits from disk, if applicable.
:param type_id: id of the content type
:type type_id: basestring
:param content_unit_ids: list of content unit ids to delete; None means delete them all
:type content_unit_ids: iterable or None
:return: count of units deleted
:rtype: int
"""
# get the model matching the type
content_model = plugin_api.get_unit_model_by_id(type_id)
if content_unit_ids:
query_sets = []
for page in plugin_misc.paginate(content_unit_ids):
qs = content_model.objects(id__in=page).only('id', '_storage_path')
query_sets.append(qs)
content_units = itertools.chain(*query_sets)
else:
content_units = content_model.objects.only('id', '_storage_path')
count = 0
# Paginate the content units
for units_group in plugin_misc.paginate(content_units):
# Build the list of ids to search for an easier way to access units in the group by id
unit_dict = dict()
for unit in units_group:
unit_dict[unit.id] = unit
id_list = list(unit_dict.iterkeys())
# Clear the units that are currently associated from unit_dict
non_orphan = model.RepositoryContentUnit.objects(unit_id__in=id_list)\
.distinct('unit_id')
for non_orphan_id in non_orphan:
unit_dict.pop(non_orphan_id)
# Remove the unit, lazy catalog entries, and any content in storage.
for unit_to_delete in unit_dict.itervalues():
model.LazyCatalogEntry.objects(
unit_id=str(unit_to_delete.id),
unit_type_id=str(type_id)
).delete()
unit_to_delete.delete()
if unit_to_delete._storage_path:
OrphanManager.delete_orphaned_file(unit_to_delete._storage_path)
count += 1
return count
def get_unit_model_querysets(repo_id, model_class, repo_content_unit_q=None):
"""
Return a generator of mongoengine.queryset.QuerySet objects that collectively represent the
units in the specified repo that are of the type corresponding to the model_class and that
match the optional query.
Results are broken up into multiple QuerySet objects, because units are requested by their ID,
and we do not want to exceed the maximum size for a single query by stuffing too many IDs in one
QuerySet object.
You are welcome and encouraged to convert the return value into one generator of ContentUnit
objects by using itertools.chain()
:param repo_id: ID of the repo whose units should be queried
:type repo_id: str
:param model_class: a subclass of ContentUnit that defines a unit model
:type model_class: pulp.server.db.model.ContentUnit
:param repo_content_unit_q: any additional filters that should be applied to the
RepositoryContentUnit search
:type repo_content_unit_q: mongoengine.Q
:return: generator of mongoengine.queryset.QuerySet
:rtype: generator
"""
for chunk in paginate(get_associated_unit_ids(repo_id,
model_class._content_type_id.default,
repo_content_unit_q)):
yield model_class.objects(id__in=chunk)
def delete_orphan_content_units_by_type(type_id):
"""
Delete the orphaned content units for the given content type.
This method only applies to new style content units that are loaded via entry points
NOTE: this method deletes the content unit's bits from disk, if applicable.
:param type_id: id of the content type
:type type_id: basestring
"""
# get the model matching the type
content_model = plugin_api.get_unit_model_by_id(type_id)
content_units = content_model.objects().only('id', 'storage_path')
# Paginate the content units
for units_group in plugin_misc.paginate(content_units):
# Build the list of ids to search for an easier way to access units in the group by id
unit_dict = dict()
for unit in units_group:
unit_dict[unit.id] = unit
id_list = list(unit_dict.iterkeys())
# Clear the units that are currently associated from unit_dict
non_orphan = model.RepositoryContentUnit.objects(unit_id__in=id_list)\
.distinct('unit_id')
for non_orphan_id in non_orphan:
unit_dict.pop(non_orphan_id)
# Remove the unit and any references on disk
for unit_to_delete in unit_dict.itervalues():
unit_to_delete.delete()
if unit_to_delete.storage_path:
OrphanManager.delete_orphaned_file(
unit_to_delete.storage_path)
def get_multiple_units_by_keys_dicts(self,
content_type,
unit_keys_dicts,
model_fields=None):
"""
Look up multiple content units in the collection for the given content
type collection that match the list of keys dictionaries.
:param content_type: unique id of content collection
:type content_type: str
:param unit_keys_dicts: list of dictionaries whose key, value pairs can
uniquely identify a content unit
:type unit_keys_dicts: list of dict's
:param model_fields: fields of each content unit to report,
None means all fields
:type model_fields: None or list of str's
:return: tuple of content units found in the content type collection
that match the given unit keys dictionaries
:rtype: (possibly empty) tuple of dict's
:raises ValueError: if any of the keys dictionaries are invalid
"""
collection = content_types_db.type_units_collection(content_type)
for segment in paginate(unit_keys_dicts, page_size=50):
spec = _build_multi_keys_spec(content_type, segment)
cursor = collection.find(spec, fields=model_fields)
for unit_dict in cursor:
yield unit_dict
def delete_orphan_content_units_by_type(type_id):
"""
Delete the orphaned content units for the given content type.
This method only applies to new style content units that are loaded via entry points
NOTE: this method deletes the content unit's bits from disk, if applicable.
:param type_id: id of the content type
:type type_id: basestring
"""
# get the model matching the type
content_model = plugin_api.get_unit_model_by_id(type_id)
content_units = content_model.objects().only('id', 'storage_path')
# Paginate the content units
for units_group in plugin_misc.paginate(content_units):
# Build the list of ids to search for an easier way to access units in the group by id
unit_dict = dict()
for unit in units_group:
unit_dict[unit.id] = unit
id_list = list(unit_dict.iterkeys())
# Clear the units that are currently associated from unit_dict
non_orphan = model.RepositoryContentUnit.objects(unit_id__in=id_list)\
.distinct('unit_id')
for non_orphan_id in non_orphan:
unit_dict.pop(non_orphan_id)
# Remove the unit and any references on disk
for unit_to_delete in unit_dict.itervalues():
unit_to_delete.delete()
if unit_to_delete.storage_path:
OrphanManager.delete_orphaned_file(unit_to_delete.storage_path)
def process_main(self, item=None):
"""
given the passed-in unit keys, determine which of them already exist in
pulp, and save those with the conduit found on the parent.
:param item: The item to process or none if get_iterator is not defined
:param item: object or None
"""
# any units that are already in pulp
units_we_already_had = set()
# If available_units was defined in the constructor, let's use it. Otherwise let's use the
# default of self.parent.available_units
available_units = self.available_units or self.parent.available_units
for units_group in misc.paginate(available_units, self.unit_pagination_size):
# Get this group of units
query = units_controller.find_units(units_group)
for found_unit in query:
units_we_already_had.add(hash(found_unit))
repo_controller.associate_single_unit(self.get_repo().repo_obj, found_unit)
for unit in units_group:
if hash(unit) not in units_we_already_had:
self.units_to_download.append(unit)
def process_main(self, item=None):
"""
given the passed-in unit keys, determine which of them already exist in
pulp, and save those with the conduit found on the parent.
:param item: The item to process or none if get_iterator is not defined
:param item: object or None
"""
# any units that are already in pulp
units_we_already_had = set()
# If available_units was defined in the constructor, let's use it. Otherwise let's use the
# default of self.parent.available_units
if self.available_units is not None:
available_units = self.available_units
else:
available_units = self.parent.available_units
for units_group in misc.paginate(available_units,
self.unit_pagination_size):
# Get this group of units
query = units_controller.find_units(units_group)
for found_unit in query:
units_we_already_had.add(hash(found_unit))
repo_controller.associate_single_unit(self.get_repo().repo_obj,
found_unit)
for unit in units_group:
if hash(unit) not in units_we_already_had:
self.units_to_download.append(unit)
def queue_regenerate_applicability_for_repos(repo_criteria):
"""
Queue a group of tasks to generate and save applicability data affected by given updated
repositories.
:param repo_criteria: The repo selection criteria
:type repo_criteria: dict
"""
repo_criteria = Criteria.from_dict(repo_criteria)
# Process repo criteria
repo_criteria.fields = ['id']
repo_ids = [
r.repo_id
for r in model.Repository.objects.find_by_criteria(repo_criteria)
]
task_group_id = uuid4()
for repo_id in repo_ids:
profile_hashes = RepoProfileApplicability.get_collection().find(
{'repo_id': repo_id}, {'profile_hash': 1})
for batch in paginate(profile_hashes, 10):
batch_regenerate_applicability_task.apply_async(
(repo_id, batch), **{'group_id': task_group_id})
return task_group_id
def get_existing_units(search_dicts, unit_class, repo):
"""
Get units from the given repository that match the search terms. The unit instances will only
have their unit key fields populated.
:param search_dicts: iterable of dictionaries that should be used to search units
:type search_dicts: iterable
:param unit_class: subclass representing the type of unit to search for
:type unit_class: pulp_rpm.plugins.db.models.Package
:param repo: repository to search in
:type repo: pulp.server.db.model.Repository
:return: generator of unit_class instances with only their unit key fields populated
:rtype: generator
"""
unit_fields = unit_class.unit_key_fields
for segment in paginate(search_dicts):
unit_filters = {'$or': list(segment)}
units_q = mongoengine.Q(__raw__=unit_filters)
association_q = mongoengine.Q(unit_type_id=unit_class._content_type_id.default)
for result in repo_controller.find_repo_content_units(repo, units_q=units_q,
repo_content_unit_q=association_q,
unit_fields=unit_fields,
yield_content_unit=True):
yield result
def find_units(units, pagination_size=50):
"""
Query for units matching the unit key fields of an iterable of ContentUnit objects.
This requires that all the ContentUnit objects are of the same content type.
:param units: Iterable of content units with the unit key fields specified.
:type units: iterable of pulp.server.db.model.ContentUnit
:param pagination_size: How large a page size to use when querying units.
:type pagination_size: int (default 50)
:returns: unit models that pulp already knows about.
:rtype: Generator of pulp.server.db.model.ContentUnit
"""
# get the class from the first unit
model_class = None
for units_group in misc.paginate(units, pagination_size):
q_object = mongoengine.Q()
# Build a query for the units in this group
for unit in units_group:
if model_class is None:
model_class = unit.__class__
# Build the query for all the units, the | operator here
# creates the equivalent of a mongo $or of all the unit keys
unit_q_obj = mongoengine.Q(**unit.unit_key)
q_object = q_object | unit_q_obj
# Get this group of units
query = model_class.objects(q_object)
for found_unit in query:
yield found_unit
def get_existing_units(search_dicts, unit_class, repo):
"""
Get units from the given repository that match the search terms. The unit instances will only
have their unit key fields populated.
:param search_dicts: iterable of dictionaries that should be used to search units
:type search_dicts: iterable
:param unit_class: subclass representing the type of unit to search for
:type unit_class: pulp_rpm.plugins.db.models.Package
:param repo: repository to search in
:type repo: pulp.server.db.model.Repository
:return: generator of unit_class instances with only their unit key fields populated
:rtype: generator
"""
unit_fields = unit_class.unit_key_fields
for segment in paginate(search_dicts):
unit_filters = {'$or': list(segment)}
units_q = mongoengine.Q(__raw__=unit_filters)
association_q = mongoengine.Q(unit_type_id=unit_class._content_type_id.default)
for result in repo_controller.find_repo_content_units(repo, units_q=units_q,
repo_content_unit_q=association_q,
unit_fields=unit_fields,
yield_content_unit=True):
yield result
def test_empty_list(self):
ret = misc.paginate([], 3)
self.assertTrue(inspect.isgenerator(ret))
pieces = list(ret)
self.assertEqual(pieces, [])
def test_empty_list(self):
ret = misc.paginate([], 3)
self.assertTrue(inspect.isgenerator(ret))
pieces = list(ret)
self.assertEqual(pieces, [])
def test_list(self):
iterable = list(range(10))
ret = misc.paginate(iterable, 3)
self.assertTrue(inspect.isgenerator(ret))
pieces = list(ret)
self.assertEqual(pieces, [(0,1,2), (3,4,5), (6,7,8), (9,)])
def test_generator(self):
iterable = (x for x in range(10))
ret = misc.paginate(iterable, 3)
self.assertTrue(inspect.isgenerator(ret))
pieces = list(ret)
self.assertEqual(pieces, [(0, 1, 2), (3, 4, 5), (6, 7, 8), (9, )])
def test_generator_one_page(self):
iterable = (x for x in range(10))
ret = misc.paginate(iterable, 100)
self.assertTrue(inspect.isgenerator(ret))
pieces = list(ret)
self.assertEqual(pieces, [tuple(range(10))])
def test_generator_one_page(self):
iterable = (x for x in range(10))
ret = misc.paginate(iterable, 100)
self.assertTrue(inspect.isgenerator(ret))
pieces = list(ret)
self.assertEqual(pieces, [tuple(range(10))])
def disassociate_units(repository, unit_iterable):
"""
Disassociate all units in the iterable from the repository
:param repository: The repository to update.
:type repository: pulp.server.db.model.Repository
:param unit_iterable: The units to disassociate from the repository.
:type unit_iterable: iterable of pulp.server.db.model.ContentUnit
"""
for unit_group in misc.paginate(unit_iterable):
unit_id_list = [unit.id for unit in unit_group]
qs = model.RepositoryContentUnit.objects(
repo_id=repository.repo_id, unit_id__in=unit_id_list)
qs.delete()
def disassociate_units(repository, unit_iterable):
"""
Disassociate all units in the iterable from the repository
:param repository: The repository to update.
:type repository: pulp.server.db.model.Repository
:param unit_iterable: The units to disassociate from the repository.
:type unit_iterable: iterable of pulp.server.db.model.ContentUnit
"""
for unit_group in misc.paginate(unit_iterable):
unit_id_list = [unit.id for unit in unit_group]
qs = model.RepositoryContentUnit.objects(repo_id=repository.repo_id,
unit_id__in=unit_id_list)
qs.delete()
def get_requirements(self, units):
"""
For an iterable of RPM Units, return a generator of Require() instances that
represent the requirements for those RPMs.
:param units: iterable of pulp_rpm.plugins.models.RPM
:type units: iterable
:return: generator of Require() instances
:rtype: generator
"""
for segment in paginate(units):
for unit in segment:
for require in unit.requires or []:
yield Requirement(**require)
def get_requirements(self, units):
"""
For an iterable of RPM Units, return a generator of Require() instances that
represent the requirements for those RPMs.
:param units: iterable of pulp_rpm.plugins.models.RPM
:type units: iterable
:return: generator of Require() instances
:rtype: generator
"""
for segment in paginate(units):
for unit in segment:
for require in unit.requires or []:
yield Requirement(**require)
def open_cursors(unit_ids):
"""
Get a generator of unit cursors.
:param unit_ids: A dictionary of unit_ids keyed by type_id.
:type unit_ids: dict
:return: A list of open cursors.
:rtype: generator
"""
for type_id, id_list in unit_ids.items():
for page in paginate(id_list):
query = {'_id': {'$in': page}}
collection = type_units_collection(type_id)
cursor = collection.find(query)
yield cursor
def get_existing_units(search_dicts, unit_fields, unit_type, search_method):
"""
:param search_dicts:
:param unit_fields:
:param unit_type:
:param search_method:
:return: generator of Units
"""
for segment in paginate(search_dicts):
unit_filters = {'$or': list(segment)}
criteria = UnitAssociationCriteria([unit_type], unit_filters=unit_filters,
unit_fields=unit_fields, association_fields=[])
for result in search_method(criteria):
yield result
def open_cursors(unit_ids):
"""
Get a generator of unit cursors.
:param unit_ids: A dictionary of unit_ids keyed by type_id.
:type unit_ids: dict
:return: A list of open cursors.
:rtype: generator
"""
for type_id, id_list in unit_ids.items():
for page in paginate(id_list):
query = {'_id': {'$in': page}}
collection = type_units_collection(type_id)
cursor = collection.find(query)
yield cursor
def get_requirements(self, units):
"""
For an iterable of RPM Units, return a generator of Require() instances that
represent the requirements for those RPMs.
:param units: iterable of RPMs for which a query should be performed to
retrieve their Requires entries.
:type units: iterable of pulp.plugins.model.Unit
:return: generator of Require() instances
:rtype: generator
"""
for segment in paginate(units):
unit_ids = [unit.id for unit in segment]
fields = ['requires', 'id']
for result in models.RPM.objects.filter(id__in=unit_ids).only(*fields):
for require in result.requires or []:
yield Requirement(**require)
def get_requirements(self, units):
"""
For an iterable of RPM Units, return a generator of Require() instances that
represent the requirements for those RPMs.
:param units: iterable of RPMs for which a query should be performed to
retrieve their Requires entries.
:type units: iterable of pulp.plugins.model.Unit
:return: generator of Require() instances
:rtype: generator
"""
for segment in paginate(units):
unit_ids = [unit.id for unit in segment]
fields = ['requires', 'id']
for result in models.RPM.objects.filter(id__in=unit_ids).only(
*fields):
for require in result.requires or []:
yield Requirement(**require)
def get_requirements(self, units):
"""
For an iterable of RPM Units, return a generator of Require() instances that
represent the requirements for those RPMs.
:param units: iterable of pulp_rpm.plugins.models.RPM
:type units: iterable
:return: generator of Require() instances
:rtype: generator
"""
for segment in paginate(units):
unit_ids = [unit.id for unit in segment]
# The "requires" field is required by this workflow, but is not populated on the
# incoming "units". Thus we need to re-fetch units with that field.
fields = ['requires', 'id']
for result in models.RPM.objects.filter(id__in=unit_ids).only(*fields):
for require in result.requires or []:
yield Requirement(**require)
def get_requirements(self, units):
"""
For an iterable of RPM Units, return a generator of Require() instances that
represent the requirements for those RPMs.
:param units: iterable of pulp_rpm.plugins.models.RPM
:type units: iterable
:return: generator of Require() instances
:rtype: generator
"""
for segment in paginate(units):
unit_ids = [unit.id for unit in segment]
# The "requires" field is required by this workflow, but is not populated on the
# incoming "units". Thus we need to re-fetch units with that field.
fields = ['requires', 'id']
for result in models.RPM.objects.filter(id__in=unit_ids).only(
*fields):
for require in result.requires or []:
yield Requirement(**require)
def get_content_unit_ids(content_type, unit_keys):
"""
Return a generator of ids that uniquely identify the content units that match the
given unique keys dictionaries.
:param content_type: unique id of content collection
:type content_type: str
:param unit_keys: list of keys dictionaries that uniquely identify
content units in the given content type collection
:type unit_keys: list of dicts
:return: generator of unit IDs as strings
:rtype: generator
"""
collection = content_types_db.type_units_collection(content_type)
for segment in paginate(unit_keys):
spec = _build_multi_keys_spec(content_type, segment)
fields = ['_id']
for item in collection.find(spec, fields=fields):
yield str(item['_id'])
def get_content_unit_ids(content_type, unit_keys):
"""
Return a generator of ids that uniquely identify the content units that match the
given unique keys dictionaries.
:param content_type: unique id of content collection
:type content_type: str
:param unit_keys: list of keys dictionaries that uniquely identify
content units in the given content type collection
:type unit_keys: list of dicts
:return: generator of unit IDs as strings
:rtype: generator
"""
collection = content_types_db.type_units_collection(content_type)
for segment in paginate(unit_keys):
spec = _build_multi_keys_spec(content_type, segment)
fields = ["_id"]
for item in collection.find(spec, fields=fields):
yield str(item["_id"])
def get_requirements(self, units):
"""
For an iterable of RPM Units, return a generator of Require() instances that
represent the requirements for those RPMs.
:param units: iterable of RPMs for which a query should be performed to
retrieve their Requires entries.
:type units: iterable of pulp.plugins.model.Unit
:return: generator of Require() instances
:rtype: generator
"""
for segment in paginate(units):
search_dicts = [unit.unit_key for unit in segment]
filters = {'$or': search_dicts}
fields = list(models.RPM.UNIT_KEY_NAMES)
fields.extend(['requires', 'id'])
criteria = UnitAssociationCriteria(type_ids=[models.RPM.TYPE], unit_filters=filters, unit_fields=fields)
for result in self.search_method(criteria):
for require in result.metadata.get('requires', []):
yield Requirement(**require)
def process_main(self):
"""
given the passed-in unit keys, determine which of them already exist in
pulp, and save those with the conduit found on the parent.
"""
# any units that are already in pulp
units_we_already_had = set()
# mongodb throws exceptions for too big queries, so we spilt it into multiple smaller ones
for page in misc.paginate(self.parent.available_units, 50):
# for any unit that is already in pulp, save it into the repo
for unit_dict in self.content_query_manager.get_multiple_units_by_keys_dicts(
self.unit_type, page, self.unit_key_fields):
unit = self._dict_to_unit(unit_dict)
self.get_conduit().save_unit(unit)
units_we_already_had.add(unit)
for unit_key in self.parent.available_units:
# build a temp Unit instance just to use its comparison feature
unit = Unit(self.unit_type, unit_key, {}, '')
if unit not in units_we_already_had:
self.units_to_download.append(unit_key)
def get_requirements(self, units):
"""
For an iterable of RPM Units, return a generator of Require() instances that
represent the requirements for those RPMs.
:param units: iterable of RPMs for which a query should be performed to
retrieve their Requires entries.
:type units: iterable of pulp.plugins.model.Unit
:return: generator of Require() instances
:rtype: generator
"""
for segment in paginate(units):
search_dicts = [unit.unit_key for unit in segment]
filters = {'$or': search_dicts}
fields = list(models.RPM.UNIT_KEY_NAMES)
fields.extend(['requires', 'id'])
criteria = UnitAssociationCriteria(type_ids=[models.RPM.TYPE],
unit_filters=filters,
unit_fields=fields)
for result in self.search_method(criteria):
for require in result.metadata.get('requires', []):
yield Requirement(**require)
def queue_regenerate_applicability_for_repos(repo_criteria):
"""
Queue a group of tasks to generate and save applicability data affected by given updated
repositories.
:param repo_criteria: The repo selection criteria
:type repo_criteria: dict
"""
repo_criteria = Criteria.from_dict(repo_criteria)
# Process repo criteria
repo_criteria.fields = ['id']
repo_ids = [r.repo_id for r in model.Repository.objects.find_by_criteria(repo_criteria)]
task_group_id = uuid4()
for repo_id in repo_ids:
profile_hashes = RepoProfileApplicability.get_collection().find(
{'repo_id': repo_id}, {'profile_hash': 1})
for batch in paginate(profile_hashes, 10):
batch_regenerate_applicability_task.apply_async((repo_id, batch),
**{'group_id': task_group_id})
return task_group_id
def get_multiple_units_by_keys_dicts(self, content_type, unit_keys_dicts, model_fields=None):
"""
Look up multiple content units in the collection for the given content
type collection that match the list of keys dictionaries.
:param content_type: unique id of content collection
:type content_type: str
:param unit_keys_dicts: list of dictionaries whose key, value pairs can
uniquely identify a content unit
:type unit_keys_dicts: list of dict's
:param model_fields: fields of each content unit to report,
None means all fields
:type model_fields: None or list of str's
:return: tuple of content units found in the content type collection
that match the given unit keys dictionaries
:rtype: (possibly empty) tuple of dict's
:raises ValueError: if any of the keys dictionaries are invalid
"""
collection = content_types_db.type_units_collection(content_type)
for segment in paginate(unit_keys_dicts, page_size=50):
spec = _build_multi_keys_spec(content_type, segment)
cursor = collection.find(spec, fields=model_fields)
for unit_dict in cursor:
yield unit_dict
def get_all_existing_units(search_dicts, unit_fields, unit_type, search_method):
"""
Get all existing units on the server which match given search_dicts using
given search_method.
:param search_dicts: unit keys generator
:type search_dicts: iterator of unit keys
:param unit_fields: unit fields to be requested to the search_method
:type unit_fields: list or tuple
:param unit_type: unit type
:type unit_type: basestring
:param search_method: search method to be used to search for non-repo-specific units
:type search_method: a search method accepting a unit type and
pulp.server.db.criteria.Criteria as parameters
:return: generator of Units found using the search_method
:rtype: iterator of pulp.plugins.model.Unit
"""
# Instead of separate query for each unit, we are using paginate to query
# for a lot of units at once.
for segment in paginate(search_dicts):
unit_filters = {'$or': list(segment)}
criteria = Criteria(filters=unit_filters, fields=unit_fields)
for result in search_method(unit_type, criteria):
yield result
def associate(source_repo, dest_repo, import_conduit, config, units=None):
"""
This is the primary method to call when a copy operation is desired. This
gets called directly by the Importer
Certain variables are set to "None" as the method progresses so that they
may be garbage collected.
:param source_repo: source repo
:type source_repo: pulp.server.db.model.Repository
:param dest_repo: destination repo
:type dest_repo: pulp.server.db.model.Repository
:param import_conduit: import conduit passed to the Importer
:type import_conduit: pulp.plugins.conduits.unit_import.ImportUnitConduit
:param config: config object for the distributor
:type config: pulp.plugins.config.PluginCallConfiguration
:param units: iterable of ContentUnit objects to copy
:type units: iterable
:return: List of associated units.
"""
if units is None:
# this might use a lot of RAM since RPMs tend to have lots of metadata
# TODO: so we should probably do something about that
units = repo_controller.find_repo_content_units(
source_repo, yield_content_unit=True)
# get config items that we care about
recursive = config.get(constants.CONFIG_RECURSIVE)
if recursive is None:
recursive = False
associated_units = set(
[_associate_unit(dest_repo, unit) for unit in units])
# allow garbage collection
units = None
associated_units |= copy_rpms(
(unit for unit in associated_units if isinstance(unit, models.RPM)),
source_repo, dest_repo, import_conduit, recursive)
# return here if we shouldn't get child units
if not recursive:
return list(associated_units)
group_ids, rpm_names, rpm_search_dicts = identify_children_to_copy(
associated_units)
# ------ get group children of the categories ------
for page in paginate(group_ids):
group_units = models.PackageGroup.objects.filter(
repo_id=source_repo.repo_id, package_group_id__in=page)
if group_units.count() > 0:
associated_units |= set(
associate(source_repo, dest_repo, import_conduit, config,
group_units))
# ------ get RPM children of errata ------
wanted_rpms = get_rpms_to_copy_by_key(rpm_search_dicts, import_conduit,
source_repo)
rpm_search_dicts = None
rpms_to_copy = filter_available_rpms(wanted_rpms, import_conduit,
source_repo)
associated_units |= copy_rpms(rpms_to_copy, source_repo, dest_repo,
import_conduit, recursive)
rpms_to_copy = None
# ------ get RPM children of groups ------
names_to_copy = get_rpms_to_copy_by_name(rpm_names, import_conduit,
dest_repo)
associated_units |= copy_rpms_by_name(names_to_copy, source_repo,
dest_repo, import_conduit, recursive)
return list(associated_units)
def associate(source_repo, dest_repo, import_conduit, config, units=None):
"""
This is the primary method to call when a copy operation is desired. This
gets called directly by the Importer
Certain variables are set to "None" as the method progresses so that they
may be garbage collected.
:param source_repo: source repo
:type source_repo: pulp.server.db.model.Repository
:param dest_repo: destination repo
:type dest_repo: pulp.server.db.model.Repository
:param import_conduit: import conduit passed to the Importer
:type import_conduit: pulp.plugins.conduits.unit_import.ImportUnitConduit
:param config: config object for the distributor
:type config: pulp.plugins.config.PluginCallConfiguration
:param units: iterable of ContentUnit objects to copy
:type units: iterable
:return: List of associated units.
"""
if units is None:
# this might use a lot of RAM since RPMs tend to have lots of metadata
# TODO: so we should probably do something about that
units = repo_controller.find_repo_content_units(source_repo, yield_content_unit=True)
# get config items that we care about
recursive = config.get(constants.CONFIG_RECURSIVE)
if recursive is None:
recursive = False
associated_units = set([_associate_unit(dest_repo, unit) for unit in units])
# allow garbage collection
units = None
associated_units |= copy_rpms(
(unit for unit in associated_units if isinstance(unit, models.RPM)),
source_repo, dest_repo, import_conduit, recursive)
# return here if we shouldn't get child units
if not recursive:
return list(associated_units)
group_ids, rpm_names, rpm_search_dicts = identify_children_to_copy(associated_units)
# ------ get group children of the categories ------
for page in paginate(group_ids):
group_units = models.PackageGroup.objects.filter(repo_id=source_repo.repo_id,
package_group_id__in=page)
if group_units.count() > 0:
associated_units |= set(
associate(source_repo, dest_repo, import_conduit, config, group_units))
# ------ get RPM children of errata ------
wanted_rpms = get_rpms_to_copy_by_key(rpm_search_dicts, import_conduit, source_repo)
rpm_search_dicts = None
rpms_to_copy = filter_available_rpms(wanted_rpms, import_conduit, source_repo)
associated_units |= copy_rpms(rpms_to_copy, source_repo, dest_repo, import_conduit, recursive)
rpms_to_copy = None
# ------ get RPM children of groups ------
names_to_copy = get_rpms_to_copy_by_name(rpm_names, import_conduit, dest_repo)
associated_units |= copy_rpms_by_name(names_to_copy, source_repo, dest_repo,
import_conduit, recursive)
return list(associated_units)
