def test_empty_list(self):
ret = utils.paginate([], 3)
self.assertTrue(inspect.isgenerator(ret))
pieces = list(ret)
self.assertEqual(pieces, [])
def test_generator_one_page(self):
iterable = (x for x in range(10))
ret = utils.paginate(iterable, 100)
self.assertTrue(inspect.isgenerator(ret))
pieces = list(ret)
self.assertEqual(pieces, [tuple(range(10))])
def test_generator(self):
iterable = (x for x in range(10))
ret = utils.paginate(iterable, 3)
self.assertTrue(inspect.isgenerator(ret))
pieces = list(ret)
self.assertEqual(pieces, [(0,1,2), (3,4,5), (6,7,8), (9,)])
def get_existing_units(search_dicts, unit_fields, unit_type, search_method):
"""
:param search_dicts:
:param unit_fields:
:param unit_type:
:param search_method:
:return: generator of Units
"""
for segment in paginate(search_dicts):
unit_filters = {'$or': list(segment)}
criteria = UnitAssociationCriteria([unit_type], unit_filters=unit_filters,
unit_fields=unit_fields, association_fields=[])
for result in search_method(criteria):
yield result
def get_requirements(self, units):
"""
For an iterable of RPM Units, return a generator of Require() instances that
represent the requirements for those RPMs.
:param units: iterable of RPMs for which a query should be performed to
retrieve their Requires entries.
:type units: iterable of pulp.plugins.model.Unit
:return: generator of Require() instances
:rtype: generator
"""
for segment in paginate(units):
search_dicts = [unit.unit_key for unit in segment]
filters = {'$or': search_dicts}
fields = list(models.RPM.UNIT_KEY_NAMES)
fields.extend(['requires', 'id'])
criteria = UnitAssociationCriteria(type_ids=[models.RPM.TYPE], unit_filters=filters, unit_fields=fields)
for result in self.search_method(criteria):
for require in result.metadata.get('requires', []):
yield Requirement(**require)
def associate_already_downloaded_units(unit_type, units_to_download, sync_conduit):
"""
Given a generator of Package instances, this method checks if a package with given
type and unit key already exists in Pulp. This means that the package is already
downloaded and just needs to be associated with the given repository.
After importing already downloaded units to the repository, it returns a generator
of the remaining Package instances which need to be downloaded.
:param unit_type: unit type
:type unit_type: basestring
:param units_to_download: generator of pulp_rpm.plugins.db.models.Package instances that
should be considered for download
:type units_to_download: generator
:param sync_conduit: sync conduit
:type sync_conduit: pulp.plugins.conduits.repo_sync.RepoSyncConduit
:return: generator of pulp_rpm.plugins.db.models.Package instances that
need to be downloaded
:rtype: generator
"""
model = models.TYPE_MAP[unit_type]
unit_fields = model.UNIT_KEY_NAMES
# Instead of separate query for each unit, we are using paginate to query
# for a lot of units at once.
for units_page in paginate(units_to_download):
unit_filters = {'$or': [unit.unit_key for unit in units_page]}
criteria = Criteria(filters=unit_filters, fields=unit_fields)
result = sync_conduit.search_all_units(unit_type, criteria)
result_unit_keys = [unit.unit_key for unit in result]
for unit in units_page:
if unit.unit_key in result_unit_keys:
# Since unit is already downloaded, call respective sync_conduit calls to import
# the unit in given repository.
downloaded_unit = sync_conduit.init_unit(unit_type, unit.unit_key,
unit.metadata, unit.relative_path)
sync_conduit.save_unit(downloaded_unit)
else:
yield unit
def get_requirements(units, search_method):
"""
For an iterable of RPMs, return a generator of Require() instances that
represent the requirements for those RPMs.
:param units: iterable of RPMs for which a query should be performed to
retrieve their Requires entries.
:type units: iterable of pulp_rpm.common.models.RPM.NAMEDTUPLE
:param search_method: method that takes a UnitAssociationCriteria and
performs a search within a repository. Usually this
will be a method on a conduit such as "conduit.get_units"
:type search_method: function
:return: generator of Require() instances
"""
for segment in paginate(units):
search_dicts = [unit.unit_key for unit in segment]
filters = {'$or': search_dicts}
fields = list(models.RPM.UNIT_KEY_NAMES)
fields.extend(['requires', 'id'])
criteria = UnitAssociationCriteria(type_ids=[models.RPM.TYPE], unit_filters=filters, unit_fields=fields)
for result in search_method(criteria):
for require in result.metadata.get('requires', []):
yield Requirement(**require)
