def get_least_busy_worker():
"""
Return the Worker instance with the lowest num_reservations. This function makes no
guarantees about which Worker gets returned in the event of a tie.
:returns: The Worker instance with the lowest num_reservations.
:rtype: pulp.server.db.model.resources.Worker
"""
# Build a mapping of queue names to number of reservations against them
workers = filter_workers(criteria.Criteria())
reservation_map = {}
for worker in workers:
reservation_map[worker.queue_name] = {
'num_reservations': 0,
'worker': worker
}
if not reservation_map:
raise exceptions.NoWorkers()
for reserved_resource in resources.ReservedResource.get_collection().find(
):
if reserved_resource['assigned_queue'] in reservation_map:
reservation_map[reserved_resource['assigned_queue']]['num_reservations'] += \
reserved_resource['num_reservations']
# Now let's flatten the reservation map into a list of 3-tuples, where the first element
# is the num_reservations on the queue, the second element is a pseudorandom tie-breaker, and
# the third is the worker. This will be easy to sort and get the least busy worker
results = [(v['num_reservations'], random.random(), v['worker'])
for k, v in reservation_map.items()]
results.sort()
# Since results is sorted by least busy worker, we can just return the first one
return results[0][2]
def get_unreserved_worker():
"""
Return the Worker instance that has no reserved_resource entries associated with it. If there
are no unreserved workers a pulp.server.exceptions.NoWorkers exception is raised.
:raises NoWorkers: If all workers have reserved_resource entries associated with them.
:returns: The Worker instance that has no reserved_resource entries associated with it.
:rtype: pulp.server.db.model.resources.Worker
"""
# Build a mapping of queue names to Worker objects
workers_dict = dict((worker['name'], worker) for worker in filter_workers(criteria.Criteria()))
worker_names = [name for name in workers_dict.keys()]
reserved_names = [r['worker_name'] for r in resources.ReservedResource.get_collection().find()]
# Find an unreserved worker using set differences of the names, and filter
# out workers that should not be assigned work.
# NB: this is a little messy but set comprehensions are in python 2.7+
unreserved_workers = set(filter(_is_worker, worker_names)) - set(reserved_names)
try:
return workers_dict[unreserved_workers.pop()]
except KeyError:
# All workers are reserved
raise NoWorkers()
def get_worker_for_reservation(resource_id):
"""
Return the Worker instance that is associated with a reservation of type resource_id. If
there are no workers with that reservation_id type a pulp.server.exceptions.NoWorkers exception
is raised.
:param resource_id: The name of the resource you wish to reserve for your task.
:raises NoWorkers: If all workers have reserved_resource entries associated with them.
:type resource_id: basestring
:returns: The Worker instance that has a reserved_resource entry of type
`resource_id` associated with it.
:rtype: pulp.server.db.model.resources.Worker
"""
reservation = resources.ReservedResource.get_collection().find_one(
{'resource_id': resource_id})
if reservation:
find_worker_by_name = criteria.Criteria(
{'_id': reservation['worker_name']})
worker_bson = resources.Worker.get_collection().query(
find_worker_by_name)[0]
return resources.Worker.from_bson(worker_bson)
else:
raise NoWorkers()
def _process_manifest(manifest, conduit):
"""
This method reads the given package manifest to determine which versions of the package are
available at the feed repo. It then compares these versions to the versions that are in the
repository that is being synchronized, as well as to the versions that are available in
Pulp. For packages that are in Pulp but are not in the repository, it will create the
association without downloading the packages. For package versions which are not available
in Pulp, it will return a list of dictionaries describing the missing packages so that the
DownloadPackagesStep can retrieve them later. Each dictionary has the following keys: name,
version, url, and checksum. The checksum is given in md5, as per the upstream PyPI feed.
:param manifest: A package manifest in JSON format, describing the versions of a package
that are available for download.
:type manifest: basestring
:param conduit: The sync conduit. This is used to query Pulp for available packages.
:type conduit: pulp.plugins.conduits.repo_sync.RepoSyncConduit
:return: A list of dictionaries, describing the packages that need to be downloaded.
:rtype: list
"""
manifest = json.loads(manifest)
name = manifest['info']['name']
all_versions = set(manifest['releases'].keys())
# Find the versions that we have in Pulp
search = criteria.Criteria(filters={'name': name}, fields=['name', 'version'])
versions_in_pulp = set(
[u.unit_key['version'] for u in
conduit.search_all_units(constants.PACKAGE_TYPE_ID, criteria=search)])
# Find the versions that we have in the repo already
search = criteria.UnitAssociationCriteria(unit_filters={'name': name},
unit_fields=['name', 'version'])
versions_in_repo = set([u.unit_key['version'] for u in
conduit.get_units(criteria=search)])
# These versions are in Pulp, but are not associated with this repository. Associate them.
versions_to_associate = list(versions_in_pulp - versions_in_repo)
if versions_to_associate:
conduit.associate_existing(
constants.PACKAGE_TYPE_ID,
[{'name': name, 'version': v} for v in versions_to_associate])
# We don't have these versions in Pulp yet. Let's download them!
versions_to_dl = all_versions - versions_in_pulp
packages_to_dl = []
for v in versions_to_dl:
for package in manifest['releases'][v]:
if package['packagetype'] == 'sdist' and package['filename'][-4:] != '.zip':
packages_to_dl.append({'name': name, 'version': v, 'url': package['url'],
'checksum': package['md5_digest']})
return packages_to_dl
def test_full(self):
c = criteria.Criteria(filters={'name': {
'$in': ['a', 'b']
}},
sort=(('name', 'ascending'), ),
limit=10,
skip=10,
fields=('name', 'id'))
ret = c.as_dict()
self.assertTrue(isinstance(ret['filters'], dict))
self.assertEqual(ret['limit'], 10)
self.assertEqual(ret['skip'], 10)
self.assertEqual(ret['fields'], c.fields)
self.assertEqual(set(ret.keys()), FIELDS)
def test_from_dict_accepts_as_dict_as_input(self):
"""
Verify that from_dict() accepts the output of as_dict() as input.
"""
filters = {'some': 'filters'}
sort = ['sort_item']
limit = 42
skip = 64
fields = ['a_field']
criteria_1 = criteria.Criteria(filters, sort, limit, skip, fields)
criteria_2 = criteria.Criteria.from_dict(criteria_1.as_dict())
self.assertTrue(isinstance(criteria_2, criteria.Criteria))
self.assertEqual(criteria_2.filters, criteria_1.filters)
self.assertEqual(criteria_2.sort, criteria_1.sort)
self.assertEqual(criteria_2.limit, criteria_1.limit)
self.assertEqual(criteria_2.skip, criteria_1.skip)
self.assertEqual(criteria_2.fields, criteria_1.fields)
def test_empty(self):
c = criteria.Criteria()
ret = c.as_dict()
self.assertTrue(isinstance(ret, dict))
for field in FIELDS:
self.assertTrue(ret[field] is None)
