def _delete_worker(name, normal_shutdown=False):
"""
Delete the Worker with _id name from the database, cancel any associated tasks and reservations
If the worker shutdown normally, no message is logged, otherwise an error level message is
logged. Default is to assume the worker did not shut down normally.
Any resource reservations associated with this worker are cleaned up by this function.
Any tasks associated with this worker are explicitly canceled.
:param name: The name of the worker you wish to delete.
:type name: basestring
:param normal_shutdown: True if the worker shutdown normally, False otherwise. Defaults to
False.
:type normal_shutdown: bool
"""
if normal_shutdown is False:
msg = _('The worker named %(name)s is missing. Canceling the tasks in its queue.')
msg = msg % {'name': name}
_logger.error(msg)
# Delete the worker document
Worker.objects(name=name).delete()
# Delete all reserved_resource documents for the worker
ReservedResource.objects(worker_name=name).delete()
# Cancel all of the tasks that were assigned to this worker's queue
for task_status in TaskStatus.objects(worker_name=name,
state__in=constants.CALL_INCOMPLETE_STATES):
cancel(task_status['task_id'])
# Delete working directory
common_utils.delete_worker_working_directory(name)
def _delete_worker(name, normal_shutdown=False):
"""
Delete the Worker with _id name from the database, cancel any associated tasks and reservations
If the worker shutdown normally, no message is logged, otherwise an error level message is
logged. Default is to assume the worker did not shut down normally.
Any resource reservations associated with this worker are cleaned up by this function.
Any tasks associated with this worker are explicitly canceled.
:param name: The name of the worker you wish to delete.
:type name: basestring
:param normal_shutdown: True if the worker shutdown normally, False otherwise. Defaults to
False.
:type normal_shutdown: bool
"""
if normal_shutdown is False:
msg = _(
'The worker named %(name)s is missing. Canceling the tasks in its queue.'
)
msg = msg % {'name': name}
_logger.error(msg)
# Delete the worker document
Worker.objects(name=name).delete()
# Delete all reserved_resource documents for the worker
ReservedResource.objects(worker_name=name).delete()
# Cancel all of the tasks that were assigned to this worker's queue
for task_status in TaskStatus.objects(
worker_name=name, state__in=constants.CALL_INCOMPLETE_STATES):
cancel(task_status['task_id'])
def _release_resource(task_id):
"""
Do not queue this task yourself. It will be used automatically when your task is dispatched by
the _queue_reserved_task task.
When a resource-reserving task is complete, this method releases the resource by removing the
ReservedResource object by UUID.
:param task_id: The UUID of the task that requested the reservation
:type task_id: basestring
"""
running_task_qs = TaskStatus.objects.filter(
task_id=task_id, state=constants.CALL_RUNNING_STATE)
for running_task in running_task_qs:
new_task = Task()
msg = _(
'The task status %(task_id)s exited immediately for some reason. Marking as '
'errored. Check the logs for more details')
runtime_exception = RuntimeError(msg % {'task_id': task_id})
class MyEinfo(object):
traceback = None
new_task.on_failure(runtime_exception, task_id, (), {}, MyEinfo)
ReservedResource.objects(task_id=task_id).delete()
def _release_resource(task_id):
"""
Do not queue this task yourself. It will be used automatically when your task is dispatched by
the _queue_reserved_task task.
When a resource-reserving task is complete, this method releases the resource by removing the
ReservedResource object by UUID.
:param task_id: The UUID of the task that requested the reservation
:type task_id: basestring
"""
ReservedResource.objects(task_id=task_id).delete()
def _release_resource(task_id):
"""
Do not queue this task yourself. It will be used automatically when your task is dispatched by
the _queue_reserved_task task.
When a resource-reserving task is complete, this method releases the resource by removing the
ReservedResource object by UUID.
:param task_id: The UUID of the task that requested the reservation
:type task_id: basestring
"""
ReservedResource.objects(task_id=task_id).delete()
def _delete_worker(name, normal_shutdown=False):
"""
Delete the Worker with _id name from the database, cancel any associated tasks and reservations
If the worker shutdown normally, no message is logged, otherwise an error level message is
logged. Default is to assume the worker did not shut down normally.
Any resource reservations associated with this worker are cleaned up by this function.
Any tasks associated with this worker are explicitly canceled.
:param name: The name of the worker you wish to delete.
:type name: basestring
:param normal_shutdown: True if the worker shutdown normally, False otherwise. Defaults to
False.
:type normal_shutdown: bool
"""
if normal_shutdown is False:
msg = _(
'The worker named %(name)s is missing. Canceling the tasks in its queue.'
)
msg = msg % {'name': name}
_logger.error(msg)
else:
msg = _("Cleaning up shutdown worker '%s'.") % name
_logger.info(msg)
# Delete the worker document
Worker.objects(name=name).delete()
# Delete all reserved_resource documents for the worker
ReservedResource.objects(worker_name=name).delete()
# If the worker is a resource manager, we also need to delete the associated lock
if name.startswith(RESOURCE_MANAGER_WORKER_NAME):
ResourceManagerLock.objects(name=name).delete()
# If the worker is a scheduler, we also need to delete the associated lock
if name.startswith(SCHEDULER_WORKER_NAME):
CeleryBeatLock.objects(name=name).delete()
# Cancel all of the tasks that were assigned to this worker's queue
for task_status in TaskStatus.objects(
worker_name=name, state__in=constants.CALL_INCOMPLETE_STATES):
cancel(task_status['task_id'], revoke_task=False)
def _delete_worker(name, normal_shutdown=False):
"""
Delete the Worker with _id name from the database, cancel any associated tasks and reservations
If the worker shutdown normally, no message is logged, otherwise an error level message is
logged. Default is to assume the worker did not shut down normally.
Any resource reservations associated with this worker are cleaned up by this function.
Any tasks associated with this worker are explicitly canceled.
:param name: The name of the worker you wish to delete.
:type name: basestring
:param normal_shutdown: True if the worker shutdown normally, False otherwise. Defaults to
False.
:type normal_shutdown: bool
"""
if normal_shutdown is False:
msg = _('The worker named %(name)s is missing. Canceling the tasks in its queue.')
msg = msg % {'name': name}
_logger.error(msg)
else:
msg = _("Cleaning up shutdown worker '%s'.") % name
_logger.info(msg)
# Delete the worker document
Worker.objects(name=name).delete()
# Delete all reserved_resource documents for the worker
ReservedResource.objects(worker_name=name).delete()
# If the worker is a resource manager, we also need to delete the associated lock
if name.startswith(RESOURCE_MANAGER_WORKER_NAME):
ResourceManagerLock.objects(name=name).delete()
# If the worker is a scheduler, we also need to delete the associated lock
if name.startswith(SCHEDULER_WORKER_NAME):
CeleryBeatLock.objects(name=name).delete()
# Cancel all of the tasks that were assigned to this worker's queue
for task_status in TaskStatus.objects(worker_name=name,
state__in=constants.CALL_INCOMPLETE_STATES):
cancel(task_status['task_id'], revoke_task=False)
def _release_resource(task_id):
"""
Do not queue this task yourself. It will be used automatically when your task is dispatched by
the _queue_reserved_task task.
When a resource-reserving task is complete, this method releases the resource by removing the
ReservedResource object by UUID.
:param task_id: The UUID of the task that requested the reservation
:type task_id: basestring
"""
running_task_qs = TaskStatus.objects.filter(task_id=task_id, state=constants.CALL_RUNNING_STATE)
for running_task in running_task_qs:
new_task = Task()
exception = PulpCodedException(error_codes.PLP0049, task_id=task_id)
class MyEinfo(object):
traceback = None
new_task.on_failure(exception, task_id, (), {}, MyEinfo)
ReservedResource.objects(task_id=task_id).delete()
def _release_resource(task_id):
"""
Do not queue this task yourself. It will be used automatically when your task is dispatched by
the _queue_reserved_task task.
When a resource-reserving task is complete, this method releases the resource by removing the
ReservedResource object by UUID.
:param task_id: The UUID of the task that requested the reservation
:type task_id: basestring
"""
running_task_qs = TaskStatus.objects.filter(task_id=task_id, state=constants.CALL_RUNNING_STATE)
for running_task in running_task_qs:
new_task = Task()
exception = PulpCodedException(error_codes.PLP0049, task_id=task_id)
class MyEinfo(object):
traceback = None
new_task.on_failure(exception, task_id, (), {}, MyEinfo)
ReservedResource.objects(task_id=task_id).delete()
def _release_resource(task_id):
"""
Do not queue this task yourself. It will be used automatically when your task is dispatched by
the _queue_reserved_task task.
When a resource-reserving task is complete, this method releases the resource by removing the
ReservedResource object by UUID.
:param task_id: The UUID of the task that requested the reservation
:type task_id: basestring
"""
running_task_qs = TaskStatus.objects.filter(task_id=task_id, state=constants.CALL_RUNNING_STATE)
for running_task in running_task_qs:
new_task = Task()
msg = _('The task status %(task_id)s exited immediately for some reason. Marking as '
'errored. Check the logs for more details')
runtime_exception = RuntimeError(msg % {'task_id': task_id})
class MyEinfo(object):
traceback = None
new_task.on_failure(runtime_exception, task_id, (), {}, MyEinfo)
ReservedResource.objects(task_id=task_id).delete()
def get_worker_for_reservation(resource_id):
"""
Return the Worker instance that is associated with a reservation of type resource_id. If
there are no workers with that reservation_id type a pulp.server.exceptions.NoWorkers
exception is raised.
:param resource_id: The name of the resource you wish to reserve for your task.
:raises NoWorkers: If all workers have reserved_resource entries associated with them.
:type resource_id: basestring
:returns: The Worker instance that has a reserved_resource entry of type
`resource_id` associated with it.
:rtype: pulp.server.db.model.resources.Worker
"""
reservation = ReservedResource.objects(resource_id=resource_id).first()
if reservation:
return Worker.objects(name=reservation['worker_name']).first()
else:
raise NoWorkers()
def get_worker_for_reservation(resource_id):
"""
Return the Worker instance that is associated with a reservation of type resource_id. If
there are no workers with that reservation_id type a pulp.server.exceptions.NoWorkers
exception is raised.
:param resource_id: The name of the resource you wish to reserve for your task.
:raises NoWorkers: If all workers have reserved_resource entries associated with them.
:type resource_id: basestring
:returns: The Worker instance that has a reserved_resource entry of type
`resource_id` associated with it.
:rtype: pulp.server.db.model.resources.Worker
"""
reservation = ReservedResource.objects(resource_id=resource_id).first()
if reservation:
return Worker.objects(name=reservation['worker_name']).first()
else:
raise NoWorkers()
def get_worker_for_reservation_list(resources):
"""
Return the Worker instance that is associated with the reservations described by the 'resources'
list. This will be either an existing Worker that is dealing with at least one of the specified
resources, or an available idle Worker. We sleep and retry the request until it can be
fulfilled.
:param resources: A list of the names of the resources you wish to reserve for your task.
:type resources: list
:returns: The Worker instance that has a reserved_resource entry associated with it
for each resource in 'resources'
:rtype: pulp.server.db.model.resources.Worker
"""
_logger.debug('get_worker_for_reservation_list [%s]' % resources)
# We leave this loop once we find a Worker to return - otherwise, sleep and try again
while True:
reservation_workers = set([
reservation['worker_name']
for reservation in ReservedResource.objects(
resource_id__in=resources)
])
_logger.debug('...num-RR is %d' % len(reservation_workers))
if len(reservation_workers
) == 1: # Exactly one worker holds any of the desired resources
_logger.debug('...one-holds')
return Worker.objects(name=list(reservation_workers)[0]).first()
elif len(reservation_workers
) == 0: # No worker holds any of the desired resources
_logger.debug('...zero-holds')
try:
worker = _get_unreserved_worker()
return worker
except NoWorkers:
_logger.debug('...unresolved NoWorkers - WAIT')
pass
else:
_logger.debug('...multiple-holds - WAIT')
time.sleep(0.25)