def check_and_cancel_missing_tasks():
"""Cancel any unexecuted Tasks which are no longer in the RQ registry.
In some situations such as a restart of Redis, Jobs can be dropped from the Redis
queues and "forgotten". Therefore the Task will never be marked completed in Pulp
and are never cleaned up. This results in stray resource locks that cause workers
to deadlock.
We go through all of the tasks which are in an incomplete state and check that RQ
still has a record of the job. If not, we cancel it.
"""
redis_conn = connection.get_redis_connection()
assigned_and_unfinished_tasks = Task.objects.filter(
state__in=TASK_INCOMPLETE_STATES,
worker__in=Worker.objects.online_workers())
for task in assigned_and_unfinished_tasks:
try:
Job.fetch(str(task.pk), connection=redis_conn)
except NoSuchJobError:
cancel(task.pk)
# Also go through all of the tasks that were still queued up on the resource manager
for task in Task.objects.filter(worker__isnull=True):
try:
Job.fetch(str(task._resource_job_id), connection=redis_conn)
except NoSuchJobError:
cancel(task.pk)
def enqueue_with_reservation(func,
resources,
args=None,
kwargs=None,
options=None):
"""
Enqueue a message to Pulp workers with a reservation.
This method provides normal enqueue functionality, while also requesting necessary locks for
serialized urls. No two tasks that claim the same resource can execute concurrently. It
accepts resources which it transforms into a list of urls (one for each resource).
This does not dispatch the task directly, but instead promises to dispatch it later by
encapsulating the desired task through a call to a :func:`_queue_reserved_task` task. See
the docblock on :func:`_queue_reserved_task` for more information on this.
This method creates a :class:`pulpcore.app.models.Task` object. Pulp expects to poll on a
task just after calling this method, so a Task entry needs to exist for it
before it returns.
Args:
func (callable): The function to be run by RQ when the necessary locks are acquired.
resources (list): A list of resources to reserve guaranteeing that only one task reserves
these resources. Each resource can be either a (str) resource URL or a
(django.models.Model) resource instance.
args (tuple): The positional arguments to pass on to the task.
kwargs (dict): The keyword arguments to pass on to the task.
options (dict): The options to be passed on to the task.
Returns (rq.job.job): An RQ Job instance as returned by RQ's enqueue function
Raises:
ValueError: When `resources` is an unsupported type.
"""
if not args:
args = tuple()
if not kwargs:
kwargs = dict()
if not options:
options = dict()
def as_url(r):
if isinstance(r, str):
return r
if isinstance(r, Model):
return util.get_url(r)
raise ValueError(_('Must be (str|Model)'))
resources = {as_url(r) for r in resources}
inner_task_id = str(uuid.uuid4())
redis_conn = connection.get_redis_connection()
parent_kwarg = {}
Task.objects.create(pk=inner_task_id,
state=TASK_STATES.WAITING,
name=f'{func.__module__}.{func.__name__}',
**parent_kwarg)
q = Queue('resource-manager', connection=redis_conn)
task_args = (func, inner_task_id, list(resources), args, kwargs, options)
q.enqueue(_queue_reserved_task, args=task_args, job_timeout=TASK_TIMEOUT)
return Job(id=inner_task_id, connection=redis_conn)
def cancel(task_id):
"""
Cancel the task that is represented by the given task_id.
This method cancels only the task with given task_id, not the spawned tasks. This also updates
task's state to 'canceled'.
Args:
task_id (str): The ID of the task you wish to cancel
Raises:
MissingResource: if a task with given task_id does not exist
"""
try:
task_status = Task.objects.get(pk=task_id)
except Task.DoesNotExist:
raise MissingResource(task=task_id)
if task_status.state in TASK_FINAL_STATES:
# If the task is already done, just stop
msg = _("Task [{task_id}] already in a final state: {state}")
_logger.debug(msg.format(task_id=task_id, state=task_status.state))
return task_status
_logger.info(_("Canceling task: {id}").format(id=task_id))
redis_conn = connection.get_redis_connection()
job = Job(id=str(task_status.pk), connection=redis_conn)
resource_job = Job(id=str(task_status._resource_job_id),
connection=redis_conn)
task_status.state = TASK_STATES.CANCELED
task_status.save()
try:
send_stop_job_command(redis_conn, job.get_id())
send_stop_job_command(redis_conn, resource_job.get_id())
except (InvalidJobOperation, NoSuchJobError):
# We don't care if the job isn't currently running when we try to cancel
pass
# A hack to ensure that we aren't deleting resources still being used by the workhorse
time.sleep(0.5)
resource_job.delete()
job.delete()
with transaction.atomic():
for report in task_status.progress_reports.all():
if report.state not in TASK_FINAL_STATES:
report.state = TASK_STATES.CANCELED
report.save()
_delete_incomplete_resources(task_status)
task_status.release_resources()
return task_status
def _enqueue_with_reservation(
func, resources, args=None, kwargs=None, options=None, task_group=None
):
if not args:
args = tuple()
if not kwargs:
kwargs = dict()
if not options:
options = dict()
def as_url(r):
if isinstance(r, str):
return r
if isinstance(r, Model):
return util.get_url(r)
raise ValueError(_("Must be (str|Model)"))
resources = {as_url(r) for r in resources}
inner_task_id = str(uuid.uuid4())
resource_task_id = str(uuid.uuid4())
redis_conn = connection.get_redis_connection()
current_job = get_current_job(connection=redis_conn)
parent_kwarg = {}
json.dumps(args, cls=NonJSONWarningEncoder)
json.dumps(kwargs, cls=NonJSONWarningEncoder)
if current_job:
# set the parent task of the spawned task to the current task ID (same as rq Job ID)
parent_kwarg["parent_task"] = Task.objects.get(pk=current_job.id)
with transaction.atomic():
task = Task.objects.create(
pk=inner_task_id,
_resource_job_id=resource_task_id,
state=TASK_STATES.WAITING,
logging_cid=(GuidMiddleware.get_guid() or ""),
task_group=task_group,
name=f"{func.__module__}.{func.__name__}",
**parent_kwarg,
)
for resource in resources:
reservation_record = ReservedResourceRecord.objects.get_or_create(resource=resource)[0]
TaskReservedResourceRecord.objects.create(resource=reservation_record, task=task)
task_args = (func, inner_task_id, list(resources), args, kwargs, options)
try:
q = Queue("resource-manager", connection=redis_conn)
q.enqueue(
_queue_reserved_task,
job_id=resource_task_id,
args=task_args,
job_timeout=TASK_TIMEOUT,
)
except RedisConnectionError as e:
task.set_failed(e, None)
return Job(id=inner_task_id, connection=redis_conn)
def create_profile_db_and_connection():
"""
Create a profile db from this tasks UUID and a sqlite3 connection to that databases.
The database produced has three tables with the following SQL format:
The `stages` table stores info about the pipeline itself and stores 3 fields
* uuid - the uuid of the stage
* name - the name of the stage
* num - the number of the stage starting at 0
The `traffic` table stores 3 fields:
* uuid - the uuid of the stage this queue feeds into
* waiting_time - the amount of time the item is waiting in the queue before it enters the stage.
* service_time - the service time the item spent in the stage.
The `system` table stores 3 fields:
* uuid - The uuid of stage this queue feeds into
* length - The length of items in this queue, measured just before each arrival.
* interarrival_time - The amount of time since the last arrival.
"""
debug_data_dir = "/var/lib/pulp/debug/"
pathlib.Path(debug_data_dir).mkdir(parents=True, exist_ok=True)
redis_conn = connection.get_redis_connection()
current_job = get_current_job(connection=redis_conn)
if current_job:
db_path = debug_data_dir + current_job.id
else:
db_path = debug_data_dir + uuid.uuid4()
import sqlite3
global CONN
CONN = sqlite3.connect(db_path)
c = CONN.cursor()
# Create table
c.execute(
"""CREATE TABLE stages
(uuid varchar(36), name text, num int)"""
)
# Create table
c.execute(
"""CREATE TABLE traffic
(uuid varchar(36), waiting_time real, service_time real)"""
)
# Create table
c.execute(
"""CREATE TABLE system
(uuid varchar(36), length int, interarrival_time real)"""
)
return CONN
def cancel(task_id):
"""
Cancel the task that is represented by the given task_id.
This method cancels only the task with given task_id, not the spawned tasks. This also updates
task's state to 'canceled'.
Args:
task_id (str): The ID of the task you wish to cancel
Raises:
MissingResource: if a task with given task_id does not exist
"""
try:
task_status = Task.objects.get(pk=task_id)
except Task.DoesNotExist:
raise MissingResource(task=task_id)
if task_status.state in TASK_FINAL_STATES:
# If the task is already done, just stop
msg = _("Task [{task_id}] already in a completed state: {state}")
_logger.info(msg.format(task_id=task_id, state=task_status.state))
return task_status
redis_conn = connection.get_redis_connection()
job = Job(id=str(task_status.pk), connection=redis_conn)
resource_job = Job(id=str(task_status._resource_job_id),
connection=redis_conn)
if job.is_started:
redis_conn.sadd(TASKING_CONSTANTS.KILL_KEY, job.get_id())
if resource_job.is_started:
redis_conn.sadd(TASKING_CONSTANTS.KILL_KEY, resource_job.get_id())
resource_job.delete()
job.delete()
# A hack to ensure that we aren't deleting resources still being used by the workhorse
time.sleep(1.5)
with transaction.atomic():
task_status.state = TASK_STATES.CANCELED
for report in task_status.progress_reports.all():
if report.state not in TASK_FINAL_STATES:
report.state = TASK_STATES.CANCELED
report.save()
task_status.save()
_delete_incomplete_resources(task_status)
task_status.release_resources()
_logger.info(_("Task canceled: {id}.").format(id=task_id))
return task_status
def _enqueue_with_reservation(func,
resources,
args=None,
kwargs=None,
options=None,
task_group=None):
if not args:
args = tuple()
if not kwargs:
kwargs = dict()
if not options:
options = dict()
resources = _validate_and_get_resources(resources)
inner_task_id = str(uuid.uuid4())
resource_task_id = str(uuid.uuid4())
args_as_json = json.dumps(args, cls=UUIDEncoder)
kwargs_as_json = json.dumps(kwargs, cls=UUIDEncoder)
redis_conn = connection.get_redis_connection()
current_job = get_current_job(connection=redis_conn)
parent_kwarg = {}
if current_job:
# set the parent task of the spawned task to the current task ID (same as rq Job ID)
parent_kwarg["parent_task"] = Task.objects.get(pk=current_job.id)
with transaction.atomic():
task = Task.objects.create(
pk=inner_task_id,
_resource_job_id=resource_task_id,
state=TASK_STATES.WAITING,
logging_cid=(get_guid() or ""),
task_group=task_group,
name=f"{func.__module__}.{func.__name__}",
args=args_as_json,
kwargs=kwargs_as_json,
reserved_resources_record=resources,
**parent_kwarg,
)
task_args = (func, inner_task_id, resources, args, kwargs, options)
try:
q = Queue("resource-manager", connection=redis_conn)
q.enqueue(
_queue_reserved_task,
job_id=resource_task_id,
args=task_args,
job_timeout=TASK_TIMEOUT,
)
except RedisConnectionError as e:
task.set_failed(e, None)
return Job(id=inner_task_id, connection=redis_conn)
def __init__(self, queues, **kwargs):
kwargs['name'] = kwargs['name'].replace('%h', socket.getfqdn())
kwargs['connection'] = get_redis_connection()
if kwargs['name'].startswith(TASKING_CONSTANTS.WORKER_PREFIX):
queues = [Queue(kwargs['name'], connection=kwargs['connection'])]
if kwargs['name'].startswith(TASKING_CONSTANTS.RESOURCE_MANAGER_WORKER_NAME):
queues = [Queue('resource_manager', connection=kwargs['connection'])]
kwargs['default_worker_ttl'] = TASKING_CONSTANTS.WORKER_TTL
kwargs['job_monitoring_interval'] = TASKING_CONSTANTS.JOB_MONITORING_INTERVAL
return super().__init__(queues, **kwargs)
def _get_redis_conn_status():
"""
Returns True if pulp can connect to Redis
Returns:
bool: True if pulp can connect to Redis. False otherwise.
"""
conn = get_redis_connection()
try:
conn.ping()
except Exception:
_logger.error(_('Connection to Redis failed during status check!'))
return False
else:
return True
def _get_redis_conn_status():
"""
Returns True if pulp can connect to Redis
Returns:
bool: True if pulp can connect to Redis. False otherwise.
"""
conn = get_redis_connection()
try:
conn.ping()
except Exception:
_logger.error(_('Connection to Redis failed during status check!'))
return False
else:
return True
def __init__(self, queues, **kwargs):
kwargs['name'] = kwargs['name'].replace('%h', socket.getfqdn())
kwargs['connection'] = get_redis_connection()
if kwargs['name'].startswith(TASKING_CONSTANTS.WORKER_PREFIX):
queues = [Queue(kwargs['name'], connection=kwargs['connection'])]
if kwargs['name'].startswith(
TASKING_CONSTANTS.RESOURCE_MANAGER_WORKER_NAME):
queues = [
Queue('resource_manager', connection=kwargs['connection'])
]
kwargs['default_worker_ttl'] = TASKING_CONSTANTS.WORKER_TTL
kwargs[
'job_monitoring_interval'] = TASKING_CONSTANTS.JOB_MONITORING_INTERVAL
return super().__init__(queues, **kwargs)
def enqueue_with_reservation(func, resources, args=None, kwargs=None, options=None):
"""
Enqueue a message to Pulp workers with a reservation.
This method provides normal enqueue functionality, while also requesting necessary locks for
serialized urls. No two tasks that claim the same resource can execute concurrently. It
accepts resources which it transforms into a list of urls (one for each resource).
This does not dispatch the task directly, but instead promises to dispatch it later by
encapsulating the desired task through a call to a :func:`_queue_reserved_task` task. See
the docblock on :func:`_queue_reserved_task` for more information on this.
This method creates a :class:`pulpcore.app.models.Task` object. Pulp expects to poll on a
task just after calling this method, so a Task entry needs to exist for it
before it returns.
Args:
func (callable): The function to be run by RQ when the necessary locks are acquired.
resources (list): A list of resources to reserve guaranteeing that only one task
reserves these resources
args (tuple): The positional arguments to pass on to the task.
kwargs (dict): The keyword arguments to pass on to the task.
options (dict): The options to be passed on to the task.
Returns (rq.job.job): An RQ Job instance as returned by RQ's enqueue function
"""
if not args:
args = tuple()
if not kwargs:
kwargs = dict()
if not options:
options = dict()
resources = {util.get_url(resource) for resource in resources}
inner_task_id = str(uuid.uuid4())
Task.objects.create(pk=inner_task_id, state=TASK_STATES.WAITING)
redis_conn = connection.get_redis_connection()
q = Queue('resource_manager', connection=redis_conn)
task_args = (func, inner_task_id, list(resources), args, kwargs, options)
q.enqueue(_queue_reserved_task, args=task_args, timeout=TASK_TIMEOUT)
return Job(id=inner_task_id, connection=redis_conn)
def cancel(task_id):
"""
Cancel the task that is represented by the given task_id.
This method cancels only the task with given task_id, not the spawned tasks. This also updates
task's state to 'canceled'.
:param task_id: The ID of the task you wish to cancel
:type task_id: basestring
:raises MissingResource: if a task with given task_id does not exist
"""
try:
task_status = Task.objects.get(pk=task_id)
except Task.DoesNotExist:
raise MissingResource(task_id)
if task_status.state in TASK_FINAL_STATES:
# If the task is already done, just stop
msg = _('Task [{task_id}] already in a completed state: {state}')
_logger.info(msg.format(task_id=task_id, state=task_status.state))
return
redis_conn = connection.get_redis_connection()
job = Job(id=str(task_id), connection=redis_conn)
if job.is_started:
redis_conn.sadd(TASKING_CONSTANTS.KILL_KEY, job.get_id())
job.delete()
# A hack to ensure that we aren't deleting resources still being used by the workhorse
time.sleep(1.5)
with transaction.atomic():
task_status.state = TASK_STATES.CANCELED
task_status.save()
_delete_incomplete_resources(task_status)
_logger.info(_('Task canceled: {id}.').format(id=task_id))
def __init__(self):
"""Creates synchronous cache instance"""
self.redis = get_redis_connection()
def cancel(task_id):
"""
Cancel the task that is represented by the given task_id.
This method cancels only the task with given task_id, not the spawned tasks. This also updates
task's state to either 'canceled' or 'canceling'.
Args:
task_id (str): The ID of the task you wish to cancel
Raises:
MissingResource: if a task with given task_id does not exist
"""
try:
task_status = Task.objects.get(pk=task_id)
except Task.DoesNotExist:
raise MissingResource(task=task_id)
if task_status.state in TASK_FINAL_STATES:
# If the task is already done, just stop
msg = _("Task [{task_id}] already in a final state: {state}")
_logger.debug(msg.format(task_id=task_id, state=task_status.state))
return task_status
_logger.info(_("Canceling task: {id}").format(id=task_id))
if settings.USE_NEW_WORKER_TYPE:
task = task_status
# This is the only valid transition without holding the task lock
rows = Task.objects.filter(pk=task.pk,
state__in=TASK_INCOMPLETE_STATES).update(
state=TASK_STATES.CANCELING)
# Notify the worker that might be running that task and other workers to clean up
with db_connection.cursor() as cursor:
cursor.execute("SELECT pg_notify('pulp_worker_cancel', %s)",
(str(task.pk), ))
cursor.execute("NOTIFY pulp_worker_wakeup")
if rows == 1:
task.refresh_from_db()
return task
redis_conn = connection.get_redis_connection()
job = Job(id=str(task_status.pk), connection=redis_conn)
resource_job = Job(id=str(task_status._resource_job_id),
connection=redis_conn)
task_status.state = TASK_STATES.CANCELED
task_status.save()
resource_job.cancel()
job.cancel()
try:
send_stop_job_command(redis_conn, job.get_id())
send_stop_job_command(redis_conn, resource_job.get_id())
except (InvalidJobOperation, NoSuchJobError):
# We don't care if the job isn't currently running when we try to cancel
pass
# A hack to ensure that we aren't deleting resources still being used by the workhorse
time.sleep(0.5)
with transaction.atomic():
for report in task_status.progress_reports.all():
if report.state not in TASK_FINAL_STATES:
report.state = TASK_STATES.CANCELED
report.save()
_delete_incomplete_resources(task_status)
task_status.release_resources()
return task_status
def enqueue_with_reservation(
func, resources, args=None, kwargs=None, options=None, task_group=None
):
"""
Enqueue a message to Pulp workers with a reservation.
This method provides normal enqueue functionality, while also requesting necessary locks for
serialized urls. No two tasks that claim the same resource can execute concurrently. It
accepts resources which it transforms into a list of urls (one for each resource).
This does not dispatch the task directly, but instead promises to dispatch it later by
encapsulating the desired task through a call to a :func:`_queue_reserved_task` task. See
the docblock on :func:`_queue_reserved_task` for more information on this.
This method creates a :class:`pulpcore.app.models.Task` object. Pulp expects to poll on a
task just after calling this method, so a Task entry needs to exist for it
before it returns.
Args:
func (callable): The function to be run by RQ when the necessary locks are acquired.
resources (list): A list of resources to reserve guaranteeing that only one task reserves
these resources. Each resource can be either a (str) resource URL or a
(django.models.Model) resource instance.
args (tuple): The positional arguments to pass on to the task.
kwargs (dict): The keyword arguments to pass on to the task.
options (dict): The options to be passed on to the task.
task_group (pulpcore.app.models.TaskGroup): A TaskGroup to add the created Task to.
Returns (rq.job.job): An RQ Job instance as returned by RQ's enqueue function
Raises:
ValueError: When `resources` is an unsupported type.
"""
if not args:
args = tuple()
if not kwargs:
kwargs = dict()
if not options:
options = dict()
def as_url(r):
if isinstance(r, str):
return r
if isinstance(r, Model):
return util.get_url(r)
raise ValueError(_("Must be (str|Model)"))
resources = {as_url(r) for r in resources}
inner_task_id = str(uuid.uuid4())
resource_task_id = str(uuid.uuid4())
redis_conn = connection.get_redis_connection()
current_job = get_current_job(connection=redis_conn)
parent_kwarg = {}
if current_job:
# set the parent task of the spawned task to the current task ID (same as rq Job ID)
parent_kwarg["parent_task"] = Task.objects.get(pk=current_job.id)
with transaction.atomic():
task = Task.objects.create(
pk=inner_task_id,
_resource_job_id=resource_task_id,
state=TASK_STATES.WAITING,
logging_cid=(GuidMiddleware.get_guid() or ""),
task_group=task_group,
name=f"{func.__module__}.{func.__name__}",
**parent_kwarg,
)
for resource in resources:
reservation_record = ReservedResourceRecord.objects.get_or_create(resource=resource)[0]
TaskReservedResourceRecord.objects.create(resource=reservation_record, task=task)
task_args = (func, inner_task_id, list(resources), args, kwargs, options)
try:
q = Queue("resource-manager", connection=redis_conn)
q.enqueue(
_queue_reserved_task,
job_id=resource_task_id,
args=task_args,
job_timeout=TASK_TIMEOUT,
)
except RedisConnectionError as e:
task.set_failed(e, None)
return Job(id=inner_task_id, connection=redis_conn)
def _queue_reserved_task(func, inner_task_id, resources, inner_args,
inner_kwargs, options):
"""
A task that encapsulates another task to be dispatched later.
This task being encapsulated is called the "inner" task, and a task name, UUID, and accepts a
list of positional args and keyword args for the inner task. These arguments are named
inner_args and inner_kwargs. inner_args is a list, and inner_kwargs is a dictionary passed to
the inner task as positional and keyword arguments using the * and ** operators.
The inner task is dispatched into a dedicated queue for a worker that is decided at dispatch
time. The logic deciding which queue receives a task is controlled through the
find_worker function.
Args:
func (basestring): The function to be called
inner_task_id (basestring): The UUID to be set on the task being called. By providing
the UUID, the caller can have an asynchronous reference to the inner task
that will be dispatched.
resources (basestring): The urls of the resource you wish to reserve for your task.
The system will ensure that no other tasks that want that same reservation will run
concurrently with yours.
inner_args (tuple): The positional arguments to pass on to the task.
inner_kwargs (dict): The keyword arguments to pass on to the task.
options (dict): For all options accepted by enqueue see the RQ docs
"""
redis_conn = connection.get_redis_connection()
task_status = Task.objects.get(pk=inner_task_id)
task_name = func.__module__ + '.' + func.__name__
while True:
if task_name == "pulpcore.app.tasks.orphan.orphan_cleanup":
if ReservedResource.objects.exists():
# wait until there are no reservations
time.sleep(0.25)
continue
else:
task_status.state = TASK_STATES.RUNNING
task_status.save()
q = Queue('resource_manager',
connection=redis_conn,
async=False)
q.enqueue(func,
args=inner_args,
kwargs=inner_kwargs,
job_id=inner_task_id,
timeout=TASK_TIMEOUT,
**options)
task_status.state = TASK_STATES.COMPLETED
task_status.save()
return
try:
worker = _acquire_worker(resources)
except Worker.DoesNotExist:
# no worker is ready so we need to wait
time.sleep(0.25)
continue
try:
worker.lock_resources(task_status, resources)
except IntegrityError:
# we have a worker but we can't create the reservations so wait
time.sleep(0.25)
else:
# we have a worker with the locks
break
task_status.worker = worker
task_status.save()
try:
q = Queue(worker.name, connection=redis_conn)
q.enqueue(func,
args=inner_args,
kwargs=inner_kwargs,
job_id=inner_task_id,
timeout=TASK_TIMEOUT,
**options)
finally:
q.enqueue(_release_resources, args=(inner_task_id, ))
def _queue_reserved_task(func, inner_task_id, resources, inner_args, inner_kwargs, options):
"""
A task that encapsulates another task to be dispatched later.
This task being encapsulated is called the "inner" task, and a task name, UUID, and accepts a
list of positional args and keyword args for the inner task. These arguments are named
inner_args and inner_kwargs. inner_args is a list, and inner_kwargs is a dictionary passed to
the inner task as positional and keyword arguments using the * and ** operators.
The inner task is dispatched into a dedicated queue for a worker that is decided at dispatch
time. The logic deciding which queue receives a task is controlled through the
find_worker function.
Args:
func (basestring): The function to be called
inner_task_id (basestring): The UUID to be set on the task being called. By providing
the UUID, the caller can have an asynchronous reference to the inner task
that will be dispatched.
resources (basestring): The urls of the resource you wish to reserve for your task.
The system will ensure that no other tasks that want that same reservation will run
concurrently with yours.
inner_args (tuple): The positional arguments to pass on to the task.
inner_kwargs (dict): The keyword arguments to pass on to the task.
options (dict): For all options accepted by enqueue see the RQ docs
"""
redis_conn = connection.get_redis_connection()
task_status = Task.objects.get(pk=inner_task_id)
task_name = func.__module__ + '.' + func.__name__
while True:
if task_name == "pulpcore.app.tasks.orphan.orphan_cleanup":
if ReservedResource.objects.exists():
# wait until there are no reservations
time.sleep(0.25)
continue
else:
task_status.state = TASK_STATES.RUNNING
task_status.save()
q = Queue('resource_manager', connection=redis_conn, async=False)
q.enqueue(func, args=inner_args, kwargs=inner_kwargs, job_id=inner_task_id,
timeout=TASK_TIMEOUT, **options)
task_status.state = TASK_STATES.COMPLETED
task_status.save()
return
try:
worker = _acquire_worker(resources)
except Worker.DoesNotExist:
# no worker is ready so we need to wait
time.sleep(0.25)
continue
try:
worker.lock_resources(task_status, resources)
except IntegrityError:
# we have a worker but we can't create the reservations so wait
time.sleep(0.25)
else:
# we have a worker with the locks
break
task_status.worker = worker
task_status.save()
try:
q = Queue(worker.name, connection=redis_conn)
q.enqueue(func, args=inner_args, kwargs=inner_kwargs, job_id=inner_task_id,
timeout=TASK_TIMEOUT, **options)
finally:
q.enqueue(_release_resources, args=(inner_task_id, ))
def _queue_reserved_task(func, inner_task_id, resources, inner_args, inner_kwargs, options):
"""
A task that encapsulates another task to be dispatched later.
This task being encapsulated is called the "inner" task, and a task name, UUID, and accepts a
list of positional args and keyword args for the inner task. These arguments are named
inner_args and inner_kwargs. inner_args is a list, and inner_kwargs is a dictionary passed to
the inner task as positional and keyword arguments using the * and ** operators.
The inner task is dispatched into a dedicated queue for a worker that is decided at dispatch
time. The logic deciding which queue receives a task is controlled through the
find_worker function.
Args:
func (basestring): The function to be called
inner_task_id (basestring): The task_id to be set on the task being called. By providing
the UUID, the caller can have an asynchronous reference to the inner task
that will be dispatched.
resources (basestring): The urls of the resource you wish to reserve for your task.
The system will ensure that no other tasks that want that same reservation will run
concurrently with yours.
inner_args (tuple): The positional arguments to pass on to the task.
inner_kwargs (dict): The keyword arguments to pass on to the task.
options (dict): For all options accepted by enqueue see the RQ docs
"""
redis_conn = connection.get_redis_connection()
task_status = Task.objects.get(pk=inner_task_id)
GuidMiddleware.set_guid(task_status.logging_cid)
task_name = func.__module__ + "." + func.__name__
while True:
if task_name == "pulpcore.app.tasks.orphan.orphan_cleanup":
if ReservedResource.objects.exists():
# wait until there are no reservations
time.sleep(0.25)
continue
else:
rq_worker = util.get_current_worker()
worker = Worker.objects.get(name=rq_worker.name)
task_status.worker = worker
task_status.set_running()
q = Queue("resource-manager", connection=redis_conn, is_async=False)
try:
q.enqueue(
func,
args=inner_args,
kwargs=inner_kwargs,
job_id=inner_task_id,
job_timeout=TASK_TIMEOUT,
**options,
)
task_status.set_completed()
except RedisConnectionError as e:
task_status.set_failed(e, None)
return
try:
with transaction.atomic():
# lock the worker - there is a similar lock in mark_worker_offline()
worker = _acquire_worker(resources)
# Attempt to lock all resources by their urls. Must be atomic to prevent deadlocks.
for resource in resources:
if worker.reservations.filter(resource=resource).exists():
reservation = worker.reservations.get(resource=resource)
else:
reservation = ReservedResource.objects.create(
worker=worker, resource=resource
)
TaskReservedResource.objects.create(resource=reservation, task=task_status)
except (Worker.DoesNotExist, IntegrityError):
# if worker is ready, or we have a worker but we can't create the reservations, wait
time.sleep(0.25)
else:
# we have a worker with the locks
break
task_status.worker = worker
task_status.save()
try:
q = Queue(worker.name, connection=redis_conn)
q.enqueue(
func,
args=inner_args,
kwargs=inner_kwargs,
job_id=inner_task_id,
job_timeout=TASK_TIMEOUT,
**options,
)
except RedisConnectionError as e:
task_status.set_failed(e, None)
