def apply_async(cls,
args=None,
kwargs=None,
queue=None,
uuid=None,
**kw):
task_id = uuid or str(uuid4())
args = args or []
kwargs = kwargs or {}
queue = (queue or getattr(cls.queue, 'im_func', cls.queue))
if not queue:
msg = f'{cls.name}: Queue value required and may not be None'
logger.error(msg)
raise ValueError(msg)
obj = {
'uuid': task_id,
'args': args,
'kwargs': kwargs,
'task': cls.name
}
guid = GuidMiddleware.get_guid()
if guid:
obj['guid'] = guid
obj.update(**kw)
if callable(queue):
queue = queue()
if not settings.IS_TESTING(sys.argv):
with pg_bus_conn() as conn:
conn.notify(queue, json.dumps(obj))
return (obj, queue)
def _enqueue_with_reservation(
func, resources, args=None, kwargs=None, options=None, task_group=None
):
if not args:
args = tuple()
if not kwargs:
kwargs = dict()
if not options:
options = dict()
def as_url(r):
if isinstance(r, str):
return r
if isinstance(r, Model):
return util.get_url(r)
raise ValueError(_("Must be (str|Model)"))
resources = {as_url(r) for r in resources}
inner_task_id = str(uuid.uuid4())
resource_task_id = str(uuid.uuid4())
redis_conn = connection.get_redis_connection()
current_job = get_current_job(connection=redis_conn)
parent_kwarg = {}
json.dumps(args, cls=NonJSONWarningEncoder)
json.dumps(kwargs, cls=NonJSONWarningEncoder)
if current_job:
# set the parent task of the spawned task to the current task ID (same as rq Job ID)
parent_kwarg["parent_task"] = Task.objects.get(pk=current_job.id)
with transaction.atomic():
task = Task.objects.create(
pk=inner_task_id,
_resource_job_id=resource_task_id,
state=TASK_STATES.WAITING,
logging_cid=(GuidMiddleware.get_guid() or ""),
task_group=task_group,
name=f"{func.__module__}.{func.__name__}",
**parent_kwarg,
)
for resource in resources:
reservation_record = ReservedResourceRecord.objects.get_or_create(resource=resource)[0]
TaskReservedResourceRecord.objects.create(resource=reservation_record, task=task)
task_args = (func, inner_task_id, list(resources), args, kwargs, options)
try:
q = Queue("resource-manager", connection=redis_conn)
q.enqueue(
_queue_reserved_task,
job_id=resource_task_id,
args=task_args,
job_timeout=TASK_TIMEOUT,
)
except RedisConnectionError as e:
task.set_failed(e, None)
return Job(id=inner_task_id, connection=redis_conn)
def __init__(self, model=None):
self.parent_workflow_job_id = None
self.host_map = {}
self.guid = GuidMiddleware.get_guid()
self.job_created = None
self.recent_event_timings = deque(
maxlen=settings.MAX_WEBSOCKET_EVENT_RATE)
self.dispatcher = CallbackQueueDispatcher()
self.safe_env = {}
self.event_ct = 0
self.model = model
def filter(self, record: LogRecord) -> bool:
"""
Determines that the specified record is to be logged.
From the docs:
Is the specified record to be logged? Returns 0 for no, nonzero for
yes. If deemed appropriate, the record may be modified in-place.
:param record: Log record
:return: True
"""
record.correlation_id = GuidMiddleware.get_guid()
return True
def filter(self, record):
guid = GuidMiddleware.get_guid() or '-'
if MODE == 'development':
guid = guid[:8]
record.guid = guid
return True
def enqueue_with_reservation(
func, resources, args=None, kwargs=None, options=None, task_group=None
):
"""
Enqueue a message to Pulp workers with a reservation.
This method provides normal enqueue functionality, while also requesting necessary locks for
serialized urls. No two tasks that claim the same resource can execute concurrently. It
accepts resources which it transforms into a list of urls (one for each resource).
This does not dispatch the task directly, but instead promises to dispatch it later by
encapsulating the desired task through a call to a :func:`_queue_reserved_task` task. See
the docblock on :func:`_queue_reserved_task` for more information on this.
This method creates a :class:`pulpcore.app.models.Task` object. Pulp expects to poll on a
task just after calling this method, so a Task entry needs to exist for it
before it returns.
Args:
func (callable): The function to be run by RQ when the necessary locks are acquired.
resources (list): A list of resources to reserve guaranteeing that only one task reserves
these resources. Each resource can be either a (str) resource URL or a
(django.models.Model) resource instance.
args (tuple): The positional arguments to pass on to the task.
kwargs (dict): The keyword arguments to pass on to the task.
options (dict): The options to be passed on to the task.
task_group (pulpcore.app.models.TaskGroup): A TaskGroup to add the created Task to.
Returns (rq.job.job): An RQ Job instance as returned by RQ's enqueue function
Raises:
ValueError: When `resources` is an unsupported type.
"""
if not args:
args = tuple()
if not kwargs:
kwargs = dict()
if not options:
options = dict()
def as_url(r):
if isinstance(r, str):
return r
if isinstance(r, Model):
return util.get_url(r)
raise ValueError(_("Must be (str|Model)"))
resources = {as_url(r) for r in resources}
inner_task_id = str(uuid.uuid4())
resource_task_id = str(uuid.uuid4())
redis_conn = connection.get_redis_connection()
current_job = get_current_job(connection=redis_conn)
parent_kwarg = {}
if current_job:
# set the parent task of the spawned task to the current task ID (same as rq Job ID)
parent_kwarg["parent_task"] = Task.objects.get(pk=current_job.id)
with transaction.atomic():
task = Task.objects.create(
pk=inner_task_id,
_resource_job_id=resource_task_id,
state=TASK_STATES.WAITING,
logging_cid=(GuidMiddleware.get_guid() or ""),
task_group=task_group,
name=f"{func.__module__}.{func.__name__}",
**parent_kwarg,
)
for resource in resources:
reservation_record = ReservedResourceRecord.objects.get_or_create(resource=resource)[0]
TaskReservedResourceRecord.objects.create(resource=reservation_record, task=task)
task_args = (func, inner_task_id, list(resources), args, kwargs, options)
try:
q = Queue("resource-manager", connection=redis_conn)
q.enqueue(
_queue_reserved_task,
job_id=resource_task_id,
args=task_args,
job_timeout=TASK_TIMEOUT,
)
except RedisConnectionError as e:
task.set_failed(e, None)
return Job(id=inner_task_id, connection=redis_conn)
