async def django_guid_api_usage(request: HttpRequest) -> JsonResponse:
"""
Uses each API function
"""
logger.info('Current GUID: %s', get_guid())
set_guid('another guid')
logger.info('Current GUID: %s', get_guid())
clear_guid()
logger.info('Current GUID: %s', get_guid())
return JsonResponse({'detail': ':)'})
def apply_async(cls,
args=None,
kwargs=None,
queue=None,
uuid=None,
**kw):
task_id = uuid or str(uuid4())
args = args or []
kwargs = kwargs or {}
queue = queue or getattr(cls.queue, 'im_func', cls.queue)
if not queue:
msg = f'{cls.name}: Queue value required and may not be None'
logger.error(msg)
raise ValueError(msg)
obj = {
'uuid': task_id,
'args': args,
'kwargs': kwargs,
'task': cls.name,
'time_pub': time.time()
}
guid = get_guid()
if guid:
obj['guid'] = guid
if bind_kwargs:
obj['bind_kwargs'] = bind_kwargs
obj.update(**kw)
if callable(queue):
queue = queue()
if not settings.IS_TESTING(sys.argv):
with pg_bus_conn() as conn:
conn.notify(queue, json.dumps(obj))
return (obj, queue)
def __init__(self, model=None):
self.parent_workflow_job_id = None
self.host_map = {}
self.guid = get_guid()
self.job_created = None
self.recent_event_timings = deque(maxlen=settings.MAX_WEBSOCKET_EVENT_RATE)
self.dispatcher = CallbackQueueDispatcher()
self.safe_env = {}
self.event_ct = 0
self.model = model
self.update_attempts = int(settings.DISPATCHER_DB_DOWNTOWN_TOLLERANCE / 5)
def __init__(self, model=None):
self.parent_workflow_job_id = None
self.host_map = {}
self.guid = get_guid()
self.job_created = None
self.recent_event_timings = deque(
maxlen=settings.MAX_WEBSOCKET_EVENT_RATE)
self.dispatcher = CallbackQueueDispatcher()
self.safe_env = {}
self.event_ct = 0
self.model = model
def dispatch(func, resources, args=None, kwargs=None, task_group=None):
"""
Enqueue a message to Pulp workers with a reservation.
This method provides normal enqueue functionality, while also requesting necessary locks for
serialized urls. No two tasks that claim the same resource can execute concurrently. It
accepts resources which it transforms into a list of urls (one for each resource).
This method creates a :class:`pulpcore.app.models.Task` object and returns it.
The values in `args` and `kwargs` must be JSON serializable, but may contain instances of
``uuid.UUID``.
Args:
func (callable): The function to be run by RQ when the necessary locks are acquired.
resources (list): A list of resources to this task needs exclusive access to while running.
Each resource can be either a `str` or a `django.models.Model` instance.
args (tuple): The positional arguments to pass on to the task.
kwargs (dict): The keyword arguments to pass on to the task.
task_group (pulpcore.app.models.TaskGroup): A TaskGroup to add the created Task to.
Returns (pulpcore.app.models.Task): The Pulp Task that was created.
Raises:
ValueError: When `resources` is an unsupported type.
"""
if settings.USE_NEW_WORKER_TYPE:
args_as_json = json.dumps(args, cls=UUIDEncoder)
kwargs_as_json = json.dumps(kwargs, cls=UUIDEncoder)
resources = _validate_and_get_resources(resources)
with transaction.atomic():
task = Task.objects.create(
state=TASK_STATES.WAITING,
logging_cid=(get_guid() or ""),
task_group=task_group,
name=f"{func.__module__}.{func.__name__}",
args=args_as_json,
kwargs=kwargs_as_json,
parent_task=Task.current(),
reserved_resources_record=resources,
)
# Notify workers
with db_connection.connection.cursor() as cursor:
cursor.execute("NOTIFY pulp_worker_wakeup")
return task
else:
RQ_job_id = _enqueue_with_reservation(func,
resources=resources,
args=args,
kwargs=kwargs,
task_group=task_group)
return Task.objects.get(pk=RQ_job_id.id)
def _enqueue_with_reservation(func,
resources,
args=None,
kwargs=None,
options=None,
task_group=None):
if not args:
args = tuple()
if not kwargs:
kwargs = dict()
if not options:
options = dict()
resources = _validate_and_get_resources(resources)
inner_task_id = str(uuid.uuid4())
resource_task_id = str(uuid.uuid4())
args_as_json = json.dumps(args, cls=UUIDEncoder)
kwargs_as_json = json.dumps(kwargs, cls=UUIDEncoder)
redis_conn = connection.get_redis_connection()
current_job = get_current_job(connection=redis_conn)
parent_kwarg = {}
if current_job:
# set the parent task of the spawned task to the current task ID (same as rq Job ID)
parent_kwarg["parent_task"] = Task.objects.get(pk=current_job.id)
with transaction.atomic():
task = Task.objects.create(
pk=inner_task_id,
_resource_job_id=resource_task_id,
state=TASK_STATES.WAITING,
logging_cid=(get_guid() or ""),
task_group=task_group,
name=f"{func.__module__}.{func.__name__}",
args=args_as_json,
kwargs=kwargs_as_json,
reserved_resources_record=resources,
**parent_kwarg,
)
task_args = (func, inner_task_id, resources, args, kwargs, options)
try:
q = Queue("resource-manager", connection=redis_conn)
q.enqueue(
_queue_reserved_task,
job_id=resource_task_id,
args=task_args,
job_timeout=TASK_TIMEOUT,
)
except RedisConnectionError as e:
task.set_failed(e, None)
return Job(id=inner_task_id, connection=redis_conn)
def publish_task_from_worker_or_request(headers: dict, **kwargs) -> None:
"""
Called when a request or celery worker publishes a task to the worker pool
by calling task.delay(), task.apply_async() or using another equivalent method.
This is where we transfer state from a parent process to a child process.
"""
guid = get_guid()
logger.info('Setting task request header as %s', guid)
headers[settings.guid_header_name] = guid
if settings.integration_settings.celery.log_parent:
current = celery_current.get()
if current:
headers[parent_header] = current
def test_worker_prerun_guid_does_not_exist(monkeypatch, mocker: MockerFixture,
mock_uuid):
"""
Tests that a GUID is set if it does not exist
"""
mock_task = mocker.Mock()
mock_task.request = {'Correlation-ID': None}
mocked_settings = deepcopy(django_settings.DJANGO_GUID)
mocked_settings['INTEGRATIONS'] = [CeleryIntegration(log_parent=False)]
with override_settings(DJANGO_GUID=mocked_settings):
settings = Settings()
monkeypatch.setattr('django_guid.integrations.celery.signals.settings',
settings)
worker_prerun(mock_task)
assert get_guid() == '704ae5472cae4f8daa8f2cc5a5a8mock'
def test_worker_prerun_guid_exists(monkeypatch, mocker: MockerFixture,
two_unique_uuid4):
"""
Tests that GUID is set to the GUID if a GUID exists in the task object.
"""
mock_task = mocker.Mock()
mock_task.request = {'Correlation-ID': '704ae5472cae4f8daa8f2cc5a5a8mock'}
mocked_settings = deepcopy(django_settings.DJANGO_GUID)
mocked_settings['INTEGRATIONS'] = [CeleryIntegration(log_parent=False)]
with override_settings(DJANGO_GUID=mocked_settings):
settings = Settings()
monkeypatch.setattr('django_guid.integrations.celery.signals.settings',
settings)
worker_prerun(mock_task)
assert get_guid() == '704ae5472cae4f8daa8f2cc5a5a8mock'
def test_cleanup(monkeypatch, mocker: MockerFixture):
"""
Test that cleanup works as expected
"""
set_guid('123')
celery_current.set('123')
celery_parent.set('123')
mocked_settings = deepcopy(django_settings.DJANGO_GUID)
mocked_settings['INTEGRATIONS'] = [CeleryIntegration(log_parent=True)]
with override_settings(DJANGO_GUID=mocked_settings):
settings = Settings()
monkeypatch.setattr('django_guid.integrations.celery.signals.settings',
settings)
clean_up(task=mocker.Mock())
assert [get_guid(), celery_current.get(),
celery_parent.get()] == [None, None, None]
def test_worker_prerun_guid_log_parent_with_origin(monkeypatch,
mocker: MockerFixture,
mock_uuid_two_unique):
"""
Tests that depth works when there is an origin
"""
from django_guid.integrations.celery.signals import parent_header
mock_task = mocker.Mock()
mock_task.request = {
'Correlation-ID': None,
parent_header: '1234'
} # No origin
mocked_settings = deepcopy(django_settings.DJANGO_GUID)
mocked_settings['INTEGRATIONS'] = [CeleryIntegration(log_parent=True)]
with override_settings(DJANGO_GUID=mocked_settings):
settings = Settings()
monkeypatch.setattr('django_guid.integrations.celery.signals.settings',
settings)
worker_prerun(mock_task)
assert get_guid() == '704ae5472cae4f8daa8f2cc5a5a8mock'
assert celery_current.get() == 'c494886651cd4baaa8654e4d24a8mock'
assert celery_parent.get() == '1234'
