def write(self, preferred_queue, body):
if 'guid' in body:
GuidMiddleware.set_guid(body['guid'])
try:
# when the cluster heartbeat occurs, clean up internally
if isinstance(body,
dict) and 'cluster_node_heartbeat' in body['task']:
self.cleanup()
if self.should_grow:
self.up()
# we don't care about "preferred queue" round robin distribution, just
# find the first non-busy worker and claim it
workers = self.workers[:]
random.shuffle(workers)
for w in workers:
if not w.busy:
w.put(body)
break
else:
return super(AutoscalePool, self).write(preferred_queue, body)
except Exception:
for conn in connections.all():
# If the database connection has a hiccup, re-establish a new
# connection
conn.close_if_unusable_or_obsolete()
logger.exception('failed to write inbound message')
def run_callable(self, body):
'''
Given some AMQP message, import the correct Python code and run it.
'''
task = body['task']
uuid = body.get('uuid', '<unknown>')
args = body.get('args', [])
kwargs = body.get('kwargs', {})
if 'guid' in body:
GuidMiddleware.set_guid(body.pop('guid'))
_call = TaskWorker.resolve_callable(task)
if inspect.isclass(_call):
# the callable is a class, e.g., RunJob; instantiate and
# return its `run()` method
_call = _call().run
# don't print kwargs, they often contain launch-time secrets
logger.debug('task {} starting {}(*{})'.format(uuid, task, args))
return _call(*args, **kwargs)
def run():
ppid = os.getppid()
logger.warn('periodic beat started')
while True:
if os.getppid() != ppid:
# if the parent PID changes, this process has been orphaned
# via e.g., segfault or sigkill, we should exit too
pid = os.getpid()
logger.warn(f'periodic beat exiting gracefully pid:{pid}')
raise SystemExit()
try:
for conn in connections.all():
# If the database connection has a hiccup, re-establish a new
# connection
conn.close_if_unusable_or_obsolete()
GuidMiddleware.set_guid(GuidMiddleware._generate_guid())
self.run_pending()
except Exception:
logger.exception(
'encountered an error while scheduling periodic tasks')
time.sleep(idle_seconds)
def perform_job(self, job, queue):
"""
Set the :class:`pulpcore.app.models.Task` to running and init logging.
This method is called by the worker's work horse thread (the forked child) just before the
task begins executing.
Args:
job (rq.job.Job): The job to perform
queue (rq.queue.Queue): The Queue associated with the job
"""
try:
task = Task.objects.get(pk=job.get_id())
except Task.DoesNotExist:
pass
else:
task.set_running()
user = get_users_with_perms(task).first()
_set_current_user(user)
GuidMiddleware.set_guid(task.logging_cid)
with TaskWorkingDirectory(job):
return super().perform_job(job, queue)
def perform_work(self, body):
try:
flush = body.get('event') == 'FLUSH'
if flush:
self.last_event = ''
if not flush:
event_map = {
'job_id': JobEvent,
'ad_hoc_command_id': AdHocCommandEvent,
'project_update_id': ProjectUpdateEvent,
'inventory_update_id': InventoryUpdateEvent,
'system_job_id': SystemJobEvent,
}
job_identifier = 'unknown job'
for key, cls in event_map.items():
if key in body:
job_identifier = body[key]
break
self.last_event = f'\n\t- {cls.__name__} for #{job_identifier} ({body.get("event", "")} {body.get("uuid", "")})' # noqa
if body.get('event') == 'EOF':
try:
if 'guid' in body:
GuidMiddleware.set_guid(body['guid'])
final_counter = body.get('final_counter', 0)
logger.info(
'Event processing is finished for Job {}, sending notifications'
.format(job_identifier))
# EOF events are sent when stdout for the running task is
# closed. don't actually persist them to the database; we
# just use them to report `summary` websocket events as an
# approximation for when a job is "done"
emit_channel_notification(
'jobs-summary',
dict(group_name='jobs',
unified_job_id=job_identifier,
final_counter=final_counter))
# Additionally, when we've processed all events, we should
# have all the data we need to send out success/failure
# notification templates
uj = UnifiedJob.objects.get(pk=job_identifier)
if isinstance(uj, Job):
# *actual playbooks* send their success/failure
# notifications in response to the playbook_on_stats
# event handling code in main.models.events
pass
elif hasattr(uj, 'send_notification_templates'):
handle_success_and_failure_notifications.apply_async(
[uj.id])
except Exception:
logger.exception(
'Worker failed to emit notifications: Job {}'.
format(job_identifier))
finally:
self.subsystem_metrics.inc(
'callback_receiver_events_in_memory', -1)
GuidMiddleware.set_guid('')
return
skip_websocket_message = body.pop('skip_websocket_message',
False)
event = cls.create_from_data(**body)
if skip_websocket_message:
event._skip_websocket_message = True
self.buff.setdefault(cls, []).append(event)
retries = 0
while retries <= self.MAX_RETRIES:
try:
self.flush(force=flush)
break
except (OperationalError, InterfaceError, InternalError):
if retries >= self.MAX_RETRIES:
logger.exception(
'Worker could not re-establish database connectivity, giving up on one or more events.'
)
return
delay = 60 * retries
logger.exception(
'Database Error Saving Job Event, retry #{i} in {delay} seconds:'
.format(i=retries + 1, delay=delay))
django_connection.close()
time.sleep(delay)
retries += 1
except DatabaseError:
logger.exception('Database Error Saving Job Event')
break
except Exception as exc:
tb = traceback.format_exc()
logger.error('Callback Task Processor Raised Exception: %r', exc)
logger.error('Detail: {}'.format(tb))
def _queue_reserved_task(func, inner_task_id, resources, inner_args, inner_kwargs, options):
"""
A task that encapsulates another task to be dispatched later.
This task being encapsulated is called the "inner" task, and a task name, UUID, and accepts a
list of positional args and keyword args for the inner task. These arguments are named
inner_args and inner_kwargs. inner_args is a list, and inner_kwargs is a dictionary passed to
the inner task as positional and keyword arguments using the * and ** operators.
The inner task is dispatched into a dedicated queue for a worker that is decided at dispatch
time. The logic deciding which queue receives a task is controlled through the
find_worker function.
Args:
func (basestring): The function to be called
inner_task_id (basestring): The task_id to be set on the task being called. By providing
the UUID, the caller can have an asynchronous reference to the inner task
that will be dispatched.
resources (basestring): The urls of the resource you wish to reserve for your task.
The system will ensure that no other tasks that want that same reservation will run
concurrently with yours.
inner_args (tuple): The positional arguments to pass on to the task.
inner_kwargs (dict): The keyword arguments to pass on to the task.
options (dict): For all options accepted by enqueue see the RQ docs
"""
redis_conn = connection.get_redis_connection()
task_status = Task.objects.get(pk=inner_task_id)
GuidMiddleware.set_guid(task_status.logging_cid)
task_name = func.__module__ + "." + func.__name__
while True:
if task_name == "pulpcore.app.tasks.orphan.orphan_cleanup":
if ReservedResource.objects.exists():
# wait until there are no reservations
time.sleep(0.25)
continue
else:
rq_worker = util.get_current_worker()
worker = Worker.objects.get(name=rq_worker.name)
task_status.worker = worker
task_status.set_running()
q = Queue("resource-manager", connection=redis_conn, is_async=False)
try:
q.enqueue(
func,
args=inner_args,
kwargs=inner_kwargs,
job_id=inner_task_id,
job_timeout=TASK_TIMEOUT,
**options,
)
task_status.set_completed()
except RedisConnectionError as e:
task_status.set_failed(e, None)
return
try:
with transaction.atomic():
# lock the worker - there is a similar lock in mark_worker_offline()
worker = _acquire_worker(resources)
# Attempt to lock all resources by their urls. Must be atomic to prevent deadlocks.
for resource in resources:
if worker.reservations.filter(resource=resource).exists():
reservation = worker.reservations.get(resource=resource)
else:
reservation = ReservedResource.objects.create(
worker=worker, resource=resource
)
TaskReservedResource.objects.create(resource=reservation, task=task_status)
except (Worker.DoesNotExist, IntegrityError):
# if worker is ready, or we have a worker but we can't create the reservations, wait
time.sleep(0.25)
else:
# we have a worker with the locks
break
task_status.worker = worker
task_status.save()
try:
q = Queue(worker.name, connection=redis_conn)
q.enqueue(
func,
args=inner_args,
kwargs=inner_kwargs,
job_id=inner_task_id,
job_timeout=TASK_TIMEOUT,
**options,
)
except RedisConnectionError as e:
task_status.set_failed(e, None)