def emit_event_detail(serializer, relation, **kwargs): instance = kwargs['instance'] created = kwargs['created'] if created: event_serializer = serializer(instance) consumers.emit_channel_notification( '-'.join([ event_serializer.get_group_name(instance), str(getattr(instance, relation)) ]), event_serializer.data)
def _websocket_emit_status(self, status): try: status_data = dict(job_id=self.id, status=status) status_data.update(self.websocket_emit_data()) status_data['group_name'] = 'jobs' emit_channel_notification('jobs-status_changed', status_data) except IOError: # includes socket errors logger.exception( '%s failed to emit channel msg about status change', self.log_format)
def update_computed_fields(self): future_rs = tzcron.Schedule(self.schedule.crontab, pytz.utc) next_run_actual = next(future_rs) if next_run_actual is not None: if not datetime_exists(next_run_actual): # skip imaginary dates, like 2:30 on DST boundaries next_run_actual = next(future_rs) next_run_actual = next_run_actual.astimezone(pytz.utc) self.next_run = next_run_actual emit_channel_notification('schedules-changed', dict(id=self.id, group_name='schedules'))
def handle(self, *args, **kwargs): i=6 while i < 200: consumers.emit_channel_notification("job_events-4", {"id": 16139540, "type": "job_event", "url": "/api/v1/job_events/16139540/", "related": {"job": "/api/v1/jobs/38631/"}, "summary_fields": { "job": { "id": 4, "name": "Backup Job Backup Mail from Knet knet.milkywan.cloud", "status": "running", "failed": False, "elapsed": "0.000" } }, "created": "2020-04-11T17:50:38.159331+00:00", "modified": "2020-04-11T17:50:38.159340+00:00", "job": 4, "event": "verbose", "counter": i, "event_display": "Verbose", "event_data": {}, "event_level": 0, "failed": False, "changed": False, "uuid": "", "task": "", "stdout": "Merging into master chunks insddex {}...".format(i), "start_line": 8, "end_line": 9, "verbosity": 0, "event_name": "verbose", "group_name": "job_events" }) i = i+1
def handle(self, *args, **kwargs): consumers.emit_channel_notification( "jobs-status_changed", { "job_id": 38631, "status": "successful", "job_name": "Backup Job Backup mail from Knet", "group_name": "jobs" }) consumers.emit_channel_notification( "jobs-status_changed", { "job_id": 38631, "status": "failed", "job_name": "Backup Job Backup mail from Knet", "group_name": "jobs" }) consumers.emit_channel_notification( "jobs-status_changed", { "job_id": 38631, "status": "running", "job_name": "Backup Job Backup mail from Knet", "group_name": "jobs" })
def callback_worker(self, queue_actual, idx): signal_handler = WorkerSignalHandler() while not signal_handler.kill_now: try: body = queue_actual.get(block=True, timeout=1) except QueueEmpty: continue except Exception as e: logger.error("Exception on worker thread, restarting: " + str(e)) continue try: event_map = { 'job_id': JobEvent, 'catalog': Catalog, } if not any([key in body for key in event_map]): raise Exception('Payload does not have a job identifier') if settings.DEBUG: from pygments import highlight from pygments.lexers import PythonLexer from pygments.formatters import Terminal256Formatter from pprint import pformat logger.info('Body: {}'.format( highlight(pformat(body, width=160), PythonLexer(), Terminal256Formatter( style='friendly')))[:1024 * 4]) def _save_event_data(): for key, cls in event_map.items(): if key in body: cls.create_from_data(**body) job_identifier = 'unknown job' for key in event_map.keys(): if key in body: job_identifier = body[key] break if body.get('event') == 'EOF': try: msg = 'Event processing is finished for Job {}, sending notifications' logger.info(msg.format(job_identifier)) # EOF events are sent when stdout for the running task is # closed. don't actually persist them to the database; we # just use them to report `summary` websocket events as an # approximation for when a job is "done" emit_channel_notification( 'jobs-summary', dict(group_name='jobs', job_id=job_identifier)) # Additionally, when we've processed all events, we should # have all the data we need to send out success/failure # notification templates j = Job.objects.get(pk=job_identifier) if hasattr(j, 'send_notification_templates'): retries = 0 while retries < 5: if j.finished: state = 'succeeded' if j.status == 'successful' else 'failed' j.send_notification_templates(state) break else: # wait a few seconds to avoid a race where the # events are persisted _before_ the UJ.status # changes from running -> successful retries += 1 time.sleep(1) j = Job.objects.get(pk=job_identifier) except Exception: logger.exception( 'Worker failed to emit notifications: Job {}'. format(job_identifier)) continue retries = 0 while retries <= self.MAX_RETRIES: try: _save_event_data() break except (OperationalError, InterfaceError, InternalError): if retries >= self.MAX_RETRIES: msg = 'Worker could not re-establish database connection, shutting down gracefully: Job {}' logger.exception(msg.format(job_identifier)) os.kill(os.getppid(), signal.SIGINT) return delay = 60 * retries logger.exception( 'Database Error Saving Job Event, retry #{i} in {delay} seconds:' .format(i=retries + 1, delay=delay)) django_connection.close() time.sleep(delay) retries += 1 except DatabaseError: logger.exception( 'Database Error Saving Job Event for Job {}'. format(job_identifier)) break except Exception as exc: import traceback tb = traceback.format_exc() logger.error('Callback Task Processor Raised Exception: %r', exc) logger.error('Detail: {}'.format(tb))