def __sendNotification(self, subject, message='', **kwargs): try: contentDict = { 'job_id': self.JobId, 'export_id': self.job.get('job_group_id'), 'archive_id': self.ArchiveId, 'project': self.Project, 'type': self.Type, 'ingest_server': self.hostname, 'dataproduct': self.DataProduct, 'srm_url': self.PrimaryUri } if 'ObservationId' in self.job: contentDict['otdb_id'] = self.job['ObservationId'] if self.lta_site: contentDict['lta_site'] = self.lta_site if message: contentDict['message'] = message for k,v in list(kwargs.items()): contentDict[k] = v msg = EventMessage(subject="%s.%s" % (INGEST_NOTIFICATION_PREFIX, subject), content=contentDict) msg.ttl = 48*3600 #remove message from queue's when not picked up within 48 hours logger.info('Sending notification %s: %s' % (subject, str(contentDict).replace('\n', ' '))) self.event_bus.send(msg) except Exception as e: logger.error(str(e))
def run(self): with self.event_bus: while True: try: self.__clearFinishedJobs() with self.__lock: starting_threads = [ job_thread_dict['thread'] for job_thread_dict in list(self.__running_jobs.values()) if 'pipeline' not in job_thread_dict ] pipelines = [ job_thread_dict['pipeline'] for job_thread_dict in list(self.__running_jobs.values()) if 'pipeline' in job_thread_dict ] initializing_pipelines = [ pipeline for pipeline in pipelines if pipeline.status == IngestPipeline.STATUS_INITIALIZING ] transferring_pipelines = [ pipeline for pipeline in pipelines if pipeline.status == IngestPipeline.STATUS_TRANSFERRING ] finalizing_pipelines = [ pipeline for pipeline in pipelines if pipeline.status == IngestPipeline.STATUS_FINALIZING ] finished_pipelines = [ pipeline for pipeline in pipelines if pipeline.status == IngestPipeline.STATUS_FINISHED ] log_interval = 5 if self.__running_jobs else 60 if datetime.utcnow( ) - self.__running_jobs_log_timestamp > timedelta( seconds=log_interval): status_log_line = "status: running %s jobs: #starting=%d, #transferring=%d, #finalizing=%d, #finished=%d, bandwith used on network interface(s) %s %s (%s), load=%.1f" % ( len(self.__running_jobs), len(initializing_pipelines) + len(starting_threads), len(transferring_pipelines), len(finalizing_pipelines), len(finished_pipelines), NET_IF_TO_MONITOR, humanreadablesize(self.__prev_used_bandwidth, 'bps'), humanreadablesize(self.__prev_used_bandwidth / 8, 'Bps'), os.getloadavg()[0]) logger.info(status_log_line) self.__running_jobs_log_timestamp = datetime.utcnow() msg = EventMessage(subject="%s.%s" % (INGEST_NOTIFICATION_PREFIX, 'TransferServiceStatus'), content={ 'ingest_server': socket.gethostname(), 'message': status_log_line }) msg.ttl = 3600 # remove message from queue's when not picked up within 1 hours self.event_bus.send(msg) time.sleep(5) except KeyboardInterrupt: break except Exception as e: logger.error(e)