def __sendNotification(self, subject, message='', **kwargs):
try:
contentDict = { 'job_id': self.JobId,
'export_id': self.job.get('job_group_id'),
'archive_id': self.ArchiveId,
'project': self.Project,
'type': self.Type,
'ingest_server': self.hostname,
'dataproduct': self.DataProduct,
'srm_url': self.PrimaryUri }
if 'ObservationId' in self.job:
contentDict['otdb_id'] = self.job['ObservationId']
if self.lta_site:
contentDict['lta_site'] = self.lta_site
if message:
contentDict['message'] = message
for k,v in list(kwargs.items()):
contentDict[k] = v
msg = EventMessage(subject="%s.%s" % (INGEST_NOTIFICATION_PREFIX, subject), content=contentDict)
msg.ttl = 48*3600 #remove message from queue's when not picked up within 48 hours
logger.info('Sending notification %s: %s' % (subject, str(contentDict).replace('\n', ' ')))
self.event_bus.send(msg)
except Exception as e:
logger.error(str(e))
def run(self):
with self.event_bus:
while True:
try:
self.__clearFinishedJobs()
with self.__lock:
starting_threads = [
job_thread_dict['thread'] for job_thread_dict in
list(self.__running_jobs.values())
if 'pipeline' not in job_thread_dict
]
pipelines = [
job_thread_dict['pipeline'] for job_thread_dict in
list(self.__running_jobs.values())
if 'pipeline' in job_thread_dict
]
initializing_pipelines = [
pipeline for pipeline in pipelines
if pipeline.status ==
IngestPipeline.STATUS_INITIALIZING
]
transferring_pipelines = [
pipeline for pipeline in pipelines
if pipeline.status ==
IngestPipeline.STATUS_TRANSFERRING
]
finalizing_pipelines = [
pipeline for pipeline in pipelines if
pipeline.status == IngestPipeline.STATUS_FINALIZING
]
finished_pipelines = [
pipeline for pipeline in pipelines if
pipeline.status == IngestPipeline.STATUS_FINISHED
]
log_interval = 5 if self.__running_jobs else 60
if datetime.utcnow(
) - self.__running_jobs_log_timestamp > timedelta(
seconds=log_interval):
status_log_line = "status: running %s jobs: #starting=%d, #transferring=%d, #finalizing=%d, #finished=%d, bandwith used on network interface(s) %s %s (%s), load=%.1f" % (
len(self.__running_jobs),
len(initializing_pipelines) +
len(starting_threads), len(transferring_pipelines),
len(finalizing_pipelines), len(finished_pipelines),
NET_IF_TO_MONITOR,
humanreadablesize(self.__prev_used_bandwidth,
'bps'),
humanreadablesize(self.__prev_used_bandwidth / 8,
'Bps'), os.getloadavg()[0])
logger.info(status_log_line)
self.__running_jobs_log_timestamp = datetime.utcnow()
msg = EventMessage(subject="%s.%s" %
(INGEST_NOTIFICATION_PREFIX,
'TransferServiceStatus'),
content={
'ingest_server':
socket.gethostname(),
'message':
status_log_line
})
msg.ttl = 3600 # remove message from queue's when not picked up within 1 hours
self.event_bus.send(msg)
time.sleep(5)
except KeyboardInterrupt:
break
except Exception as e:
logger.error(e)
