def __sendNotification(self, subject, message='', **kwargs):
try:
contentDict = { 'job_id': self.JobId,
'export_id': self.job.get('job_group_id'),
'archive_id': self.ArchiveId,
'project': self.Project,
'type': self.Type,
'ingest_server': self.hostname,
'dataproduct': self.DataProduct,
'srm_url': self.PrimaryUri }
if 'ObservationId' in self.job:
contentDict['otdb_id'] = self.job['ObservationId']
if self.lta_site:
contentDict['lta_site'] = self.lta_site
if message:
contentDict['message'] = message
for k,v in list(kwargs.items()):
contentDict[k] = v
msg = EventMessage(subject="%s.%s" % (INGEST_NOTIFICATION_PREFIX, subject), content=contentDict)
msg.ttl = 48*3600 #remove message from queue's when not picked up within 48 hours
logger.info('Sending notification %s: %s' % (subject, str(contentDict).replace('\n', ' ')))
self.event_bus.send(msg)
except Exception as e:
logger.error(str(e))
def testPredecessors(self):
"""
Request the resources for a simulated obsid 1, with the following predecessor tree:
1 requires 2, 3
2 requires 3
3 requires nothing
"""
with RATaskSpecified(otdb_notification_busname=self.busname,
notification_busname=self.busname) as jts:
# Send fake status update
with ToBus(self.status_service) as tb:
msg = EventMessage(
content={
"treeID": 1,
"state": "prescheduled",
"time_of_change": datetime.datetime(2016, 1, 1),
})
tb.send(msg)
# Wait for message to arrive
self.assertTrue(self.trigger.wait())
# Verify message
self.assertEqual(self.trigger.args[0], 1)
resourceIndicators = self.trigger.args[2]
self.assertIn("1", resourceIndicators)
self.assertIn("2", resourceIndicators)
self.assertIn("3", resourceIndicators)
# Make sure we only requested exactly three parsets
self.assertEqual(self.requested_parsets, 3)
def _sendNotification(self, user, IP):
msg = EventMessage(subject=TRIGGER_SUBMISSION_NOTIFICATION_SUBJECT,
content="Trigger received by " + str(user) +
" (IP:" + IP + ")")
try:
notification_bus.send(msg)
except Exception as err:
logger.error("Could not send notification ->" + str(err))
def _sendNotification(self, subject, content):
try:
msg = EventMessage(subject="%s.%s" %
(DEFAULT_DM_NOTIFICATION_PREFIX, subject),
content=content)
logger.info('Sending notification with subject %s to %s: %s',
msg.subject, self.exchange, msg.content)
self.send(msg)
except Exception as e:
logger.error(str(e))
def createAndSendSpecifiedTask(self, main_id, status):
# Construct root node of tree
resultTree = self.get_specification_with_predecessors(
main_id, "otdb", status, {})
logger.info("Sending result: %s" % resultTree)
# Put result on bus
msg = EventMessage(content=resultTree)
self.send_bus.send(msg)
logger.info("Result sent")
def onObservationPrescheduled(self, treeId, modificationTime):
logger.info("Processing obs ID %s", treeId)
# Request the parset
main_obsID = treeId
main_parset, _ = self.parset_rpc(OtdbID=main_obsID)
# Construct a dict of all the parsets we retrieved
parsets = {}
parsets[main_obsID] = main_parset
logger.info("Processing predecessors")
# Collect the initial set of predecessors
request_obsIDs = set(predecessors(main_parset))
logger.info("Processing %s", request_obsIDs)
# Iterate recursively over all known predecessor obsIDs, and request their parsets
while request_obsIDs:
obsID = request_obsIDs.pop()
if obsID in parsets:
# Predecessor lists can overlap -- we already have this one
continue
logger.info("Fetching predecessor %s", obsID)
# Request predecessor parset
parsets[obsID], _ = self.parset_rpc(OtdbID=obsID)
# Add the list of predecessors
request_obsIDs = request_obsIDs.union(predecessors(parsets[obsID]))
# Convert parsets to resource indicators
logger.info("Extracting resource indicators")
resourceIndicators = dict([(str(obsID),
resourceIndicatorsFromParset(parset))
for (obsID, parset) in parsets.iteritems()])
# Construct and send result message
logger.info("Sending result")
result = {
"sasID": main_obsID,
"state": "prescheduled",
"time_of_change": modificationTime,
"resource_indicators": resourceIndicators,
}
# Put result on bus
msg = EventMessage(content=result)
self.send_bus.send(msg)
logger.info("Result sent")
def _sendNotification(self, subject, contentDict):
try:
if subject and contentDict:
msg = EventMessage(subject="%s.%s" %
(DEFAULT_RADB_NOTIFICATION_PREFIX, subject),
content=contentDict)
logger.info('Sending notification %s to %s: %s', subject,
self.event_bus.exchange,
str(contentDict).replace('\n', ' '))
self.event_bus.send(msg)
except Exception as e:
logger.error(str(e))
def TaskSetStatus(self, OtdbID, NewStatus, UpdateTimestamps):
print "***** TaskSetStatus(%s,%s) *****" % (OtdbID, NewStatus)
# Broadcast the state change
content = {
"treeID": OtdbID,
"state": NewStatus,
"time_of_change": datetime.datetime.utcnow()
}
msg = EventMessage(context=DEFAULT_OTDB_NOTIFICATION_SUBJECT,
content=content)
self.notification_bus.send(msg)
return {'OtdbID': OtdbID, 'MomID': None, 'Success': True}
def _send_notification(user, host, project, trigger_id, metadata):
try:
content = {
"user": user,
"host": host,
"project": project,
"trigger_id": trigger_id,
"metadata": metadata
}
msg = EventMessage(subject=DEFAULT_TRIGGER_NOTIFICATION_SUBJECT,
content=content)
with ToBus(exchange=DEFAULT_BUSNAME,
broker=DEFAULT_BROKER) as notification_bus:
notification_bus.send(msg)
except Exception as err:
logger.error("Could not send notification ->" + str(err))
def TaskSetStatus(self, OtdbID, NewStatus, UpdateTimestamps):
logger.info("***** TaskSetStatus(%s,%s) *****", OtdbID, NewStatus)
global otdb_status
otdb_status[OtdbID] = NewStatus
# Broadcast the state change
content = {
"treeID": OtdbID,
"state": NewStatus,
"time_of_change": datetime.datetime.utcnow()
}
msg = EventMessage(subject=DEFAULT_OTDB_NOTIFICATION_SUBJECT,
content=content)
self.notification_bus.send(msg)
return {'OtdbID': OtdbID, 'MomID': None, 'Success': True}
def createAndSendSpecifiedTask(self, main_id, status):
spec = Specification(self.otdbrpc, self.momrpc, self.radbrpc)
spec.status = status
spec.read_from_OTDB_with_predecessors(main_id, "otdb", {})
spec.read_from_mom()
spec.update_start_end_times()
# spec.insert_into_radb() is still done in resource_assigner for now.
result_tree = spec.as_dict()
if spec.status == status:
logger.info("Sending result: %s" % result_tree)
# Put result on bus
msg = EventMessage(subject=DEFAULT_RA_TASK_SPECIFIED_NOTIFICATION_SUBJECT,
content=result_tree)
with self.send_bus:
self.send_bus.send(msg)
logger.info("Result sent")
else:
logger.warning("Problem retrieving task %i" % main_id)
def _sendDiskUsageChangedNotification(self,
path,
disk_usage,
otdb_id=None):
try:
msg = EventMessage(subject='%s.DiskUsageChanged' %
DEFAULT_DM_NOTIFICATION_PREFIX,
content={
'path':
path,
'disk_usage':
disk_usage,
'disk_usage_readable':
humanreadablesize(disk_usage),
'otdb_id':
otdb_id
})
logger.info('Sending notification with subject %s to %s: %s',
msg.subject, self.event_bus.exchange, msg.content)
self.event_bus.send(msg)
except Exception as e:
logger.error(str(e))
def _sendNotification(self, subject, payload, timestampFields=None):
try:
content = json.loads(payload)
if 'new' in content and content[
'new'] and 'old' in content and content['old']:
# check if new and old are equal.
# however, new and old can be based on different views,
# so, only check the values for the keys they have in common
new_keys = set(content['new'].keys())
old_keys = set(content['old'].keys())
common_keys = new_keys & old_keys
equal_valued_keys = [
k for k in common_keys
if content['new'][k] == content['old'][k]
]
if len(equal_valued_keys) == len(common_keys):
logger.info(
'new and old values are equal, not sending notification. %s'
% (content['new']))
return
if timestampFields:
content = self._formatTimestampsAsIso(timestampFields, content)
except Exception as e:
logger.error('Could not parse payload: %s\n%s' % (payload, e))
content = None
try:
msg = EventMessage(context=self.notification_prefix + subject,
content=content)
logger.info('Sending notification %s: %s' %
(subject, str(content).replace('\n', ' ')))
self.event_bus.send(msg)
except Exception as e:
logger.error(str(e))
if not os.path.exists(os.path.dirname(treestatuseventfilename)):
os.mkdirs(os.path.dirname(treestatuseventfilename))
with open(treestatuseventfilename, 'w') as f:
f.write(start_time.strftime("%Y-%m-%d %H:%M:%S"))
except Exception as e:
logger.error(e)
try:
logger.info("start_time=%s, polling database" % (start_time,))
record_list = PollForStatusChanges(start_time, otdb_connection)
for (treeid, state, modtime, creation) in record_list:
content = { "treeID" : treeid, "state" : allowed_states.get(state, "unknown_state"),
"time_of_change" : modtime }
msg = EventMessage(context=DEFAULT_OTDB_NOTIFICATION_SUBJECT, content=content)
logger.info("sending message treeid %s state %s modtime %s" % (treeid, allowed_states.get(state, "unknown_state"), modtime))
send_bus.send(msg)
logger.info("new start_time:=%s" % (creation,))
try:
with open(treestatuseventfilename, 'w') as f:
f.write(creation)
except Exception as e:
logger.error(e)
except FunctionError, exc_info:
logger.error(exc_info)
except Exception as e:
logger.error(e)
def run(self):
with self.event_bus:
while True:
try:
self.__clearFinishedJobs()
with self.__lock:
starting_threads = [
job_thread_dict['thread'] for job_thread_dict in
list(self.__running_jobs.values())
if 'pipeline' not in job_thread_dict
]
pipelines = [
job_thread_dict['pipeline'] for job_thread_dict in
list(self.__running_jobs.values())
if 'pipeline' in job_thread_dict
]
initializing_pipelines = [
pipeline for pipeline in pipelines
if pipeline.status ==
IngestPipeline.STATUS_INITIALIZING
]
transferring_pipelines = [
pipeline for pipeline in pipelines
if pipeline.status ==
IngestPipeline.STATUS_TRANSFERRING
]
finalizing_pipelines = [
pipeline for pipeline in pipelines if
pipeline.status == IngestPipeline.STATUS_FINALIZING
]
finished_pipelines = [
pipeline for pipeline in pipelines if
pipeline.status == IngestPipeline.STATUS_FINISHED
]
log_interval = 5 if self.__running_jobs else 60
if datetime.utcnow(
) - self.__running_jobs_log_timestamp > timedelta(
seconds=log_interval):
status_log_line = "status: running %s jobs: #starting=%d, #transferring=%d, #finalizing=%d, #finished=%d, bandwith used on network interface(s) %s %s (%s), load=%.1f" % (
len(self.__running_jobs),
len(initializing_pipelines) +
len(starting_threads), len(transferring_pipelines),
len(finalizing_pipelines), len(finished_pipelines),
NET_IF_TO_MONITOR,
humanreadablesize(self.__prev_used_bandwidth,
'bps'),
humanreadablesize(self.__prev_used_bandwidth / 8,
'Bps'), os.getloadavg()[0])
logger.info(status_log_line)
self.__running_jobs_log_timestamp = datetime.utcnow()
msg = EventMessage(subject="%s.%s" %
(INGEST_NOTIFICATION_PREFIX,
'TransferServiceStatus'),
content={
'ingest_server':
socket.gethostname(),
'message':
status_log_line
})
msg.ttl = 3600 # remove message from queue's when not picked up within 1 hours
self.event_bus.send(msg)
time.sleep(5)
except KeyboardInterrupt:
break
except Exception as e:
logger.error(e)
def create_service(busname, dbcreds, state_file_path='~/.lofar/otdb_treestatusevent_state'):
alive = True
connected = False
otdb_connection = None
with ToBus(busname) as send_bus:
while alive:
while alive and not connected:
# Connect to the database
try:
otdb_connection = pg.connect(**dbcreds.pg_connect_options())
connected = True
logger.info("Connected to database %s" % (dbcreds,))
# Get list of allowed tree states
allowed_states = {}
for (state_nr, name) in otdb_connection.query("select id,name from treestate").getresult():
allowed_states[state_nr] = name
except (TypeError, SyntaxError, pg.InternalError) as e:
connected = False
logger.error("Not connected to database %s, retry in 5 seconds: %s" % (dbcreds, e))
time.sleep(5)
# When we are connected we can poll the database
if connected:
# Get start_time (= creation time of last retrieved record if any)
try:
treestatuseventfilename = os.path.expanduser(state_file_path)
with open(treestatuseventfilename, 'r') as f:
line = f.readline()
if line.rfind('.') > 0:
start_time = datetime.datetime.strptime(line, "%Y-%m-%d %H:%M:%S.%f")
else:
start_time = datetime.datetime.strptime(line, "%Y-%m-%d %H:%M:%S")
except Exception as e:
logger.warning(e)
# start scanning from events since 'now'
# this timestamp will be stored in the treestatuseventfilename file
start_time = datetime.datetime.utcnow()
try:
logger.info("creating %s" % (treestatuseventfilename,))
if not os.path.exists(os.path.dirname(treestatuseventfilename)):
os.makedirs(os.path.dirname(treestatuseventfilename))
with open(treestatuseventfilename, 'w') as f:
f.write(start_time.strftime("%Y-%m-%d %H:%M:%S"))
except Exception as e:
logger.error(e)
try:
logger.debug("start_time=%s, polling database" % (start_time,))
record_list = PollForStatusChanges(start_time, otdb_connection)
for (treeid, state, modtime, creation) in record_list:
content = { "treeID" : treeid, "state" : allowed_states.get(state, "unknown_state"),
"time_of_change" : modtime }
msg = EventMessage(subject=DEFAULT_OTDB_NOTIFICATION_SUBJECT, content=content)
logger.info("sending message treeid %s state %s modtime %s" % (treeid, allowed_states.get(state, "unknown_state"), modtime))
send_bus.send(msg)
logger.debug("new start_time:=%s" % (creation,))
try:
with open(treestatuseventfilename, 'w') as f:
f.write(creation)
except Exception as e:
logger.error(e)
except FunctionError as exc_info:
logger.error(exc_info)
except Exception as e:
logger.error(e)
# Redetermine the database status.
connected = (otdb_connection and otdb_connection.status == 1)
time.sleep(2)
