def __sendNotification(self, subject, message='', **kwargs): try: contentDict = { 'job_id': self.JobId, 'export_id': self.job.get('job_group_id'), 'archive_id': self.ArchiveId, 'project': self.Project, 'type': self.Type, 'ingest_server': self.hostname, 'dataproduct': self.DataProduct, 'srm_url': self.PrimaryUri } if 'ObservationId' in self.job: contentDict['otdb_id'] = self.job['ObservationId'] if self.lta_site: contentDict['lta_site'] = self.lta_site if message: contentDict['message'] = message for k,v in list(kwargs.items()): contentDict[k] = v msg = EventMessage(subject="%s.%s" % (INGEST_NOTIFICATION_PREFIX, subject), content=contentDict) msg.ttl = 48*3600 #remove message from queue's when not picked up within 48 hours logger.info('Sending notification %s: %s' % (subject, str(contentDict).replace('\n', ' '))) self.event_bus.send(msg) except Exception as e: logger.error(str(e))
def testPredecessors(self): """ Request the resources for a simulated obsid 1, with the following predecessor tree: 1 requires 2, 3 2 requires 3 3 requires nothing """ with RATaskSpecified(otdb_notification_busname=self.busname, notification_busname=self.busname) as jts: # Send fake status update with ToBus(self.status_service) as tb: msg = EventMessage( content={ "treeID": 1, "state": "prescheduled", "time_of_change": datetime.datetime(2016, 1, 1), }) tb.send(msg) # Wait for message to arrive self.assertTrue(self.trigger.wait()) # Verify message self.assertEqual(self.trigger.args[0], 1) resourceIndicators = self.trigger.args[2] self.assertIn("1", resourceIndicators) self.assertIn("2", resourceIndicators) self.assertIn("3", resourceIndicators) # Make sure we only requested exactly three parsets self.assertEqual(self.requested_parsets, 3)
def _sendNotification(self, user, IP): msg = EventMessage(subject=TRIGGER_SUBMISSION_NOTIFICATION_SUBJECT, content="Trigger received by " + str(user) + " (IP:" + IP + ")") try: notification_bus.send(msg) except Exception as err: logger.error("Could not send notification ->" + str(err))
def _sendNotification(self, subject, content): try: msg = EventMessage(subject="%s.%s" % (DEFAULT_DM_NOTIFICATION_PREFIX, subject), content=content) logger.info('Sending notification with subject %s to %s: %s', msg.subject, self.exchange, msg.content) self.send(msg) except Exception as e: logger.error(str(e))
def createAndSendSpecifiedTask(self, main_id, status): # Construct root node of tree resultTree = self.get_specification_with_predecessors( main_id, "otdb", status, {}) logger.info("Sending result: %s" % resultTree) # Put result on bus msg = EventMessage(content=resultTree) self.send_bus.send(msg) logger.info("Result sent")
def onObservationPrescheduled(self, treeId, modificationTime): logger.info("Processing obs ID %s", treeId) # Request the parset main_obsID = treeId main_parset, _ = self.parset_rpc(OtdbID=main_obsID) # Construct a dict of all the parsets we retrieved parsets = {} parsets[main_obsID] = main_parset logger.info("Processing predecessors") # Collect the initial set of predecessors request_obsIDs = set(predecessors(main_parset)) logger.info("Processing %s", request_obsIDs) # Iterate recursively over all known predecessor obsIDs, and request their parsets while request_obsIDs: obsID = request_obsIDs.pop() if obsID in parsets: # Predecessor lists can overlap -- we already have this one continue logger.info("Fetching predecessor %s", obsID) # Request predecessor parset parsets[obsID], _ = self.parset_rpc(OtdbID=obsID) # Add the list of predecessors request_obsIDs = request_obsIDs.union(predecessors(parsets[obsID])) # Convert parsets to resource indicators logger.info("Extracting resource indicators") resourceIndicators = dict([(str(obsID), resourceIndicatorsFromParset(parset)) for (obsID, parset) in parsets.iteritems()]) # Construct and send result message logger.info("Sending result") result = { "sasID": main_obsID, "state": "prescheduled", "time_of_change": modificationTime, "resource_indicators": resourceIndicators, } # Put result on bus msg = EventMessage(content=result) self.send_bus.send(msg) logger.info("Result sent")
def _sendNotification(self, subject, contentDict): try: if subject and contentDict: msg = EventMessage(subject="%s.%s" % (DEFAULT_RADB_NOTIFICATION_PREFIX, subject), content=contentDict) logger.info('Sending notification %s to %s: %s', subject, self.event_bus.exchange, str(contentDict).replace('\n', ' ')) self.event_bus.send(msg) except Exception as e: logger.error(str(e))
def TaskSetStatus(self, OtdbID, NewStatus, UpdateTimestamps): print "***** TaskSetStatus(%s,%s) *****" % (OtdbID, NewStatus) # Broadcast the state change content = { "treeID": OtdbID, "state": NewStatus, "time_of_change": datetime.datetime.utcnow() } msg = EventMessage(context=DEFAULT_OTDB_NOTIFICATION_SUBJECT, content=content) self.notification_bus.send(msg) return {'OtdbID': OtdbID, 'MomID': None, 'Success': True}
def _send_notification(user, host, project, trigger_id, metadata): try: content = { "user": user, "host": host, "project": project, "trigger_id": trigger_id, "metadata": metadata } msg = EventMessage(subject=DEFAULT_TRIGGER_NOTIFICATION_SUBJECT, content=content) with ToBus(exchange=DEFAULT_BUSNAME, broker=DEFAULT_BROKER) as notification_bus: notification_bus.send(msg) except Exception as err: logger.error("Could not send notification ->" + str(err))
def TaskSetStatus(self, OtdbID, NewStatus, UpdateTimestamps): logger.info("***** TaskSetStatus(%s,%s) *****", OtdbID, NewStatus) global otdb_status otdb_status[OtdbID] = NewStatus # Broadcast the state change content = { "treeID": OtdbID, "state": NewStatus, "time_of_change": datetime.datetime.utcnow() } msg = EventMessage(subject=DEFAULT_OTDB_NOTIFICATION_SUBJECT, content=content) self.notification_bus.send(msg) return {'OtdbID': OtdbID, 'MomID': None, 'Success': True}
def createAndSendSpecifiedTask(self, main_id, status): spec = Specification(self.otdbrpc, self.momrpc, self.radbrpc) spec.status = status spec.read_from_OTDB_with_predecessors(main_id, "otdb", {}) spec.read_from_mom() spec.update_start_end_times() # spec.insert_into_radb() is still done in resource_assigner for now. result_tree = spec.as_dict() if spec.status == status: logger.info("Sending result: %s" % result_tree) # Put result on bus msg = EventMessage(subject=DEFAULT_RA_TASK_SPECIFIED_NOTIFICATION_SUBJECT, content=result_tree) with self.send_bus: self.send_bus.send(msg) logger.info("Result sent") else: logger.warning("Problem retrieving task %i" % main_id)
def _sendDiskUsageChangedNotification(self, path, disk_usage, otdb_id=None): try: msg = EventMessage(subject='%s.DiskUsageChanged' % DEFAULT_DM_NOTIFICATION_PREFIX, content={ 'path': path, 'disk_usage': disk_usage, 'disk_usage_readable': humanreadablesize(disk_usage), 'otdb_id': otdb_id }) logger.info('Sending notification with subject %s to %s: %s', msg.subject, self.event_bus.exchange, msg.content) self.event_bus.send(msg) except Exception as e: logger.error(str(e))
def _sendNotification(self, subject, payload, timestampFields=None): try: content = json.loads(payload) if 'new' in content and content[ 'new'] and 'old' in content and content['old']: # check if new and old are equal. # however, new and old can be based on different views, # so, only check the values for the keys they have in common new_keys = set(content['new'].keys()) old_keys = set(content['old'].keys()) common_keys = new_keys & old_keys equal_valued_keys = [ k for k in common_keys if content['new'][k] == content['old'][k] ] if len(equal_valued_keys) == len(common_keys): logger.info( 'new and old values are equal, not sending notification. %s' % (content['new'])) return if timestampFields: content = self._formatTimestampsAsIso(timestampFields, content) except Exception as e: logger.error('Could not parse payload: %s\n%s' % (payload, e)) content = None try: msg = EventMessage(context=self.notification_prefix + subject, content=content) logger.info('Sending notification %s: %s' % (subject, str(content).replace('\n', ' '))) self.event_bus.send(msg) except Exception as e: logger.error(str(e))
if not os.path.exists(os.path.dirname(treestatuseventfilename)): os.mkdirs(os.path.dirname(treestatuseventfilename)) with open(treestatuseventfilename, 'w') as f: f.write(start_time.strftime("%Y-%m-%d %H:%M:%S")) except Exception as e: logger.error(e) try: logger.info("start_time=%s, polling database" % (start_time,)) record_list = PollForStatusChanges(start_time, otdb_connection) for (treeid, state, modtime, creation) in record_list: content = { "treeID" : treeid, "state" : allowed_states.get(state, "unknown_state"), "time_of_change" : modtime } msg = EventMessage(context=DEFAULT_OTDB_NOTIFICATION_SUBJECT, content=content) logger.info("sending message treeid %s state %s modtime %s" % (treeid, allowed_states.get(state, "unknown_state"), modtime)) send_bus.send(msg) logger.info("new start_time:=%s" % (creation,)) try: with open(treestatuseventfilename, 'w') as f: f.write(creation) except Exception as e: logger.error(e) except FunctionError, exc_info: logger.error(exc_info) except Exception as e: logger.error(e)
def run(self): with self.event_bus: while True: try: self.__clearFinishedJobs() with self.__lock: starting_threads = [ job_thread_dict['thread'] for job_thread_dict in list(self.__running_jobs.values()) if 'pipeline' not in job_thread_dict ] pipelines = [ job_thread_dict['pipeline'] for job_thread_dict in list(self.__running_jobs.values()) if 'pipeline' in job_thread_dict ] initializing_pipelines = [ pipeline for pipeline in pipelines if pipeline.status == IngestPipeline.STATUS_INITIALIZING ] transferring_pipelines = [ pipeline for pipeline in pipelines if pipeline.status == IngestPipeline.STATUS_TRANSFERRING ] finalizing_pipelines = [ pipeline for pipeline in pipelines if pipeline.status == IngestPipeline.STATUS_FINALIZING ] finished_pipelines = [ pipeline for pipeline in pipelines if pipeline.status == IngestPipeline.STATUS_FINISHED ] log_interval = 5 if self.__running_jobs else 60 if datetime.utcnow( ) - self.__running_jobs_log_timestamp > timedelta( seconds=log_interval): status_log_line = "status: running %s jobs: #starting=%d, #transferring=%d, #finalizing=%d, #finished=%d, bandwith used on network interface(s) %s %s (%s), load=%.1f" % ( len(self.__running_jobs), len(initializing_pipelines) + len(starting_threads), len(transferring_pipelines), len(finalizing_pipelines), len(finished_pipelines), NET_IF_TO_MONITOR, humanreadablesize(self.__prev_used_bandwidth, 'bps'), humanreadablesize(self.__prev_used_bandwidth / 8, 'Bps'), os.getloadavg()[0]) logger.info(status_log_line) self.__running_jobs_log_timestamp = datetime.utcnow() msg = EventMessage(subject="%s.%s" % (INGEST_NOTIFICATION_PREFIX, 'TransferServiceStatus'), content={ 'ingest_server': socket.gethostname(), 'message': status_log_line }) msg.ttl = 3600 # remove message from queue's when not picked up within 1 hours self.event_bus.send(msg) time.sleep(5) except KeyboardInterrupt: break except Exception as e: logger.error(e)
def create_service(busname, dbcreds, state_file_path='~/.lofar/otdb_treestatusevent_state'): alive = True connected = False otdb_connection = None with ToBus(busname) as send_bus: while alive: while alive and not connected: # Connect to the database try: otdb_connection = pg.connect(**dbcreds.pg_connect_options()) connected = True logger.info("Connected to database %s" % (dbcreds,)) # Get list of allowed tree states allowed_states = {} for (state_nr, name) in otdb_connection.query("select id,name from treestate").getresult(): allowed_states[state_nr] = name except (TypeError, SyntaxError, pg.InternalError) as e: connected = False logger.error("Not connected to database %s, retry in 5 seconds: %s" % (dbcreds, e)) time.sleep(5) # When we are connected we can poll the database if connected: # Get start_time (= creation time of last retrieved record if any) try: treestatuseventfilename = os.path.expanduser(state_file_path) with open(treestatuseventfilename, 'r') as f: line = f.readline() if line.rfind('.') > 0: start_time = datetime.datetime.strptime(line, "%Y-%m-%d %H:%M:%S.%f") else: start_time = datetime.datetime.strptime(line, "%Y-%m-%d %H:%M:%S") except Exception as e: logger.warning(e) # start scanning from events since 'now' # this timestamp will be stored in the treestatuseventfilename file start_time = datetime.datetime.utcnow() try: logger.info("creating %s" % (treestatuseventfilename,)) if not os.path.exists(os.path.dirname(treestatuseventfilename)): os.makedirs(os.path.dirname(treestatuseventfilename)) with open(treestatuseventfilename, 'w') as f: f.write(start_time.strftime("%Y-%m-%d %H:%M:%S")) except Exception as e: logger.error(e) try: logger.debug("start_time=%s, polling database" % (start_time,)) record_list = PollForStatusChanges(start_time, otdb_connection) for (treeid, state, modtime, creation) in record_list: content = { "treeID" : treeid, "state" : allowed_states.get(state, "unknown_state"), "time_of_change" : modtime } msg = EventMessage(subject=DEFAULT_OTDB_NOTIFICATION_SUBJECT, content=content) logger.info("sending message treeid %s state %s modtime %s" % (treeid, allowed_states.get(state, "unknown_state"), modtime)) send_bus.send(msg) logger.debug("new start_time:=%s" % (creation,)) try: with open(treestatuseventfilename, 'w') as f: f.write(creation) except Exception as e: logger.error(e) except FunctionError as exc_info: logger.error(exc_info) except Exception as e: logger.error(e) # Redetermine the database status. connected = (otdb_connection and otdb_connection.status == 1) time.sleep(2)