def retrieve_instrument(self, instrument_name): """ Retrieve or create an instrument given its name """ # Get or create the instrument object from the DB for instrument in self._instruments: if str(instrument) == instrument_name: return instrument # If we haven't found it, create it. instrument = Instrument(name=instrument_name) instrument.save() self._instruments.append(instrument) return instrument
def on_message(self, headers, message): """ Process a message. @param headers: message headers @param message: JSON-encoded message content """ destination = headers["destination"] try: data_dict = json.loads(message) except: logging.error("Could not decode message from %s" % headers["destination"]) return try: instrument_id = Instrument.objects.get(name=self._instrument) except Instrument.DoesNotExist: instrument_id = Instrument(name=self._instrument) instrument_id.save() for key in data_dict: # If we find a dictionary, process its entries if type(data_dict[key])==dict: # The key is now an entry type and the entry itself # should be another dictionary for instances of that type for item in data_dict[key]: if type(data_dict[key][item])==dict: identifier = None counts = None if "id" in data_dict[key][item]: identifier = data_dict[key][item]["id"] if "counts" in data_dict[key][item]: counts = data_dict[key][item]["counts"] if identifier is not None and counts is not None: parameter_name = "%s_count_%s" % (item, identifier) store_and_cache(instrument_id, parameter_name, counts) else: store_and_cache(instrument_id, key, data_dict[key])
def listen_and_wait(self, waiting_period=1.0): """ Listen for the next message from the brokers. This method will simply return once the connection is terminated. @param waiting_period: sleep time between connection to a broker """ # Get or create the "common" instrument object from the DB. # This dummy instrument is used for heartbeats and central services. try: common_instrument = Instrument.objects.get(name='common') except Instrument.DoesNotExist: common_instrument = Instrument(name='common') common_instrument.save() # Retrieve the Parameter object for our own heartbeat try: pid_key_id = Parameter.objects.get(name="system_dasmon_listener_pid") except: pid_key_id = Parameter(name="system_dasmon_listener_pid") pid_key_id.save() last_purge_time = None last_heartbeat = 0 while True: try: if self._connection is None or self._connection.is_connected() is False: self.connect() if last_purge_time is None or time.time() - last_purge_time > PURGE_DELAY: last_purge_time = time.time() # Remove old entries delta_time = datetime.timedelta(days=PURGE_TIMEOUT) cutoff = timezone.now() - delta_time StatusVariable.objects.filter(timestamp__lte=cutoff).delete() #StatusCache.objects.filter(timestamp__lte=cutoff).delete() # Remove old PVMON entries: first, the float values PV.objects.filter(update_time__lte=time.time() - PURGE_TIMEOUT * 24 * 60 * 60).delete() old_entries = PVCache.objects.filter(update_time__lte=time.time() - PURGE_TIMEOUT * 24 * 60 * 60) for item in old_entries: if len(MonitoredVariable.objects.filter(instrument=item.instrument, pv_name=item.name)) == 0: item.delete() # Remove old PVMON entries: second, the string values PVString.objects.filter(update_time__lte=time.time() - PURGE_TIMEOUT * 24 * 60 * 60).delete() old_entries = PVStringCache.objects.filter(update_time__lte=time.time() - PURGE_TIMEOUT * 24 * 60 * 60) for item in old_entries: if len(MonitoredVariable.objects.filter(instrument=item.instrument, pv_name=item.name)) == 0: item.delete() # Remove old images delta_time = datetime.timedelta(days=IMAGE_PURGE_TIMEOUT) cutoff = timezone.now() - delta_time ReducedImage.objects.filter(created_on__lte=cutoff).delete() time.sleep(waiting_period) try: if time.time() - last_heartbeat > HEARTBEAT_DELAY: last_heartbeat = time.time() store_and_cache(common_instrument, pid_key_id, str(os.getpid())) # Send ping request if hasattr(settings, "PING_TOPIC"): from settings import PING_TOPIC, ACK_TOPIC payload = {"reply_to": ACK_TOPIC, "request_time": time.time()} t0 = time.time() self.send(PING_TOPIC, json.dumps(payload)) t=time.time() - t0 logging.error("Send time: %s", t) process_ack() else: logging.error("settings.PING_TOPIC is not defined") except: logging.error("Problem writing heartbeat %s", sys.exc_value) except: logging.error("Problem connecting to AMQ broker %s", sys.exc_value) time.sleep(5.0)
def add_status_entry(headers, data): """ Populate the reporting database with the contents of a status message of the following format: @param headers: ActiveMQ message header dictionary @param data: JSON encoded message content headers: {'expires': '0', 'timestamp': '1344613053723', 'destination': '/queue/POSTPROCESS.DATA_READY', 'persistent': 'true', 'priority': '5', 'message-id': 'ID:mac83086.ornl.gov-59780-1344536680877-8:2:1:1:1'} The data is a dictionary in a JSON format. data: {"instrument": tokens[2], "ipts": tokens[3], "run_number": run_number, "data_file": message} """ # Find the DB entry for this queue destination = headers["destination"].replace('/queue/','') status_id = StatusQueue.objects.filter(name__startswith=destination) if len(status_id)==0: status_id = StatusQueue(name=destination) status_id.save() else: status_id = status_id[0] # Process the data data_dict = json.loads(data) # Look for instrument instrument = data_dict["instrument"].lower() try: instrument_id = Instrument.objects.get(name=instrument) except Instrument.DoesNotExist: instrument_id = Instrument(name=instrument) instrument_id.save() # Look for IPTS ID ipts = data_dict["ipts"].upper() try: ipts_id = IPTS.objects.get(expt_name=ipts) except IPTS.DoesNotExist: ipts_id = IPTS(expt_name=ipts) ipts_id.save() # Add instrument to IPTS if not already in there try: if IPTS.objects.filter(id=ipts_id.id, instruments__in=[instrument_id]).count()==0: ipts_id.instruments.add(instrument_id) ipts_id.save() except: traceback.print_exc() logging.error(sys.exc_value) # Check whether we already have an entry for this run run_number = data_dict["run_number"] try: run_id = DataRun.objects.get(run_number=run_number, instrument_id=instrument_id) except DataRun.DoesNotExist: logging.info("Creating entry for run %s-%d" % (instrument, run_number)) run_id = DataRun(run_number=run_number, instrument_id=instrument_id, ipts_id=ipts_id, file=data_dict["data_file"]) run_id.save() # Add a workflow summary for this new run try: summary_id = WorkflowSummary.objects.get(run_id=run_id) except WorkflowSummary.DoesNotExist: summary_id = WorkflowSummary(run_id=run_id) summary_id.save() # Create a run status object in the DB run_status = RunStatus(run_id=run_id, queue_id=status_id, message_id=headers["message-id"]) run_status.save() # Create an information entry as necessary # Truncate to max length of DB character field if "information" in data_dict: data = data_dict["information"] mesg = (data[:198] + '..') if len(data) > 200 else data info = Information(run_status_id=run_status, description=mesg) info.save() # Create error entry as necessary if "error" in data_dict: data = data_dict["error"] mesg = (data[:198] + '..') if len(data) > 200 else data error = Error(run_status_id=run_status, description=mesg) error.save() # Update the workflow summary summary_id = WorkflowSummary.objects.get_summary(run_id) summary_id.update()