def __rule_1_after(msg): """ after the reply from CRCL is back check if the incident is spam """ try: report_id = msg['body']['reportID'] report_type = shared.processed_mgs[report_id]['inc'][ 'incidentType'] precipitation = msg['body']['precipitation'] except (KeyError, TypeError, ValueError, IndexError) as e: logger.info( "Cannot load reportID / report type/ precipitation from processed messages" ) logger.debug(str(type(e)) + str(e)) return logger.info("Validating 803 with type: " + str(report_type) + " and precipitation: " + str(round(precipitation, 2)) + " ID: " + str(report_id)) # type == Precipitation / Heavy Precipitation + precipitation == 0 ==> SPAM if precipitation < .1 and (report_type == 'Precipitation' or report_type == 'HeavyPrecipitation' or report_type == 'Heavy Precipitation' or report_type == "Blizzard"): spam = True else: spam = False return spam
def get_message(): try: return MessageQueue.__queue.get(block=True, timeout=5) # Note that the command blocks the execution of the threat (block=True) # so the listener should be on a different thread (and have a chance to add something in there) except queue.Empty: logger.debug("message_queue is empty!") return None
def process_messages(self): while True: msg = MessageQueue.get_message() if msg is None: logger.debug("No more messages, thread ends") return self.__validator.validate(msg)
def put_message(msg): try: MessageQueue.__queue.put(item=msg, block=True, timeout=5) # logger.debug("Message put in queue: " + str(msg)) return True except queue.Full as e: logger.debug("message_queue error. Could not put message in queue. (Full)") logger.debug(e) return False
def message_to_queue(message): try: message = json.loads(message) except json.decoder.JSONDecodeError as e: logger.warning("message from bus is not a valid json: " + str(e)) logger.debug(message) return message_queue.MessageQueue.put_message(message) logger.info("Message arrived from bus and inserted in queue") logger.debug(json.dumps(message)) ValidatorThreadHandler.init_validator()
def __load_dummy_messages(): """ Load Vicenza messages and add them to the message queue with a small delay between each insertion """ import random import filter_messages max_delay = 0.01 # delay in the range [0, max_delay] from uniform distribution vic_messages = filter_messages.simulateData() for m in vic_messages: logger.debug("writing TOP101 message to queue") message_queue.MessageQueue.put_message(m) # Note: pass it by value, not reference! ValidatorThreadHandler.init_validator() time.sleep(random.random() * max_delay)
def validate_803(msg): """ find the incident in the local storage and continue validation according to the info from TOP803""" logger.info("Message TOP803 is processed.") logger.debug("TOP803 message: " + str(msg)) if msg['body']['reportID'] not in shared.processed_mgs: logger.warning( "Message TOP803 does not correspond to a stored report. ID: " + str(msg['body']['reportID'])) return Validator.__incident_spam(msg['body']['reportID'], Validator.__rule_1_after(msg))
def __load_TOP030(): """ load TOP030 messages from local file and put them in the message queue """ import random import filter_messages max_delay = .1 # delay in the range [0, max_delay] from uniform distribution vic_messages = filter_messages.get030() for m in vic_messages: logger.debug("writing TOP030 message to queue") message_queue.MessageQueue.put_message( m) # Note: pass it by value, not reference! ValidatorThreadHandler.init_validator() time.sleep(random.random() * max_delay) ValidatorThreadHandler.join_validation_thread()
def send(self, topic, message): logger.info("Sending: " + str(topic)) logger.debug("Sending: " + str(topic) + ": " + str(message)) # return # Produce and flush message to bus try: self.producer.produce(topic, message.encode('utf-8'), 'key', -1, self.on_delivery) self.producer.flush() except Exception as err: print('Sending data failed') print(err) return False return True
def generate_TOP801(reportID, spam): try: header = shared.processed_mgs[reportID]['header'] except (KeyError, TypeError, ValueError, IndexError) as e: logger.error("header cannot be found in processed messages") logger.debug(str(type(e)) + str(e)) return None msg = dict() msg['header'] = Validator.generate_header(header, topic_name=shared.TOP801) msg['body'] = dict() msg['body']['incidentID'] = reportID msg['body']['spam'] = spam return msg
def _preprocess_image(img_data: dict, local_file: str) -> Image: try: image = Image.new_from_file(local_file, access='sequential') max_height = config.DEFAULT_MAX_HEIGHT max_width = config.DEFAULT_MAX_WIDTH if img_data["copyrightStatus"] and \ img_data["copyrightStatus"].lower() == "copyright": max_height = config.COPYRIGHT_MAX_HEIGHT max_width = config.COPYRIGHT_MAX_WIDTH if image.height > max_height or image.width > max_width: if image.height >= image.width: shrink_by = image.height / max_height else: shrink_by = image.width / max_width logger.debug(f"Resizing {img_data['id']} by {shrink_by}") image = image.shrink(shrink_by, shrink_by) except Error as pye: image = None Statistic.vips_err(img_data) logger.error(f"VIPs error - {pye.message}") return image
def validate(message): """ read the message and determine if it is TOP 030 or TOP 803 and invoke the corresponding method """ print("Thread id in validator: " + str(threading.get_ident())) try: inc_topic = message['header']['topicName'] except (KeyError, TypeError, ValueError, IndexError) as e: logger.warning("could not read topicName from message. Do nothing") logger.debug(e) logger.debug(message) return logger.info("Message is now processed. TOPIC: " + str(inc_topic)) if inc_topic == 'TOP030_REPORT_REQUESTED': Validator.validate_TOP030(message) elif inc_topic == 'TOP803_WEATHER_REPORT': Validator.validate_803(message) else: logger.warning( "Message read in validator is not TOP030 nor TOP803")
def listen(self, performed_action, topics=None): # Topics should be a list of topic names e.g. ['topic1', 'topic2'] if topics is None: topics = self.default_topics self.listening = True # Subscribe to topics try: self.consumer.subscribe(topics) except Exception as e: logger.error("Error @ BusConsumer.listen()") logger.debug(str(type(e)) + str(e)) return False logger.info("listener subscribed successfully to topics:" + str(topics)) # Initiate a loop for continuous listening while self.listening: msg = self.consumer.poll(0) # If a message is received and it is not an error message if msg is not None and msg.error() is None: # Add incoming message to requests database try: message_text = msg.value().decode('utf-8') except: message_text = msg.value() performed_action(message_text) # TODO: check if it works ok with the sleep .5 time.sleep(0.5) # Unsubscribe and close consumer self.consumer.unsubscribe() self.consumer.close()
def generate_header(header, topic_name, action_type=None): """ generate a dict corresponding to header with correct values and return it Read the immutable values from the parameter header Generate the values that are specific Return the header dict If something goes wrong, then return None """ try: header['topicName'] = topic_name if action_type is not None: header['actionType'] = action_type header['sentUTC'] = Validator.time_UTC() header['sender'] = 'VAL' header['msgIdentifier'] = 'VAL_' + str(uuid.uuid4()) + "_" + str( time.time()) except (KeyError, TypeError, ValueError, IndexError) as e: logger.error( "validator.Validator.generate_header: header does not have the correct fields" ) logger.debug(str(type(e)) + str(e)) return None return header
def validate_TOP030(message): """ δες αν εχει ξαναέρθει msg με το incident id που ηρθε τώρα (είτε τοπικά είτε σε ένα csv). Αν έχει ξαναέρθει, τότε μην κάνεις τίποτα. Αν δεν έχει ξαναέρθει, τότε δες αν το msg έχει attachment και incident type. Αν δεν έχει, τότε μην κάνεις τίποτα Αν έχει (incident type) Βαλε το στην λίστα των incidents που έχουν επεξεργαστεί (έχουν περάσει ή περνανε τώρα validation) Δες αν το incident type == Heatwave (?) Αν οχι, τότε βάλε spam == false Αν ναι, τότε ρώτα το CRCL για να σου πει τις καιρικές συνθήκες στην περιοχή του incident """ report_id = None report_type = None inc_long = None inc_lat = None report_time = None report_spam = None # print(message['body']['incidents']) logger.debug("Processed TOP030 message: " + str(message)) header = message['header'] try: inc_long = float(message['body']['position']['long']) inc_lat = float(message['body']['position']['lat']) except (KeyError, TypeError, ValueError, IndexError) as e: logger.info( "Incoming message does not have location, validation will stop." ) logger.debug(str(type(e)) + str(e)) logger.debug(message) return try: incidents = message['body']['incidents'] except (KeyError, TypeError) as e: logger.info("No reports in TOP030, validation will stop.") logger.debug(str(type(e)) + str(e)) logger.debug(str(message)) return if len(incidents) == 0: logger.info("No incidents in TOP030.") for inc in incidents: try: report_id = inc['reportId'] report_type = inc['incidentType'] report_time = inc['timestamp'] except (KeyError, TypeError, ValueError, IndexError) as e: logger.warning( "Incident does not have report ID / incident Type / timestamp" ) logger.debug(str(type(e)) + str(e)) return if report_id in shared.processed_mgs: logger.debug("Report already processed. ReportId: " + str(report_id)) continue shared.processed_mgs[report_id] = {'inc': inc, 'header': header} # TODO: check if spam field is already there, and if is spam=True/False stop validation (not None) logger.info("Report is checked to determine if it is spam. ID:" + str(report_id)) if Validator.__rule_1_pre(report_type) is True: logger.info("Asking CRCL for report with ID:" + str(report_id)) t_802 = Validator.generate_TOP802(message, report_id, inc_long, inc_lat, report_time) Validator.bus_prod.send(topic=t_802['header']['topicName'], message=json.dumps(t_802)) else: Validator.__incident_spam(report_id, False)