def __rule_1_after(msg):
""" after the reply from CRCL is back check if the incident is spam """
try:
report_id = msg['body']['reportID']
report_type = shared.processed_mgs[report_id]['inc'][
'incidentType']
precipitation = msg['body']['precipitation']
except (KeyError, TypeError, ValueError, IndexError) as e:
logger.info(
"Cannot load reportID / report type/ precipitation from processed messages"
)
logger.debug(str(type(e)) + str(e))
return
logger.info("Validating 803 with type: " + str(report_type) +
" and precipitation: " + str(round(precipitation, 2)) +
" ID: " + str(report_id))
# type == Precipitation / Heavy Precipitation + precipitation == 0 ==> SPAM
if precipitation < .1 and (report_type == 'Precipitation'
or report_type == 'HeavyPrecipitation'
or report_type == 'Heavy Precipitation'
or report_type == "Blizzard"):
spam = True
else:
spam = False
return spam
def get_message():
try:
return MessageQueue.__queue.get(block=True, timeout=5)
# Note that the command blocks the execution of the threat (block=True)
# so the listener should be on a different thread (and have a chance to add something in there)
except queue.Empty:
logger.debug("message_queue is empty!")
return None
def process_messages(self):
while True:
msg = MessageQueue.get_message()
if msg is None:
logger.debug("No more messages, thread ends")
return
self.__validator.validate(msg)
def put_message(msg):
try:
MessageQueue.__queue.put(item=msg, block=True, timeout=5)
# logger.debug("Message put in queue: " + str(msg))
return True
except queue.Full as e:
logger.debug("message_queue error. Could not put message in queue. (Full)")
logger.debug(e)
return False
def message_to_queue(message):
try:
message = json.loads(message)
except json.decoder.JSONDecodeError as e:
logger.warning("message from bus is not a valid json: " + str(e))
logger.debug(message)
return
message_queue.MessageQueue.put_message(message)
logger.info("Message arrived from bus and inserted in queue")
logger.debug(json.dumps(message))
ValidatorThreadHandler.init_validator()
def __load_dummy_messages():
""" Load Vicenza messages and add them to the message queue with a small delay between each insertion """
import random
import filter_messages
max_delay = 0.01 # delay in the range [0, max_delay] from uniform distribution
vic_messages = filter_messages.simulateData()
for m in vic_messages:
logger.debug("writing TOP101 message to queue")
message_queue.MessageQueue.put_message(m) # Note: pass it by value, not reference!
ValidatorThreadHandler.init_validator()
time.sleep(random.random() * max_delay)
def validate_803(msg):
""" find the incident in the local storage and continue validation according to the info from TOP803"""
logger.info("Message TOP803 is processed.")
logger.debug("TOP803 message: " + str(msg))
if msg['body']['reportID'] not in shared.processed_mgs:
logger.warning(
"Message TOP803 does not correspond to a stored report. ID: " +
str(msg['body']['reportID']))
return
Validator.__incident_spam(msg['body']['reportID'],
Validator.__rule_1_after(msg))
def __load_TOP030():
""" load TOP030 messages from local file and put them in the message queue """
import random
import filter_messages
max_delay = .1 # delay in the range [0, max_delay] from uniform distribution
vic_messages = filter_messages.get030()
for m in vic_messages:
logger.debug("writing TOP030 message to queue")
message_queue.MessageQueue.put_message(
m) # Note: pass it by value, not reference!
ValidatorThreadHandler.init_validator()
time.sleep(random.random() * max_delay)
ValidatorThreadHandler.join_validation_thread()
def send(self, topic, message):
logger.info("Sending: " + str(topic))
logger.debug("Sending: " + str(topic) + ": " + str(message))
# return
# Produce and flush message to bus
try:
self.producer.produce(topic, message.encode('utf-8'), 'key', -1,
self.on_delivery)
self.producer.flush()
except Exception as err:
print('Sending data failed')
print(err)
return False
return True
def generate_TOP801(reportID, spam):
try:
header = shared.processed_mgs[reportID]['header']
except (KeyError, TypeError, ValueError, IndexError) as e:
logger.error("header cannot be found in processed messages")
logger.debug(str(type(e)) + str(e))
return None
msg = dict()
msg['header'] = Validator.generate_header(header,
topic_name=shared.TOP801)
msg['body'] = dict()
msg['body']['incidentID'] = reportID
msg['body']['spam'] = spam
return msg
def _preprocess_image(img_data: dict, local_file: str) -> Image:
try:
image = Image.new_from_file(local_file, access='sequential')
max_height = config.DEFAULT_MAX_HEIGHT
max_width = config.DEFAULT_MAX_WIDTH
if img_data["copyrightStatus"] and \
img_data["copyrightStatus"].lower() == "copyright":
max_height = config.COPYRIGHT_MAX_HEIGHT
max_width = config.COPYRIGHT_MAX_WIDTH
if image.height > max_height or image.width > max_width:
if image.height >= image.width:
shrink_by = image.height / max_height
else:
shrink_by = image.width / max_width
logger.debug(f"Resizing {img_data['id']} by {shrink_by}")
image = image.shrink(shrink_by, shrink_by)
except Error as pye:
image = None
Statistic.vips_err(img_data)
logger.error(f"VIPs error - {pye.message}")
return image
def validate(message):
""" read the message and determine if it is TOP 030 or TOP 803 and invoke the corresponding method
"""
print("Thread id in validator: " + str(threading.get_ident()))
try:
inc_topic = message['header']['topicName']
except (KeyError, TypeError, ValueError, IndexError) as e:
logger.warning("could not read topicName from message. Do nothing")
logger.debug(e)
logger.debug(message)
return
logger.info("Message is now processed. TOPIC: " + str(inc_topic))
if inc_topic == 'TOP030_REPORT_REQUESTED':
Validator.validate_TOP030(message)
elif inc_topic == 'TOP803_WEATHER_REPORT':
Validator.validate_803(message)
else:
logger.warning(
"Message read in validator is not TOP030 nor TOP803")
def listen(self, performed_action, topics=None):
# Topics should be a list of topic names e.g. ['topic1', 'topic2']
if topics is None:
topics = self.default_topics
self.listening = True
# Subscribe to topics
try:
self.consumer.subscribe(topics)
except Exception as e:
logger.error("Error @ BusConsumer.listen()")
logger.debug(str(type(e)) + str(e))
return False
logger.info("listener subscribed successfully to topics:" +
str(topics))
# Initiate a loop for continuous listening
while self.listening:
msg = self.consumer.poll(0)
# If a message is received and it is not an error message
if msg is not None and msg.error() is None:
# Add incoming message to requests database
try:
message_text = msg.value().decode('utf-8')
except:
message_text = msg.value()
performed_action(message_text)
# TODO: check if it works ok with the sleep .5
time.sleep(0.5)
# Unsubscribe and close consumer
self.consumer.unsubscribe()
self.consumer.close()
def generate_header(header, topic_name, action_type=None):
""" generate a dict corresponding to header with correct values and return it
Read the immutable values from the parameter header
Generate the values that are specific
Return the header dict
If something goes wrong, then return None
"""
try:
header['topicName'] = topic_name
if action_type is not None:
header['actionType'] = action_type
header['sentUTC'] = Validator.time_UTC()
header['sender'] = 'VAL'
header['msgIdentifier'] = 'VAL_' + str(uuid.uuid4()) + "_" + str(
time.time())
except (KeyError, TypeError, ValueError, IndexError) as e:
logger.error(
"validator.Validator.generate_header: header does not have the correct fields"
)
logger.debug(str(type(e)) + str(e))
return None
return header
def validate_TOP030(message):
"""
δες αν εχει ξαναέρθει msg με το incident id που ηρθε τώρα (είτε τοπικά είτε σε ένα csv).
Αν έχει ξαναέρθει, τότε μην κάνεις τίποτα.
Αν δεν έχει ξαναέρθει, τότε δες αν το msg έχει attachment και incident type.
Αν δεν έχει, τότε μην κάνεις τίποτα
Αν έχει (incident type)
Βαλε το στην λίστα των incidents που έχουν επεξεργαστεί (έχουν περάσει ή περνανε τώρα validation)
Δες αν το incident type == Heatwave (?)
Αν οχι, τότε βάλε spam == false
Αν ναι, τότε ρώτα το CRCL για να σου πει τις καιρικές συνθήκες στην περιοχή του incident
"""
report_id = None
report_type = None
inc_long = None
inc_lat = None
report_time = None
report_spam = None
# print(message['body']['incidents'])
logger.debug("Processed TOP030 message: " + str(message))
header = message['header']
try:
inc_long = float(message['body']['position']['long'])
inc_lat = float(message['body']['position']['lat'])
except (KeyError, TypeError, ValueError, IndexError) as e:
logger.info(
"Incoming message does not have location, validation will stop."
)
logger.debug(str(type(e)) + str(e))
logger.debug(message)
return
try:
incidents = message['body']['incidents']
except (KeyError, TypeError) as e:
logger.info("No reports in TOP030, validation will stop.")
logger.debug(str(type(e)) + str(e))
logger.debug(str(message))
return
if len(incidents) == 0:
logger.info("No incidents in TOP030.")
for inc in incidents:
try:
report_id = inc['reportId']
report_type = inc['incidentType']
report_time = inc['timestamp']
except (KeyError, TypeError, ValueError, IndexError) as e:
logger.warning(
"Incident does not have report ID / incident Type / timestamp"
)
logger.debug(str(type(e)) + str(e))
return
if report_id in shared.processed_mgs:
logger.debug("Report already processed. ReportId: " +
str(report_id))
continue
shared.processed_mgs[report_id] = {'inc': inc, 'header': header}
# TODO: check if spam field is already there, and if is spam=True/False stop validation (not None)
logger.info("Report is checked to determine if it is spam. ID:" +
str(report_id))
if Validator.__rule_1_pre(report_type) is True:
logger.info("Asking CRCL for report with ID:" + str(report_id))
t_802 = Validator.generate_TOP802(message, report_id, inc_long,
inc_lat, report_time)
Validator.bus_prod.send(topic=t_802['header']['topicName'],
message=json.dumps(t_802))
else:
Validator.__incident_spam(report_id, False)
