def append_datastreamadapter(
datastream_adapter: DatastreamTopicAdapter,
flag_immediate_subscription: bool = False) -> bool:
try:
"""Request Append datastream information append for topic translation subscription
Method called to append information of datastream useful for following MQTT topics subscription
Args:
datastream_adapter (DatastreamTopicAdapter): it includes associated topic and relative metadata to specific datastream
flag_immediate_subscription (bool): Flag if an immediate subscription is required (used for asynchronous dynamic topic subscription
Returns:
bool: Result of operation
"""
if not datastream_adapter:
return False
if datastream_adapter.topic_associated in ServiceObservationClient.dictionary_datastreams:
return True
if LIST_DEVICE_TO_EXCLUDE and (datastream_adapter.tagId
in LIST_DEVICE_TO_EXCLUDE):
logger.info(
'ServiceObservationClient Exclude Device From Topic List: {0}'
.format(datastream_adapter.name_complete))
return False
single_tuple = (datastream_adapter.topic_associated, 0)
if single_tuple in ServiceObservationClient.list_topics:
return True
ServiceObservationClient.append_topic_list(
topic=datastream_adapter.topic_associated)
ServiceObservationClient.dictionary_datastreams[
datastream_adapter.topic_associated] = datastream_adapter
CachedComponents.increase_counter_datastreams_registered(
datastream_feature=datastream_adapter.label_matching,
increase=1)
if not flag_immediate_subscription:
return True
logger.info(
'ServiceObservationClient Immediate Topic Registration: {}'.
format(datastream_adapter.topic_associated))
ServiceObservationClient.get_mqtt_client().subscribe(
(datastream_adapter.topic_associated, 0))
return True
except Exception as ex:
logger.error(
'ServiceObservationClient append_datastreamadapter Exception: {}'
.format(ex))
return False
def received_notification_new_observation(
observable: ObservableGeneric) -> bool:
try:
CachedComponents.increase_counter_observable(increase=1)
if not LOCAL_CONFIG[
LocConfLbls.
LABEL_ENABLE_IMMEDIATEELABORATION_FEEDBYNUMBEROBS]:
return False
if OutputMessageType.OUTPUT_MESSAGE_TYPE_QUEUEDETECTIONALERT in \
LOCAL_CONFIG[LocConfLbls.LABEL_OUTPUT_MESSAGELIST_SELECTED] and \
observable.get_type_observable() == LabelObservationType.LABEL_OBSTYPE_CROWDDENSITY:
logger.info(
'WorkerTask Called request_launch_task_elaboration')
WorkerTasks.request_launch_task_elaboration()
return True
# FIXME: Missing check for request launch task elaboration
elif OutputMessageType.OUTPUT_MESSAGE_TYPE_CROWDHEATMAPOUTPUT in \
LOCAL_CONFIG[LocConfLbls.LABEL_OUTPUT_MESSAGELIST_SELECTED] and \
observable.get_type_observable() == LabelObservationType.LABEL_OBSTYPE_LOCALIZATION:
CachedComponents.increase_counter_observable_localization_unprocessed(
increase=1)
counter_wb_registered = CachedComponents.get_counter_datastreams_registered(
datastream_feature=LabelObservationType.
LABEL_OBSTYPE_LOCALIZATION)
counter_unprocessed = CachedComponents.get_counter_observable_localization_unprocessed(
)
if counter_unprocessed < counter_wb_registered:
return False
CachedComponents.set_counter_observable_localization_unprocessed(
value=0)
# max_iot_id = CachedComponents.get_maxiotid(observable_type=observable.get_type_observable())
#
# if max_iot_id is None:
# return False
#
# if max_iot_id > observable.get_datastream_id():
# return False
logger.info(
'WorkerTask Called request_launch_task_elaboration, counter_wb_registered: {0}, counterobsunproc: {1}'
.format(counter_wb_registered, counter_unprocessed))
WorkerTasks.request_launch_task_elaboration()
return True
return False
except Exception as ex:
logger.error(
'WorkerTask received_notification_new_observation Exception: {}'
.format(ex))
return False
def add_device_registration(
device_registration: DeviceRegistration) -> bool:
try:
if not device_registration:
return False
CachedComponents.checkandset_maxiotid(
observable_type=device_registration.get_obs_type(),
iot_id=device_registration.get_datastream_id())
return CacheRedisAdapter.dictionary_update_value(
label_info=UtilityCatalogCached.
LABEL_DICTIONARY_DEVICE_REGISTRATION,
key=device_registration.get_datastream_id(),
value=device_registration)
except Exception as ex:
logger.error(
'UtilityCatalogCached add_device_registration Exception: {}'.
format(ex))
return False
def adjust_startup_data():
try:
UtilityDatabase.purge_db_connections()
UtilityDatabase.update_database_startup()
running_id = UtilityDatabase.update_get_sw_running_info(
LOCAL_CONFIG[LocConfLbls.LABEL_SW_RELEASE_VERSION])
logger.info(
'UtilityStartupApplication Running ID: {0} SW_VERSION: {1}'.
format(str(running_id),
LOCAL_CONFIG[LocConfLbls.LABEL_SW_RELEASE_VERSION]))
CachedComponents.initialize_components(running_id=running_id)
ServiceObservationClient.set_running_id(running_id=running_id)
except Exception as ex:
logger.error(
'UtilityStartupApplication adjust_startup_data Exception: {}'.
format(ex))
def on_subscribe(client: mqtt.Client, userdata, mid, granted_qos):
try:
ServiceObservationClient.set_flag_subscribed(1)
CachedComponents.set_startupapplication_completed()
if not granted_qos:
logger.error(
'ServiceObservationClient on_subscribe event raised No Topics'
)
return
ServiceObservationClient.counter_number_topics = len(granted_qos)
logger.info(
'ServiceObservationClient on_subscribe event raised, flag_subscribed={0}, TopicSubscribedNumber: {1}'
.format(str(ServiceObservationClient.get_flag_subscribed()),
str(len(granted_qos))))
except Exception as ex:
logger.error(
'ServiceObservationClient onSubscribe Exception: {}'.format(
ex))
def set_correct_obs_iotid(
catalog_observations: Dict[str, List[ObservableGeneric]],
gost_url: str) -> Dict[str, List[ObservableGeneric]]:
if not catalog_observations:
return None
for type_obs in catalog_observations.keys():
observables = catalog_observations[type_obs]
if not observables:
continue
for observable in observables:
iot_id = ServiceGetObservationGOST.get_obs_id_observable(
gost_main_url=gost_url, observable=observable)
if iot_id == 0:
continue
CachedComponents.set_last_observation_id(obs_id=iot_id)
observable.obs_iot_id = ServiceGetObservationGOST.get_obs_id_observable(
gost_main_url=gost_url, observable=observable)
return catalog_observations
def get_servicecatalog_connection_info(message_output_type: OutputMessageType,
username: str = str(),
password: str = str()) -> MQTTConnectionParams:
try:
broker_connection_info = CachedComponents.get_brokeroutput_servicecatalog(output_message_type=message_output_type)
if not broker_connection_info:
return None
return broker_connection_info.to_mqtt_connection_params(username=username,
password=password)
except Exception as ex:
logger.error('get_servicecatalog_connection_info Exception: {}'.format(ex))
return None
def get_list_messages(outmessagetype: OutputMessageType) -> List[OutputMessage]:
try:
list_outputs_id = CachedComponents.get_list_outputmessage_ids(outmessagetype=outmessagetype)
if not list_outputs_id:
return None
list_output_messages = UtilityDatabase.get_list_messageoutput_byids(list_ids=list_outputs_id,
outputmessagetype=outmessagetype)
return list_output_messages
except Exception as ex:
logger.error('get_list_messages Exception: {}'.format(ex))
return None
def update_observables_timestamp(
catalog_observable: Dict[str, List[ObservableGeneric]],
dictionary_obs_time: Dict[str, datetime.datetime]) -> bool:
try:
if not catalog_observable:
return False
counter_changes = 0
for type_observable in catalog_observable:
list_observable = catalog_observable[type_observable]
if not list_observable:
continue
for observable in list_observable:
if not observable:
continue
dictionary_obs_time[observable.get_label_cache(
)] = observable.get_timestamp()
counter_changes += 1
if counter_changes == 0:
return False
logger.info(
'UtilityCatalogCached Updated Dictionary ObsTimestamps {} Elements Changed'
.format(counter_changes))
return CachedComponents.update_dictionary_observable_timestamps(
dictionary_obs_timestamp=dictionary_obs_time)
except Exception as ex:
logger.error(
'UtilityCatalogCached Updated Dictionary ObsTimestamp Exception: {}'
.format(ex))
return False
def get_observations_catalog(catalog_datastreams: Dict[str, List[DatastreamTopicAdapter]]) \
-> Dict[str, List[ObservableGeneric]]:
try:
if not catalog_datastreams:
logger.warning(
'UtilityCatalogCached CatalogDatastream is Empty')
return None
catalog_observables = dict()
dictionary_obs_time = CachedComponents.get_dictionary_observable_timestamps(
)
if not dictionary_obs_time:
dictionary_obs_time = dict()
catalog_all_observables = \
UtilityCatalogCached.get_complete_dictionary_observables(list_type_observables=catalog_datastreams.keys())
if not catalog_all_observables:
logger.info('UtilityCatalogCached No CATALOG OBSERVABLE FOUND')
return None
for type_observable in catalog_all_observables:
if type_observable not in catalog_datastreams:
logger.info(
'UtilityCatalogCached type_observable: {} not in catalog_datastream'
.format(type_observable))
continue
list_datastreams = catalog_datastreams[type_observable]
if not list_datastreams:
logger.info('UtilityCatalogCached NOT list_datastreams')
continue
list_mqtttopics = UtilityCatalogCached.\
get_complete_list_mqtttopics(list_datastreams_subject=list_datastreams)
if not list_mqtttopics:
logger.info(
'UtilityCatalogCached NO list_mqtt_topic associated to list_datastreams type_obs: {}'
.format(type_observable))
continue
list_observable_singletype = UtilityCatalogCached.\
get_observationlist_specifictype(type_observable=type_observable,
list_mqtttopics_admitted=list_mqtttopics,
dictionary_singletype_observables=catalog_all_observables[type_observable],
dictionary_obs_time=dictionary_obs_time)
if not list_observable_singletype:
logger.info(
'UtilityCatalogCached NOT list_observable_singletype type_observable: {}'
.format(type_observable))
continue
catalog_observables[
type_observable] = list_observable_singletype
UtilityCatalogCached.update_observables_timestamp(
catalog_observable=catalog_observables,
dictionary_obs_time=dictionary_obs_time)
return catalog_observables
except Exception as ex:
logger.info(
'UtilityCatalogCached get_observations_catalog Exception: {}'.
format(ex))
return None
def startup():
CachedComponents.startup()
UtilityCatalogCached.initialize_catalog()
def broker_connection(sender, instance, **kwargs):
try:
UtilityDatabase.purge_db_connections()
ServiceObservationClient.on_event_notify_obs = WorkerTasks.received_notification_new_observation
dictionary_result = UnitTestMain.launch_all_tests(
enable_tests=LOCAL_CONFIG[LocConfLbls.LABEL_ENABLE_UNIT_TESTS],
list_enabled_tests=LIST_UNITTESTS_ENABLED)
UnitTestMain.print_report(dictionary_test_results=dictionary_result)
if LOCAL_CONFIG[LocConfLbls.LABEL_ABORT_EXECUTION_AFTERUNITTESTS]:
logger.info(
'EXECUTION VOLUNTARY ABORTED AFTER UNIT TESTS EXECUTION')
return {"status", False}
UtilityStartupApplication.trace_startup_info()
UtilityStartupApplication.startup()
UtilityStartupApplication.adjust_startup_data()
ServiceCatalogClient.initialize()
ServiceUpdateDatastreamsClient.initialize(client_id=LOCAL_CONFIG[
LocConfLbls.LABEL_MQTT_CLIENT_PAHO_NAME_DATASTREAMUPDATE])
ServiceObservationClient.initialize(
client_id=LOCAL_CONFIG[
LocConfLbls.LABEL_MQTT_CLIENT_PAHO_NAME_OBSERVABLES],
username=LOCAL_CONFIG[LocConfLbls.LABEL_MQTT_CLIENT_USERNAME],
password=LOCAL_CONFIG[LocConfLbls.LABEL_MQTT_CLIENT_PASSWORD])
list_output_messages = LOCAL_CONFIG[
LocConfLbls.LABEL_OUTPUT_MESSAGELIST_SELECTED]
if list_output_messages:
for output_message in list_output_messages:
if output_message not in WP6_DICTIONARY_CATALOGOUTPUTINFO.keys(
):
continue
dictionary_sending = WP6_DICTIONARY_CATALOGOUTPUTINFO[
output_message]
broker_output_info = ServiceCatalogClient.acquire_output_broker_info(
request_catalog_url=LOCAL_CONFIG[
LocConfLbls.LABEL_WP6_CATALOG_CONNECTIONURL],
service_to_call=LOCAL_CONFIG[
LocConfLbls.
LABEL_WP6_CATALOG_POSTSERVICERETRIEVEOUTPUTINFO],
request_catalog_port=LOCAL_CONFIG[
LocConfLbls.LABEL_WP6_CATALOG_CONNECTIONPORT],
dictionary_message_sent=dictionary_sending)
if not broker_output_info:
logger.warning(
'UNABLE TO LOAD BROKER INFO. REQUEST EXIT APPLICATION')
exit()
CachedComponents.save_brokeroutput_servicecatalog(
broker_output=broker_output_info,
output_message_type=output_message)
if LocConfLbls.LABEL_BYPASS_MQTTINPUTMESSAGEACQUISITION not in LOCAL_CONFIG or \
not LOCAL_CONFIG[LocConfLbls.LABEL_BYPASS_MQTTINPUTMESSAGEACQUISITION]:
WorkerTasks.update_global_servicecatalog()
else:
logger.info("VOLUNTARY BYPASS MANAGEMENT MQTT INPUT DATA")
CachedComponents.set_startupapplication_completed()
if LOCAL_CONFIG[LocConfLbls.LABEL_MINIMUM_ACCEPTED_WRISTBAND_TO_START] > 0 and \
OutputMessageType.OUTPUT_MESSAGE_TYPE_CROWDHEATMAPOUTPUT in \
LOCAL_CONFIG[LocConfLbls.LABEL_OUTPUT_MESSAGELIST_SELECTED]:
counter_wb_registered = CachedComponents.get_counter_datastreams_registered(
datastream_feature=LabelObservationType.
LABEL_OBSTYPE_LOCALIZATION)
if counter_wb_registered < LOCAL_CONFIG[
LocConfLbls.LABEL_MINIMUM_ACCEPTED_WRISTBAND_TO_START]:
logger.info(
'HLDFAD MODULE STOPPED AFTER ACQUISITION FOR INSUFFICIENT NUMBER OF WRISTBAND, '
'{0} against {1} minimum configured'.format(
counter_wb_registered, LOCAL_CONFIG[
LocConfLbls.
LABEL_MINIMUM_ACCEPTED_WRISTBAND_TO_START]))
exit()
logger.info('HLDFAD STARTUP PHASE DONE WITH SUCCESS')
return {"status", True}
except Exception as ex:
logger.error(ex)
return {"status", False}
def handle_task_elaboration() -> bool:
try:
if not CachedComponents.check_startupapplication_completed():
logger.info(
'TASK ELABORATION BLOCKED (Registration Phase Not Completed)'
)
return False
pilot_name = LOCAL_CONFIG[LocConfLbls.LABEL_PILOT_NAME]
monitoring_area = MonitoringArea(pilot_name=pilot_name)
if WorkerTasks.appconf_checkoutputmessage_required(
OutputMessageType.OUTPUT_MESSAGE_TYPE_CROWDHEATMAPOUTPUT):
if not MONITORING_AREA:
logger.critical(
"TASK ELABORATION NOT POSSIBLE MONITORED AREA NOT SPECIFIED"
)
return False
monitoring_area_dictionary = MONITORING_AREA
monitoring_area.from_dictionary(
dictionary=monitoring_area_dictionary)
# TODO: Gestire il caso di crowd_density_global multipli
UtilityDatabase.purge_db_connections()
logger.info('TASK ELABORATION REQUEST ACTIVATION')
last_catalog_observation = \
UtilityCatalogCached.get_observations_catalog(catalog_datastreams=WorkerTasks.catalog_datastreams)
if not last_catalog_observation:
logger.info('TaskElaboration Did Not find Any measures')
return WorkerTasks.create_fake_output()
logger.info('TASK ELABORATION CALLED')
if LocConfLbls.LABEL_ENABLE_OBS_IOTIDRETRIEVE in LOCAL_CONFIG and \
LOCAL_CONFIG[LocConfLbls.LABEL_ENABLE_OBS_IOTIDRETRIEVE]:
logger.info(
'WorkerTask Elaboration Asking OBS_IOT_ID retrieve before computation'
)
last_catalog_observation = ServiceGetObservationGOST.set_correct_obs_iotid(
catalog_observations=last_catalog_observation,
gost_url=LOCAL_CONFIG[LocConfLbls.LABEL_GOST_URL])
if not Processing.real_processing_task(
last_catalog_observation=last_catalog_observation,
monitoring_area=monitoring_area,
list_output_requested=LOCAL_CONFIG[
LocConfLbls.LABEL_OUTPUT_MESSAGELIST_SELECTED]):
logger.info('TASK real_processing_task FAILED')
return False
UtilityCatalogCached.set_catalog_observable_backup(
catalog_observable=last_catalog_observation)
logger.info('TASK ELABORATION SUCCESS')
return True
except Exception as ex:
logger.error('handle_task_elaboration Exception: {}'.format(ex))
return False
def test_method_queueshapearea() -> bool:
try:
camera_id = 'LEEDS_CAM_4'
timestamp = datetime.datetime.now(tz=pytz.utc)
density_map = np.matrix(
[[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0],
[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0],
[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
[1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0],
[2.0, 5.0, 7.0, 9.0, 11.0, 1.0, 17.0, 18.0, 19.0, 20.0],
[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]])
dictionary_camera_registration = {
'camera_id': camera_id,
'camera_type': 'rgb',
'camera_position': [53.81761, -1.58331],
'ground_plane_position': [53.81761, -1.58331],
'ground_plane_size': [10, 30],
'image_2_ground_plane_matrix': [53.81761, -1.58331],
'zone_id': 'Main Entrance',
'state': 1
}
camera_registration = CameraRegistration()
camera_registration.from_dictionary(
dictionary=dictionary_camera_registration)
UtilityCatalogCached.add_device_registration(
device_registration=camera_registration)
crowd_density_local = UnitTestsUtilityObservables.test_method_create_obs_crowddensitylocal(
labe_gost=DEBUG_DEFAULT_SELECTED_GOST,
device_id=camera_id,
index_element=56,
density_matrix=density_map,
timestamp=timestamp)
if not crowd_density_local:
return False
device_registration = UtilityCatalogCached.get_device_registration(
datastream_id=camera_id)
if not device_registration:
return False
if not crowd_density_local.set_info_registration(
device_registration=device_registration):
return False
catalog_datastreams, list_datastreams = \
HelperDatastreamGenerator.debug_create_datastreams_from_dictionary(specific_argument=LabelObservationType.LABEL_OBSTYPE_CROWDDENSITY,
dictionary_topics=DICTIONARY_OBSERVABLE_TOPICS_CAMERAS)
UtilityCatalogCached.append_new_observable(
label_type_observable=crowd_density_local.get_type_observable(
),
observable=crowd_density_local)
catalog_observables = UtilityCatalogCached.get_complete_dictionary_observables(
list_type_observables=catalog_datastreams)
Processing.real_processing_task(
last_catalog_observation=catalog_observables,
monitoring_area=None)
total_element = CachedComponents.getcounter_queuedetectionalert_id(
)
counter_element = 0
while CachedComponents.getcounter_queuedetectionalert_id() > 0:
if counter_element >= total_element:
logger.error(
'Unexpected Behaviour From CachedList QueueDetection Alert. Exit'
)
return False
qdaid = CachedComponents.get_queuedetectionalert_id()
if qdaid == 0:
return False
queue_detection_alert = QueueDetectionAlert.objects.get(
qda_id=qdaid)
if not queue_detection_alert:
continue
dictionary_to_transfer = queue_detection_alert.to_dictionary()
if not dictionary_to_transfer:
continue
logger.info('Dictionary QueueShapeArea: {0}'.format(
dictionary_to_transfer))
counter_element += 1
logger.info(
'Test Processing QueueDetectionAlert Ended With Success')
return True
except Exception as ex:
logger.error('test_method_queueshapearea Exception: {}'.format(ex))
return False