def __init__(self):
self.logger = api.server.app.logger
self.devices_service = DevicesService(logger=self.logger)
self.protocol_filter_handler = ProtocolFilterHandler(self.logger)
self.redis_connection = redis.StrictRedis(REDIS_HOSTNAME, REDIS_PORT)
self.redis_connection.pubsub()
def __init__(self):
self.logger = retrieve_logger("messages_processor")
self.walrus = Walrus(host=REDIS_HOSTNAME, port=REDIS_PORT)
self.devices_messages = self.walrus.List('devices_messages')
self.thread_pool_executor = ThreadPoolExecutor(
max_workers=self.MAX_WORKERS, )
self.devices_service = DevicesService(logger=self.logger)
self.rules_service = RulesService(logger=self.logger)
self.rules_executor = RulesExecutor(logger=self.logger)
self.protocol_filter_handler = ProtocolFilterHandler(self.logger)
class DevicesCommandView(MethodView):
PATCH_REQUIRED_FIELDS = {"value"}
def __init__(self):
self.logger = api.server.app.logger
self.devices_service = DevicesService(logger=self.logger)
self.protocol_filter_handler = ProtocolFilterHandler(self.logger)
self.redis_connection = redis.StrictRedis(REDIS_HOSTNAME, REDIS_PORT)
self.redis_connection.pubsub()
def _validate_patch_body(self, body):
if not isinstance(body, dict):
return "The body must be an object"
missing_fields = self.PATCH_REQUIRED_FIELDS - body.keys()
if missing_fields:
return "Missing fields: {}".format(missing_fields)
def patch(self, device_uuid):
body = request.get_json()
validation_error_message = self._validate_patch_body(body)
if validation_error_message:
self.logger.error(
"Some error occurred during the validation of the body. Reason: {}"
.format(validation_error_message))
response = {
"message": validation_error_message,
}
return jsonify(response), HTTPStatusCodes.BAD_REQUEST
device_uuid = str(device_uuid)
device = self.devices_service.find_by_device_id(device_uuid)
if not device:
self.logger.error("Device {} wasn't found".format(device_uuid))
response = {
"message": "Device not found",
}
return jsonify(response), HTTPStatusCodes.NOT_FOUND
message = {
"device_info": {
"device_uuid": device["id"],
"protocol": device["protocol"],
"ip": device["ip"],
"port": device.get("port", 0)
},
"value": body["value"],
}
self.protocol_filter_handler.parse(message)
return jsonify({}), HTTPStatusCodes.NO_CONTENT
def __init__(self, device_info, address, logger):
super().__init__(daemon=True)
self.device_info = self._transform_sen_ml_to_json(device_info)
self.address = address
self.logger = logger
self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
self.devices_service = DevicesService(logger=self.logger)
self.register_service = RegisterService(logger=self.logger)
self.docker_service = DockerService(logger=self.logger)
self.walrus = walrus.Database(host=GASS_REDIS_HOSTNAME,
port=GASS_REDIS_PORT)
self.containers_status = self.walrus.Hash(
GASS_REDIS_DOCKET_CONTAINERS_PROTOCOLS_HASH_KEY)
self.protocol_create = {
"COAP": self._start_coap_microservice,
"MQTT": self._start_mqtt_microservice,
}
self.protocol_check = {
"COAP": self._verify_coap_microservice,
"MQTT": self._verify_mqtt_microservices,
}
def __init__(self):
self.logger = retrieve_logger("message_forward")
self.devices_service = DevicesService(logger=self.logger)
self.redis_connection = redis_connection = redis.StrictRedis(
REDIS_HOSTNAME, REDIS_PORT)
self.redis_pub_sub = redis_connection.pubsub()
self.redis_pub_sub.subscribe(MESSAGE_FORWARD_REDIS_PUBSUB)
self.thread_pool_executor = ThreadPoolExecutor(
max_workers=self.MAX_WORKERS)
self.message_forward_helper = {
"MQTT": MQTTMessageForward(),
"COAP": CoAPMessageForward(),
}
class DevicesMessagesProcessor(object):
MAX_WORKERS = 50
GASS_SERVER_GATEWAY_DEVICES_PERFORMED_ACTIONS = "{}/gateways/{}/devices/actions" \
.format(GASS_SERVER_API_URL, GATEWAY_UUID)
GASS_SERVER_GATEWAY_DEVICES_RECEIVED_ACTIONS = "{}/gateways/{}/devices/actions/test" \
.format(GASS_SERVER_API_URL, GATEWAY_UUID)
def __init__(self):
self.logger = retrieve_logger("messages_processor")
self.walrus = Walrus(host=REDIS_HOSTNAME, port=REDIS_PORT)
self.devices_messages = self.walrus.List('devices_messages')
self.thread_pool_executor = ThreadPoolExecutor(
max_workers=self.MAX_WORKERS, )
self.devices_service = DevicesService(logger=self.logger)
self.rules_service = RulesService(logger=self.logger)
self.rules_executor = RulesExecutor(logger=self.logger)
self.protocol_filter_handler = ProtocolFilterHandler(self.logger)
def _rule_is_inside_activation_interval(self, interval_start,
interval_end):
current_date = datetime.datetime.utcnow()
weekday_inside_interval = interval_start[
"weekday"] <= current_date.weekday() <= interval_end["weekday"]
if not weekday_inside_interval:
return False
hour_inside_interval = interval_start[
"hour"] <= current_date.hour <= interval_end["hour"]
if not hour_inside_interval:
return False
minute_inside_interval = interval_start[
"minute"] <= current_date.minute <= interval_end["minute"]
if not minute_inside_interval:
return False
return True
def _filter_rules_by_active_interval(self, rules):
if not rules:
return []
valid_rules = []
for rule in rules:
if not rule.get("interval"):
valid_rules.append(rule)
continue
interval_start = rule["interval"]["start"]
interval_end = rule["interval"]["end"]
if not self._rule_is_inside_activation_interval(
interval_start, interval_end):
self.logger.error(
"The rule {} should not be verified because at the moment rule isn't active"
.format(str(rule["_id"])))
continue
valid_rules.append(rule)
return valid_rules
def _send_performed_actions(self, performed_actions):
body = performed_actions
attempts = 0
while True:
self.logger.info(
"Attempts {} to send to GaaS-Server the performed actions".
format(attempts))
try:
response = requests.post(
self.GASS_SERVER_GATEWAY_DEVICES_PERFORMED_ACTIONS,
json=body)
if response.status_code == HTTPStatusCodes.CREATED:
self.logger.debug(
"The performed actions has been sent successfully to the Gass Server"
)
return
attempts += 1
except Exception as err:
self.logger.error(
"Failed to send the performed actions to the GaaS Server. Reason: {}"
.format(err), )
self.logger.info("Sleeping...")
time.sleep(5)
attempts += 1
self.logger.info(
"Retry sending the performed actions to the GaSS Server")
def _test_forward_receive_actions(self, actions):
body = actions
attempts = 0
while True:
self.logger.info(
"Attempts {} to send to GaaS-Server the received actions")
try:
response = requests.post(
self.GASS_SERVER_GATEWAY_DEVICES_RECEIVED_ACTIONS,
json=body)
if response.status_code == HTTPStatusCodes.CREATED:
self.logger.debug(
"The performed actions has been sent successfully to the Gass Server"
)
return
attempts += 1
except Exception as err:
self.logger.error(
"Failed to send the performed actions to the GaaS Server. Reason: {}"
.format(err),
exc_info=True,
)
self.logger.info("Sleeping...")
time.sleep(5)
attempts += 1
self.logger.info(
"Retry sending the performed actions to the GaSS Server")
def _send_devices_new_values(self, devices_new_values):
devices_ids = [device['id'] for device in devices_new_values]
devices = list(self.devices_service.find_multiple_devices(devices_ids))
device_id_to_device_info = {device["id"]: device for device in devices}
for device_new_value in devices_new_values:
device_id, device_value = device_new_value["id"], device_new_value[
"value"]
device_info = device_id_to_device_info[device_id]
message = {
"device_info": {
"device_uuid": device_id,
"protocol": device_info["protocol"],
"ip": device_info["ip"],
"port": device_info.get("port", 0)
},
"value": device_value
}
self.protocol_filter_handler.parse(message)
def _parse_message(self, message):
try:
self.logger.debug("Received message: {}".format(message))
device_id, new_value = message["id"], message["v"]
updated = self.devices_service.update(device_id, new_value)
if not updated:
self.logger.debug("Failed to update the sensor's value")
return
performed_actions = [{
"type": ACTIONS_TYPES.CHANGE_VALUE,
"device": device_id,
"value": new_value,
"timestamp": get_utc_timestamp(),
}]
self.logger.debug("Device has been updated")
rules_to_check = list(
self.rules_service.find_that_involves_devices([device_id]))
rules_to_check = self._filter_rules_by_active_interval(
rules_to_check)
self.logger.debug("Have to check {} rules".format(
len(rules_to_check)))
devices_new_values, performed_actions_by_rules = self.rules_executor.execute(
rules_to_check)
performed_actions.extend(performed_actions_by_rules)
self.logger.debug(
"Performed actions: {}".format(performed_actions))
self.logger.debug("Send the performed actions to the server")
self._send_performed_actions(performed_actions)
# Send the new values to devices to update their state
self.logger.debug(
"Devices new values: {}".format(devices_new_values))
if not devices_new_values:
return
self._send_devices_new_values(devices_new_values)
self.logger.debug("The new values have been sent to the devices")
except Exception as err:
self.logger.error(
"Some error occurred while the message received from device was parsed. Reason: {}"
.format(err),
exc_info=True)
def _write_request_latency(self, emit_time):
now = time.time()
latency = now - emit_time
with open("latency.txt", mode="a") as file_handler:
file_handler.write("{}\n".format(latency))
self.logger.info("Current Average Latency: {}".format(latency))
def start(self):
self.logger.debug("Waiting for messages from devices")
while True:
message = self.devices_messages.bpopleft(timeout=120)
if not message:
continue
message = json.loads(message.decode())
self.logger.info("Received message: {}".format(message))
self.thread_pool_executor.submit(self._parse_message, message)
class ClientRequestHandler(threading.Thread):
REQUIRED_FIELDS = {"id", "type", "name", "protocol", "ip"}
MAX_ATTEMPTS_CONTAINER_PROTOCOL_START = 10
def __init__(self, device_info, address, logger):
super().__init__(daemon=True)
self.device_info = self._transform_sen_ml_to_json(device_info)
self.address = address
self.logger = logger
self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
self.devices_service = DevicesService(logger=self.logger)
self.register_service = RegisterService(logger=self.logger)
self.docker_service = DockerService(logger=self.logger)
self.walrus = walrus.Database(host=GASS_REDIS_HOSTNAME,
port=GASS_REDIS_PORT)
self.containers_status = self.walrus.Hash(
GASS_REDIS_DOCKET_CONTAINERS_PROTOCOLS_HASH_KEY)
self.protocol_create = {
"COAP": self._start_coap_microservice,
"MQTT": self._start_mqtt_microservice,
}
self.protocol_check = {
"COAP": self._verify_coap_microservice,
"MQTT": self._verify_mqtt_microservices,
}
def _transform_sen_ml_to_json(self, data):
device_info = json.loads(data.decode("utf-8"))
if device_info.get("bver"):
device_info.pop("bver")
name = device_info.pop("n", None)
if name:
device_info["name"] = name
value = device_info.pop("v", None)
if value:
device_info["value"] = value
return device_info
def _validate_request_message(self):
missing_fields = self.REQUIRED_FIELDS - self.device_info.keys()
if missing_fields:
return "The request {} has some missing fields: {}".format(
self.device_info, missing_fields)
def _verify_mqtt_microservices(self):
if "MQTT" not in self.containers_status:
return False
if "MQTT-BROKER" not in self.containers_status:
return False
return True
def _verify_coap_microservice(self):
return "COAP" in self.containers_status
def _start_mqtt_microservice(self):
if "MQTT-BROKER" not in self.containers_status:
mqtt_broker_container_id = self.docker_service.start_mqtt_broker()
self.containers_status["MQTT-BROKER"] = mqtt_broker_container_id
self.logger.info("Start MQTT-BROKER container with id {}".format(
mqtt_broker_container_id))
if "MQTT" not in self.containers_status:
mqtt_microservice_container_id = self.docker_service.start_mqtt_microservice(
)
self.containers_status["MQTT"] = mqtt_microservice_container_id
self.logger.info("Start MQTT container with id {}".format(
mqtt_microservice_container_id))
def _start_coap_microservice(self):
coap_microservice_container_id = self.docker_service.start_coap_microservice(
)
self.containers_status["COAP"] = coap_microservice_container_id
self.logger.info("Start COAP container with id {}".format(
coap_microservice_container_id))
def _verify_container(self):
protocol = self.device_info["protocol"]
microservices_already_up = self.protocol_check[protocol]()
if microservices_already_up:
self.logger.info(
"There is a container for protocol: {}".format(protocol))
return
attempt = 0
while attempt < self.MAX_ATTEMPTS_CONTAINER_PROTOCOL_START:
try:
self.logger.info(
"No docker for protocol: {}. Starting the microservice".
format(protocol))
self.logger.info("Attempt : {}".format(attempt))
self.protocol_create[protocol]()
break
except Exception as err:
self.logger.error(
"Some error occurred while the start of the microservice for protocol {} . Reason: {}"
.format(protocol, err),
exc_info=True)
attempt += 1
def run(self):
self.logger.debug("Received register response: {} ---- from {}".format(
self.device_info, self.address))
error_validation_message = self._validate_request_message()
if error_validation_message:
self.logger.error(error_validation_message)
self.sock.sendto(b"MISSING FIELDS", self.address)
self.sock.close()
return
self.device_info["gateway_uuid"] = GATEWAY_UUID
devices_has_been_register = self.register_service.register_device(
self.device_info)
if not devices_has_been_register:
self.logger.critical(
"Failed to register the device to GaSS Server")
self.sock.sendto(
b"Failed to register because the GaSS server is unreachable",
self.address)
self.sock.close()
return
device = self.devices_service.find_by_device_id(self.device_info["id"])
if device:
self.logger.debug("Device {} has been already registered".format(
self.device_info["id"]))
self._verify_container()
self.sock.sendto(b"OK", self.address)
self.sock.close()
return
device = self.devices_service.create(dict(self.device_info))
if not device:
self.logger.error("Failed to create the device")
self.sock.sendto(b"FAILED TO REGISTER", self.address)
self.sock.close()
return
# Start the micro-service
# Check in microservice is on or not
# If it will create the microservice, store the container id into REDIS
# So, the monitor will be able to check which container is still alive
# And if it finds that some containers doesn't exists will create another one
self._verify_container()
try:
self.logger.debug("Send ACK to device at address: {}: {}".format(
self.address[0], self.address[1]))
except Exception as err:
print(err)
raise
self.sock.sendto(b"OK", self.address)
self.sock.close()
self.logger.debug("Devices has been registered")
self.logger.debug("Sent registering confirmation")
def __init__(self, *args, **kwars):
self.logger = kwars.get("logger") or FakeLogger()
self.rules_engine = RulesEngine(logger=self.logger)
self.devices_service = DevicesService(logger=self.logger)
class RulesExecutor(object):
def __init__(self, *args, **kwars):
self.logger = kwars.get("logger") or FakeLogger()
self.rules_engine = RulesEngine(logger=self.logger)
self.devices_service = DevicesService(logger=self.logger)
def execute(self, rules):
if not rules:
return [], []
performed_actions = []
devices_new_values = []
rules_to_check = list(rules)
for rule in rules_to_check:
rule_id = str(rule["_id"])
self.logger.debug("Process rule {}".format(rule_id))
devices = self.devices_service.find_multiple_devices(
rule["devices_involved"])
device_id_to_value = {
device["id"]: device["value"]
for device in devices
}
missing_devices = set(
rule["devices_involved"]) - device_id_to_value.keys()
if missing_devices:
self.logger.error(
"Some devices are missing {}".format(missing_devices))
continue
rule_was_triggered = self.rules_engine.evaluate_rule(
rule, device_id_to_value)
if not rule_was_triggered:
self.logger.debug(
"Rule {} has been evaluated and wasn't triggered".format(
rule_id))
continue
performed_actions.append({
"type": ACTIONS_TYPES.RULE_TRIGGERED,
"rule": rule_id,
"timestamp": get_utc_timestamp()
})
self.logger.debug(
"Rule {} has been evaluated and it was triggered. Perform the in cause actions"
.format(rule_id))
actions_to_do = rule["actions"]
if not actions_to_do:
self.logger.debug(
"No actions has to be performed for rule {}".format(
rule_id))
continue
for device_id, new_value in actions_to_do.items():
devices_new_values.append({
"id": device_id,
"value": new_value
})
return devices_new_values, performed_actions