def mqtt_on_message(client: mqtt.Client, userdata: Any, msg: mqtt.MQTTMessage) -> None: """ MQTT Callback for when a PUBLISH message is received from the server. :param client: Class instance of connection to server. :param userdata: User-defined data passed to callbacks :param msg: Contains payload, topic, qos, retain """ payload = Message.unpack(msg.payload) print(f'Received: {payload}') # TODO: validate origin format profile, broker_socket = payload.origin.rsplit("@", 1) # Copy necessary headers header = { "socket": broker_socket, "correlationID": str(payload.request_id), "profile": profile, "encoding": payload.serialization, "transport": "mqtt" } # Connect and publish to internal buffer exchange = "orchestrator" route = "response" producer = Producer(os.environ.get("QUEUE_HOST", "queue"), os.environ.get("QUEUE_PORT", "5672")) producer.publish(headers=header, message=payload.content, exchange=exchange, routing_key=route) print( f"Received: {payload} \nPlaced message onto exchange [{exchange}] queue [{route}]." )
def render_POST_advanced(self, request, response): # retrieve Content_type stored as dict of types:values (ex. "application/json": 50) encoding = [k for k, v in defines.Content_types.items() if v == request.content_type] encoding = "json" if len(encoding) != 1 else encoding[0].split("/")[1] # read custom options added for O.I.F. and retrieve them based on their number # opts = {o.name: o.value for o in request.options} profile_opt = list(filter(lambda o: o.number == 8, request.options)) route = profile_opt[0].value if len(profile_opt) == 1 else None socket_opt = list(filter(lambda o: o.number == 3, request.options)) socket = socket_opt[0].value if len(socket_opt) == 1 else None print(f"{encoding}-{type(encoding)} -- {route}-{type(route)}") if encoding and route: print(f"Sending msg to {route}") # Create headers for the orchestrator from the request headers = dict( correlationID=request.mid, socket=socket, encoding=encoding, transport="coap" # orchestratorID="orchid1234", # orchestratorID is currently an unused field, this is a placeholder ) # Send request to actuator try: producer = Producer(os.environ.get("QUEUE_HOST", "localhost"), os.environ.get("QUEUE_PORT", "5672")) producer.publish( message=decode_msg(request.payload, encoding), headers=headers, exchange="actuator", routing_key=route ) response.payload = encode_msg({ "status": 200, "status_text": "received" }, encoding) response.code = defines.Codes.CONTENT.number except Exception as e: print(e) response.payload = e return self, response else: print(f"Not enough info: {encoding} - {route}") response.payload = encode_msg({ "status": 400, "status_text": "Not enough data to send message to actuator" }, encoding) response.code = defines.Codes.BAD_REQUEST.number return self, response
def result(): encode = re.search(r"(?<=\+)(.*?)(?=\;)", request.headers["Content-type"]).group( 1) # message encoding corr_id = request.headers["X-Request-ID"] # correlation ID status = request.headers['Status'] profile, device_socket = request.headers["From"].rsplit("@", 1) # profile used, device IP:port data = safe_json({ "headers": dict(request.headers), "content": safe_json(request.data.decode('utf-8')) }) print( f"Received {status} response from {profile}@{device_socket} - {data}") print("Writing to buffer.") producer = Producer() producer.publish( message=decode_msg(request.data, encode), # message being decoded headers={ "socket": device_socket, "correlationID": corr_id, "profile": profile, "encoding": encode, "transport": "https" }, exchange="orchestrator", routing_key="response") return make_response( # Body encode_msg({ "status": 200, "status_text": "received" }, encode), # Status Code 200, # Headers { "Content-type": f"application/openc2-rsp+{encode};version=1.0", "Status": 200, # Numeric status code supplied by Actuator's OpenC2-Response "X-Request-ID": corr_id, "Date": f"{datetime.utcnow():%a, %d %b %Y %H:%M:%S GMT}", # RFC7231-7.1.1.1 -> Sun, 06 Nov 1994 08:49:37 GMT # "From": f"{profile}@{device_socket}", # "Host": f"{orc_id}@{orc_socket}", })
def send_error_response(e, header): """ If error occurs before leaving the transport on the orchestrator side, then send back a message response to the internal buffer indicating so. :param e: Exception thrown :param header: Include headers which would have been sent for Orchestrator to read. """ producer = Producer(os.environ.get("QUEUE_HOST", "localhost"), os.environ.get("QUEUE_PORT", "5672")) err = json.dumps(str(e)) print(f"Send error response: {err}") producer.publish(headers=header, message=err, exchange="orchestrator", routing_key="response")
def on_message(client: mqtt.Client, userdata: Any, message: mqtt.MQTTMessage): """ MQTT Callback for when a PUBLISH message is received from the broker, forwards to AMQP buffer :param client: Class instance of connection to server. :param userdata: User-defined data passed to callbacks :param message: Contains payload, topic, qos, retain """ payload = Message.unpack(message.payload) print(f'Received: {payload}') # copy necessary headers headers = { "socket": None, # broker_socket, "correlationID": str(payload.request_id), "orchestratorID": payload.origin.rsplit("@", 1)[0], "encoding": payload.serialization, "profile": None, # profile, "transport": "mqtt" } # Connect and publish to internal buffer exchange = "actuator" producer = Producer(os.environ.get("QUEUE_HOST", "localhost"), os.environ.get("QUEUE_PORT", "5672")) for recipient in payload.recipients: profile, broker_socket = recipient.rsplit("@", 1) headers.update( socket=broker_socket, profile=profile, ) producer.publish(headers=headers, message=payload.content, exchange=exchange, routing_key=profile) print( f"Received: {payload} \nPlaced message onto exchange [{exchange}] queue [{profile}]." )
def on_message(act: Actuator, prod: Producer, body, message): """ Function that is called when a message is received from the queue/buffer :param act: actuator instance :param prod: producer to send response :param body: encoded message :param message: message instance from queue """ headers = getattr(message, "headers", {}) headers.setdefault("profile", act.profile) msg_id = headers.get('correlationID', '') encoding = headers.get('encoding', 'json') msg = decode_msg(body, encoding) msg_rsp = act.action(msg_id=msg_id, msg=msg) print(f"{act} -> received: {msg}") print(f"{act} -> response: {msg_rsp}") if msg_rsp: prod.publish(headers=headers, message=encode_msg(msg_rsp, encoding), exchange='transport', routing_key=headers.get('transport', '').lower())
def on_message(client, userdata, msg): """ MQTT Callback for when a PUBLISH message is received from the broker, forwards to AMQP buffer :param client: Class instance of connection to server. :param userdata: User-defined data passed to callbacks :param msg: Contains payload, topic, qos, retain """ payload = json.loads(msg.payload) payload_header = payload.get("header", {}) encoding = re.search(r"(?<=\+)(.*?)(?=;)", payload_header.get("content_type", "")).group(1) profile, broker_socket = payload_header.get("to", "").rsplit("@", 1) orc_id = payload_header.get("from", "").rsplit("@", 1)[0] corr_id = payload_header.get("correlationID", "") # copy necessary headers header = { "socket": broker_socket, "correlationID": corr_id, "orchestratorID": orc_id, "encoding": encoding, "profile": profile, "transport": "mqtt" } # Connect and publish to internal buffer exchange = "actuator" producer = Producer(os.environ.get("QUEUE_HOST", "localhost"), os.environ.get("QUEUE_PORT", "5672")) producer.publish(headers=header, message=payload.get("body", ""), exchange=exchange, routing_key=profile) print( f"Received: {payload} \nPlaced message onto exchange [{exchange}] queue [{profile}]." )
def render_POST_advanced(self, request, response): # retrieve Content_type stored as dict of types:values (ex. "application/json": 50) encoding = [ k for k, v in defines.Content_types.items() if v == request.content_type ] encoding = "json" if len(encoding) != 1 else encoding[0].split("/")[1] # read custom options added for O.I.F. and retrieve them based on their number # opts = {o.name: o.value for o in request.options} # Create headers for the orchestrator from the request headers = dict( correlationID=f"{request.mid:x}", socket=(request.source[0] + ":" + str(request.source[1])), encoding=encoding, transport="coap", # orchestratorID="orchid1234", # orchestratorID is currently an unused field, this is a placeholder ) # Send response back to Orchestrator producer = Producer(os.environ.get("QUEUE_HOST", "localhost"), os.environ.get("QUEUE_PORT", "5672")) producer.publish(message=decode_msg(request.payload, encoding), headers=headers, exchange="orchestrator", routing_key="response") # build and send response response.payload = encode_msg( { "status": 200, "status_text": "received" }, encoding) response.code = defines.Codes.CONTENT.number return self, response
def process_message(body, message): """ Callback when we receive a message from internal buffer to publish to waiting flask. :param body: Contains the message to be sent. :param message: Contains data about the message as well as headers """ producer = Producer() body = body if isinstance(body, dict) else safe_json(body) rcv_headers = message.headers orc_socket = rcv_headers["source"]["transport"]["socket"] # orch IP:port orc_id = rcv_headers["source"]["orchestratorID"] # orchestrator ID corr_id = rcv_headers["source"]["correlationID"] # correlation ID for device in rcv_headers["destination"]: device_socket = device["socket"] # device IP:port encoding = device["encoding"] # message encoding if device_socket and encoding and orc_socket: for profile in device["profile"]: print(f"Sending command to {profile}@{device_socket}") rtn_headers = { "socket": device_socket, "correlationID": corr_id, "profile": profile, "encoding": encoding, "transport": "https" } try: rslt = requests.post( url=f"http://{device_socket}", headers={ "Content-type": f"application/openc2-cmd+{encoding};version=1.0", # Numeric status code supplied by Actuator's OpenC2-Response # "Status": ..., "X-Request-ID": corr_id, # RFC7231-7.1.1.1 -> Sun, 06 Nov 1994 08:49:37 GMT "Date": f"{datetime.utcnow():%a, %d %b %Y %H:%M:%S GMT}", "From": f"{orc_id}@{orc_socket}", # "Host": f"{profile}@{device_socket}" }, data=encode_msg(body, encoding) # command being encoded ) data = { "headers": dict(rslt.headers), "content": decode_msg(rslt.content.decode('utf-8'), encoding) } print(f"Response from request: {rslt.status_code} - {data}") # TODO: UPDATE HEADERS WITH RESPONSE INFO response = safe_json(data['content']) if isinstance(data['content'], dict) else data['content'] except requests.exceptions.ConnectionError as err: response = str(getattr(err, "message", err)) rtn_headers["error"] = True print(f"Connection error: {err}") except json.decoder.JSONDecodeError as err: response = str(getattr(err, "message", err)) rtn_headers["error"] = True print(f"Message error: {err}") except Exception as err: response = str(getattr(err, "message", err)) rtn_headers["error"] = True print(f"HTTP error: {err}") producer.publish( message=response, headers=rtn_headers, exchange="orchestrator", routing_key="response" ) else: response = "Destination/Encoding/Orchestrator Socket of command not specified" rcv_headers["error"] = True print(response) producer.publish( message=str(response), headers=rcv_headers, exchange="orchestrator", routing_key="response" )
class MessageQueue: _auth = FrozenDict({'username': '******', 'password': '******'}) _exchange = 'orchestrator' _consumerKey = 'response' _producerExchange = 'producer_transport' def __init__(self, hostname='127.0.0.1', port=5672, auth=_auth, exchange=_exchange, consumer_key=_consumerKey, producer_exchange=_producerExchange, callbacks=None): """ Message Queue - holds a consumer class and producer class for ease of use :param hostname: server ip/hostname to connect :param port: port the AMQP Queue is listening :param exchange: name of the default exchange :param consumer_key: key to consumer :param producer_exchange: ... :param callbacks: list of functions to call on message receive """ self._exchange = exchange if isinstance(exchange, str) else self._exchange self._consumerKey = consumer_key if isinstance( consumer_key, str) else self._consumerKey self._producerExchange = producer_exchange if isinstance( producer_exchange, str) else self._producerExchange self._publish_opts = dict(host=hostname, port=safe_cast(port, int)) self._consume_opts = dict(host=hostname, port=safe_cast(port, int), exchange=self._exchange, routing_key=self._consumerKey, callbacks=callbacks) self.producer = Producer(**self._publish_opts) self.consumer = Consumer(**self._consume_opts) def send(self, msg, headers, exchange=_producerExchange, routing_key=None): """ Publish a message to the specified que and transport :param msg: message to be published :param headers: header information for the message being sent :param exchange: exchange name :param routing_key: routing key name :return: None """ headers = headers or {} if routing_key is None: raise ValueError('Routing Key cannot be None') self.producer.publish(message=msg, headers=headers, exchange=exchange, routing_key=routing_key) def shutdown(self): """ Shutdown the connection to the queue """ self.consumer.shutdown() self.consumer.join()
def result(): encode = re.search(r"(?<=\+)(.*?)(?=\;)", request.headers["Content-type"]).group(1) # message encoding corr_id = request.headers["X-Request-ID"] # correlation ID # data = decode_msg(request.data, encode) # message being decoded # profile, device_socket = request.headers["Host"].rsplit("@", 1) # profile used, device IP:port orc_id, orc_socket = request.headers["From"].rsplit("@", 1) # orchestrator ID, orchestrator IP:port message = request.data msg_json = decode_msg(message, encode) data = safe_json({ "headers": dict(request.headers), "content": safe_json(message.decode('utf-8')) }) rsp = { "status": 200, "status_text": "received", # command id?? } # Basic verify against language schema?? # get destination actuator actuators = list(msg_json.get('actuator', {}).keys()) print(f"Received command from {orc_id}@{orc_socket} - {data}") if msg_json['action'] == "query" and "command" in msg_json['target']: print("QUERY COMMAND") cmd_id = msg_json['target']['command'] prev_cmd = state.get(cmd_id) if prev_cmd: rsp = { "status_text": "previous command found", "response": { "command": prev_cmd[0] } } else: print("Writing to buffer") producer = Producer() queue_msg = { "message": message, "headers": { "socket": orc_socket, # "device": device_socket, "correlationID": corr_id, # "profile": profile, "encoding": encode, "orchestratorID": orc_id, "transport": "http" } } if len(actuators) == 0: print('No NSIDs specified, Send to all') try: producer.publish( **queue_msg, exchange="actuator_all", exchange_type="fanout", routing_key="actuator_all" ) except Exception as e: print(f'Publish Error: {e}') else: print(f'NSIDs specified - {actuators}') for act in actuators: producer.publish( **queue_msg, exchange="actuator", routing_key=act ) print(f"Corr_id: {corr_id}") for wait in range(0, MAX_WAIT): print(f"Checking for response... {MAX_WAIT} - {wait}") rsp_cmd = state.get(corr_id) if rsp_cmd: rsp = rsp_cmd[0]['body'] break time.sleep(1) return make_response( # Body encode_msg(rsp, encode), # Status Code 200, # Headers { "Content-type": f"application/openc2-rsp+{encode};version=1.0", "Status": 200, # Numeric status code supplied by Actuator's OpenC2-Response "X-Request-ID": corr_id, "Date": f"{datetime.utcnow():%a, %d %b %Y %H:%M:%S GMT}", # RFC7231-7.1.1.1 -> Sun, 06 Nov 1994 08:49:37 GMT # "From": f"{profile}@{device_socket}", # "Host": f"{orc_id}@{orc_socket}", } )
def process_message(body, message): """ Callback when we receive a message from internal buffer to publish to waiting flask. :param body: Contains the message to be sent. :param message: Contains data about the message as well as headers """ http = urllib3.PoolManager(cert_reqs="CERT_NONE") producer = Producer() body = body if isinstance(body, dict) else safe_json(body) rcv_headers = message.headers orc_socket = rcv_headers["source"]["transport"]["socket"] # orch IP:port orc_id = rcv_headers["source"]["orchestratorID"] # orchestrator ID corr_id = rcv_headers["source"]["correlationID"] # correlation ID for device in rcv_headers["destination"]: device_socket = device["socket"] # device IP:port encoding = device["encoding"] # message encoding if device_socket and encoding and orc_socket: for profile in device["profile"]: print(f"Sending command to {profile}@{device_socket}") try: rsp = http.request( method="POST", url=f"https://{device_socket}", body=encode_msg(body, encoding), # command being encoded headers={ "Content-type": f"application/openc2-cmd+{encoding};version=1.0", # "Status": ..., # Numeric status code supplied by Actuator's OpenC2-Response "X-Request-ID": corr_id, "Date": f"{datetime.utcnow():%a, %d %b %Y %H:%M:%S GMT}", # RFC7231-7.1.1.1 -> Sun, 06 Nov 1994 08:49:37 GMT "From": f"{orc_id}@{orc_socket}", "Host": f"{profile}@{device_socket}", } ) rsp_headers = dict(rsp.headers) if "Content-type" in rsp_headers: rsp_enc = re.sub(r"^application/openc2-(cmd|rsp)\+", "", rsp_headers["Content-type"]) rsp_enc = re.sub(r"(;version=\d+\.\d+)?$", "", rsp_enc) else: rsp_enc = "json" rsp_headers = { "socket": device_socket, "correlationID": corr_id, "profile": profile, "encoding": rsp_enc, "transport": "https" } data = { "headers": rsp_headers, "content": decode_msg(rsp.data.decode("utf-8"), rsp_enc) } print(f"Response from request: {rsp.status} - {safe_json(data)}") producer.publish(message=data["content"], headers=rsp_headers, exchange="orchestrator", routing_key="response") except Exception as err: err = str(getattr(err, "message", err)) rcv_headers["error"] = True producer.publish(message=err, headers=rcv_headers, exchange="orchestrator", routing_key="response") print(f"HTTPS error: {err}") else: response = "Destination/Encoding/Orchestrator Socket of command not specified" rcv_headers["error"] = True producer.publish(message=str(response), headers=rcv_headers, exchange="orchestrator", routing_key="response") print(response)
def process_message(body: Union[dict, str], message: kombu.Message) -> None: """ Callback when we receive a message from internal buffer to publish to waiting flask. :param body: Contains the message to be sent. :param message: Contains data about the message as well as headers """ producer = Producer() body = body if isinstance(body, dict) else safe_json(body) rcv_headers = message.headers orc_socket = rcv_headers["source"]["transport"]["socket"] # orch IP:port orc_id = rcv_headers["source"]["orchestratorID"] # orchestrator ID corr_id = uuid.UUID( rcv_headers["source"]["correlationID"]) # correlation ID for device in rcv_headers["destination"]: transport = transport_cache.cache.get(device["transport"]) device_socket = f"{transport['host']}:{transport['port']}" # device IP:port encoding = device["encoding"] # message encoding path = f"/{transport['path']}" if "path" in transport else "" if device_socket and encoding and orc_socket: with Auth(transport) as auth: for profile in device["profile"]: print(f"Sending command to {profile}@{device_socket}") rtn_headers = { "socket": device_socket, "correlationID": str(corr_id), "profile": profile, "encoding": encoding, "transport": "https" } request = Message( recipients=[f"{profile}@{device_socket}"], origin=f"{orc_id}@{orc_socket}", # created=... auto generated msg_type=MessageType.Request, request_id=corr_id, content_type=SerialFormats.from_value(encoding), content=body) prod_kwargs = { "cert": (auth.clientCert, auth.clientKey) if auth.clientCert and auth.clientKey else None, "verify": auth.caCert if auth.caCert else False } try: rslt = requests.post( url= f"http{'s' if transport['prod'] else ''}://{device_socket}{path}", headers={ "Content-Type": f"application/openc2-cmd+{encoding};version=1.0", # Numeric status code supplied by Actuator's OpenC2-Response # "Status": ..., "X-Request-ID": str(request.request_id), # RFC7231-7.1.1.1 -> Sun, 06 Nov 1994 08:49:37 GMT "Date": f"{request.created:%a, %d %b %Y %H:%M:%S GMT}", "From": request.origin, # "Host": f"{profile}@{device_socket}" }, data=request.serialize(), # command being encoded **(prod_kwargs if transport["prod"] else {})) response = Message.oc2_loads(rslt.content, encoding) print( f"Response from request: {rslt.status_code} - H:{dict(rslt.headers)} - C:{response}" ) # TODO: UPDATE HEADERS WITH RESPONSE INFO response = response.content except requests.exceptions.ConnectionError as err: response = str(getattr(err, "message", err)) rtn_headers["error"] = True print(f"Connection error: {err}") except json.decoder.JSONDecodeError as err: response = str(getattr(err, "message", err)) rtn_headers["error"] = True print(f"Message error: {err} - `{rslt.content}`") except Exception as err: response = str(getattr(err, "message", err)) rtn_headers["error"] = True print(f"HTTPS error: {err}") producer.publish(message=response, headers=rtn_headers, exchange="orchestrator", routing_key="response") else: response = "Destination/Encoding/Orchestrator Socket of command not specified" rcv_headers["error"] = True print(response) producer.publish(message=str(response), headers=rcv_headers, exchange="orchestrator", routing_key="response")
def result(): encode = re.search(r"(?<=\+)(.*?)(?=\;)", request.headers["Content-type"]).group( 1) # message encoding corr_id = request.headers["X-Request-ID"] # correlation ID # data = decode_msg(request.data, encode) # message being decoded profile, device_socket = request.headers["Host"].rsplit("@", 1) # profile used, device IP:port orc_id, orc_socket = request.headers["From"].rsplit("@", 1) # orchestrator ID, orchestrator IP:port message = request.data msg_json = decode_msg(message, encode) data = safe_json({ "headers": dict(request.headers), "content": safe_json(message.decode('utf-8')) }) rsp = { "status": 200, "status_text": "received", # command id?? } print(f"Received command from {orc_id}@{orc_socket} - {data}") if msg_json['action'] == "query" and "command" in msg_json['target']: print("QUERY COMMAND") cmd_id = msg_json['target']['command'] prev_cmd = state.get(cmd_id) if prev_cmd: rsp = { "status_text": "previous command found", "response": { "command": prev_cmd } } else: print("Writing to buffer") producer = Producer() producer.publish(message=message, headers={ "socket": orc_socket, "device": device_socket, "correlationID": corr_id, "profile": profile, "encoding": encode, "orchestratorID": orc_id, "transport": "https" }, exchange="actuator", routing_key=profile) print(f"Corr_id: {corr_id}") for wait in range(0, MAX_WAIT): print(f"Checking for response... {MAX_WAIT} - {wait}") rsp_cmd = state.get(corr_id) if rsp_cmd: rsp = rsp_cmd['body'] break time.sleep(1) return make_response( # Body encode_msg(rsp, encode), # Status Code 200, # Headers { "Content-type": f"application/openc2-rsp+{encode};version=1.0", "Status": 200, # Numeric status code supplied by Actuator's OpenC2-Response "X-Request-ID": corr_id, "Date": f"{datetime.utcnow():%a, %d %b %Y %H:%M:%S GMT}", # RFC7231-7.1.1.1 -> Sun, 06 Nov 1994 08:49:37 GMT # "From": f"{profile}@{device_socket}", # "Host": f"{orc_id}@{orc_socket}", })
header = { "socket": broker_socket, "correlationID": str(payload.request_id), "profile": profile, "encoding": payload.serialization, "transport": "mqtt" } # Connect and publish to internal buffer exchange = "orchestrator" route = "response" producer = Producer(os.environ.get("QUEUE_HOST", "queue"), os.environ.get("QUEUE_PORT", "5672")) producer.publish(headers=header, message=payload.content, exchange=exchange, routing_key=route) print( f"Received: {payload} \nPlaced message onto exchange [{exchange}] queue [{route}]." ) except Exception as e: print(f"Received: {msg.payload}") print(f"MQTT message error: {e}") def mqtt_on_log(client: mqtt.Client, userdata: List[str], level: int, buf: str) -> None: """ MQTT Callback for when a PUBLISH message is received from the broker, forwards to AMQP buffer :param client: Class instance of connection to server. :param userdata: User-defined data passed to callbacks