class FunctionComponent(ResilientComponent): """Component that implements Resilient function 'mcafee_publish_to_dxl""" config_file = "dxlclient_config" def __init__(self, opts): """constructor provides access to the configuration options""" super(FunctionComponent, self).__init__(opts) try: self.config = opts.get("fn_mcafee_opendxl").get(self.config_file) if self.config is None: log.error( self.config_file + " is not set. You must set this path to run this function") raise ValueError( self.config_file + " is not set. You must set this path to run this function") # Create configuration from file for DxlClient config = DxlClientConfig.create_dxl_config_from_file(self.config) # Create client self.client = DxlClient(config) self.client.connect() except AttributeError: log.error( "There is no [fn_mcafee_opendxl] section in the config file," "please set that by running resilient-circuits config -u") raise AttributeError( "[fn_mcafee_opendxl] section is not set in the config file") @handler("reload") def _reload(self, event, opts): """Configuration options have changed, save new values""" self.config = opts.get("fn_mcafee_opendxl").get(self.config_file) @function("mcafee_publish_to_dxl") def _mcafee_publish_to_dxl_function(self, event, *args, **kwargs): """Function: A function which takes 3 inputs: mcafee_topic_name: String of the topic name. ie: /mcafee/service/epo/remote/epo1. mcafee_dxl_payload: The text of the payload to publish to the topic. mcafee_return_request: Specify whether or not to wait for and return the response. The function will publish the provided payload to the provided topic. Indicate whether acknowledgment response should be returned.""" try: yield StatusMessage("Starting...") # Get the function parameters: mcafee_topic_name = kwargs.get("mcafee_topic_name") # text if not mcafee_topic_name: yield FunctionError("mcafee_topic_name is required") mcafee_dxl_payload = kwargs.get("mcafee_dxl_payload") # text if not mcafee_dxl_payload: yield FunctionError("mcafee_dxl_payload is required") mcafee_publish_method = self.get_select_param( kwargs.get("mcafee_publish_method") ) # select, values: "Event", "Service" if not mcafee_publish_method: yield FunctionError("mcafee_publish_method is required") mcafee_wait_for_response = self.get_select_param( kwargs.get( "mcafee_wait_for_response")) # select, values: "Yes", "No" log.info("mcafee_topic_name: %s", mcafee_topic_name) log.info("mcafee_dxl_payload: %s", mcafee_dxl_payload) log.info("mcafee_publish_method: %s", mcafee_publish_method) log.info("mcafee_wait_for_response: %s", mcafee_wait_for_response) response = None # Publish Event if mcafee_publish_method == "Event": event = Event(mcafee_topic_name) event.payload = mcafee_dxl_payload yield StatusMessage("Publishing Event...") self.client.send_event(event) # Invoke Service else: req = Request(mcafee_topic_name) req.payload = mcafee_dxl_payload yield StatusMessage("Invoking Service...") if mcafee_wait_for_response == "No": self.client.async_request(req) else: response = Response( self.client.sync_request(req, timeout=300)) yield StatusMessage("Done...") r = { "mcafee_topic_name": mcafee_topic_name, "mcafee_dxl_payload": mcafee_dxl_payload, "mcafee_publish_method": mcafee_publish_method, "mcafee_wait_for_response": mcafee_wait_for_response } # Return response from publishing to topic if response is not None: r["response"] = vars(response) yield FunctionResult(r) else: yield FunctionResult(r) except Exception as e: yield FunctionError(e)
class MonitorModule(Module): # Request topic for service registry queries SERVICE_REGISTRY_QUERY_TOPIC = '/mcafee/service/dxl/svcregistry/query' # Event topics for service registry changes SERVICE_REGISTRY_REGISTER_EVENT_TOPIC = '/mcafee/event/dxl/svcregistry/register' SERVICE_REGISTRY_UNREGISTER_EVENT_TOPIC = '/mcafee/event/dxl/svcregistry/unregister' # How often(in seconds) to refresh the service list SERVICE_UPDATE_INTERVAL = 60 # How long to retain clients without any keep alive before evicting them CLIENT_RETENTION_MINUTES = 30 # A default SmartClient JSON response to show no results NO_RESULT_JSON = u"""{response:{status:0,startRow:0,endRow:0,totalRows:0,data:[]}}""" # Locks for the different dictionaries shared between Monitor Handlers _service_dict_lock = threading.Lock() _client_dict_lock = threading.Lock() _web_socket_dict_lock = threading.Lock() _pending_messages_lock = threading.Lock() def __init__(self, app): super(MonitorModule, self).__init__(app, "monitor", "Fabric Monitor", "/public/images/monitor.png", "monitor_layout") # dictionary to store DXL Client instances unique to each "session" self._client_dict = {} # dictionary to store web sockets for each "session" self._web_socket_dict = {} # dictionary to store incoming messages for each "session" self._pending_messages = {} # dictionary to cache service state self._services = {} self._message_id_topics = {} self._client_config = DxlClientConfig.create_dxl_config_from_file( self.app.bootstrap_app.client_config_path) # DXL Client to perform operations that are the same for all users(svc registry queries) self._dxl_service_client = DxlClient(self._client_config) self._dxl_service_client.connect() self._dxl_service_client.add_event_callback( MonitorModule.SERVICE_REGISTRY_REGISTER_EVENT_TOPIC, _ServiceEventCallback(self)) self._dxl_service_client.add_event_callback( MonitorModule.SERVICE_REGISTRY_UNREGISTER_EVENT_TOPIC, _ServiceEventCallback(self)) self._refresh_all_services() self._service_updater_thread = threading.Thread( target=self._service_updater) self._service_updater_thread.daemon = True self._service_updater_thread.start() self._dxl_client_cleanup_thread = threading.Thread( target=self._cleanup_dxl_clients) self._dxl_client_cleanup_thread.daemon = True self._dxl_client_cleanup_thread.start() @property def handlers(self): return [(r'/update_services', ServiceUpdateHandler, dict(module=self)), (r'/subscriptions', SubscriptionsHandler, dict(module=self)), (r'/messages', MessagesHandler, dict(module=self)), (r'/send_message', SendMessageHandler, dict(module=self)), (r'/websocket', ConsoleWebSocketHandler, dict(module=self))] @property def content(self): content = pkg_resources.resource_string(__name__, "content.html").decode("utf8") return content.replace("@PORT@", str(self.app.bootstrap_app.port)) @property def services(self): with self._service_dict_lock: return self._services.copy() @property def message_id_topics(self): return self._message_id_topics @property def client_config(self): return DxlClientConfig.create_dxl_config_from_file( self.app.bootstrap_app.client_config_path) def get_dxl_client(self, client_id): """ Retrieves the DxlClient for the given request. If there is not one associated with the incoming request it creates a new one and saves the generated client_id as a cookie :param client_id: The client identifier :return: the DxlClient specific to this "session" """ if not self._client_exists_for_connection(client_id): self._create_client_for_connection(client_id) with self._client_dict_lock: client = self._client_dict[client_id][0] if not client.connected: client.connect() logger.debug("Returning DXL client for id: %s", client_id) return client def _create_client_for_connection(self, client_id): """ Creates a DxlClient and stores it for the give client_id :param client_id: the client_id for the DxlClient """ client = DxlClient(self.client_config) client.connect() logger.info("Initializing new dxl client for client_id: %s", client_id) with self._client_dict_lock: self._client_dict[client_id] = (client, datetime.datetime.now()) def _client_exists_for_connection(self, client_id): """ Checks if there is already an existing DxlClient for the given client_id. :param client_id: the ID of the DxlClient to check for :return: whether there is an existing DxlClient for this ID """ with self._client_dict_lock: return client_id in self._client_dict @staticmethod def create_smartclient_response_wrapper(): """ Creates a wrapper object containing the standard fields required by SmartClient responses :return: an initial SmartClient response wrapper """ response_wrapper = {"response": {}} response = response_wrapper["response"] response["status"] = 0 response["startRow"] = 0 response["endRow"] = 0 response["totalRows"] = 0 response["data"] = [] return response_wrapper def create_smartclient_error_response(self, error_message): """ Creates an error response for the SmartClient UI with the given message :param error_message: The error message :return: The SmartClient response in dict form """ response_wrapper = self.create_smartclient_response_wrapper() response = response_wrapper["response"] response["status"] = -1 response["data"] = error_message return response def queue_message(self, message, client_id): """ Adds the given message to the pending messages queue for the give client. :param message: the message to enqueue :param client_id: the client the message is intended for """ with self._pending_messages_lock: if client_id not in self._pending_messages: self._pending_messages[client_id] = [] self._pending_messages[client_id].append(message) def get_messages(self, client_id): """ Retrieves the messages pending for the given client. This does not clear the queue after retrieving. :param client_id: the client to retrieve messages for :return: a List of messages for the client """ with self._pending_messages_lock: if client_id in self._pending_messages: return self._pending_messages[client_id] return None def clear_messages(self, client_id): """ Clears the pending messages for the given client. :param client_id: the client to clear messages for """ with self._pending_messages_lock: self._pending_messages[client_id] = [] def _service_updater(self): """ A thread target that will run forever and do a complete refresh of the service list on an interval or if the DXL client reconnects """ while True: with self._dxl_service_client._connected_lock: self._dxl_service_client._connected_wait_condition.wait( self.SERVICE_UPDATE_INTERVAL) if self._dxl_service_client.connected: logger.debug("Refreshing service list.") self._refresh_all_services() def _cleanup_dxl_clients(self): """ A thread target that will run forever and evict DXL clients if their clients have not sent a keep-alive """ logger.debug("DXL client cleanup thread initialized.") while True: with self._client_dict_lock: for key in list(self._client_dict): if self._client_dict[key][1] < \ (datetime.datetime.now() - datetime.timedelta( minutes=self.CLIENT_RETENTION_MINUTES)): logger.debug("Evicting DXL client for client_id: %s", key) del self._client_dict[key] time.sleep(5) def _refresh_all_services(self): """ Queries the broker for the service list and replaces the currently stored one with the new results. Notifies all connected web sockets that new services are available. """ req = Request(MonitorModule.SERVICE_REGISTRY_QUERY_TOPIC) req.payload = "{}" # Send the request dxl_response = self._dxl_service_client.sync_request(req, 5) dxl_response_dict = MessageUtils.json_payload_to_dict(dxl_response) logger.info("Service registry response: %s", dxl_response_dict) with self._service_dict_lock: self._services = {} for service_guid in dxl_response_dict["services"]: self._services[service_guid] = dxl_response_dict["services"][ service_guid] self.notify_web_sockets() def update_service(self, service_event): """ Replaces a stored service data withe the one from the provided DXL service event :param service_event: the DXL service event containing the service """ with self._service_dict_lock: self._services[service_event['serviceGuid']] = service_event def remove_service(self, service_event): """ Removes a stored service using the provided DXL service event :param service_event: the DXL service event containing the service to be removed """ with self._service_dict_lock: if service_event['serviceGuid'] in self._services: del self._services[service_event['serviceGuid']] def client_keep_alive(self, client_id): logger.debug("Client keep-alive received for client id: %s", client_id) if self._client_exists_for_connection(client_id): with self._client_dict_lock: self._client_dict[client_id] = ( self._client_dict[client_id][0], datetime.datetime.now()) @property def io_loop(self): """ Returns the Tornado IOLoop that the web console uses :return: The Tornado IOLoop instance """ return self.app.io_loop def add_web_socket(self, client_id, web_socket): """ Stores a web socket associated with the given client id :param client_id: the client id key the web socket to :param web_socket: the web socket to store """ logger.debug("Adding web socket for client: %s", client_id) with self._web_socket_dict_lock: self._web_socket_dict[client_id] = web_socket def remove_web_socket(self, client_id): """ Removes any web socket associated with the given client_id :param client_id: The client ID """ logger.debug("Removing web socket for client: %s", client_id) with self._web_socket_dict_lock: self._web_socket_dict.pop(client_id, None) def notify_web_sockets(self): """ Notifies all web sockets that there are pending service updates """ with self._web_socket_dict_lock: for key in self._web_socket_dict: try: self.io_loop.add_callback( self._web_socket_dict[key].write_message, u"serviceUpdates") except Exception: pass def get_message_topic(self, message): """ Determines the topic for the provided message. Replaces the response channel in responses with the topic of the original request :param message: The DXL message :return: The topic to use """ if (message.message_type == Message.MESSAGE_TYPE_RESPONSE or message.message_type == Message.MESSAGE_TYPE_ERROR) and \ message.request_message_id in self.message_id_topics: topic = self.message_id_topics[message.request_message_id] del self.message_id_topics[message.request_message_id] else: topic = message.destination_topic return topic
class ServiceRequester(DxlRequesterInterface): def __init__(self): self.client = None def connect(self, config_file="./dxlclient.config"): if self.isConnected(): raise DxlJythonException(1100, "Already connected to the OpenDXL broker") try: logger.info("Reading configuration file from '%s'", config_file) config = DxlClientConfig.create_dxl_config_from_file(config_file) # Initialize DXL client using our configuration self.client = DxlClient(config) # Connect to DXL Broker self.client.connect() return except Exception as e: logger.info("Exception: " + e.message) raise DxlJythonException(1000, "Unable to establish a connection with the DXL broker") def sendMessage(self, topic="/dsa/dxl/test/event2", message="Default message"): if not self.isConnected(): raise DxlJythonException(1200, "Not connected to a OpenDXL broker") try: request = Request(topic) # Encode string payload as UTF-8 request.payload = message.encode() # Send Synchronous Request with default timeout and wait for Response logger.info("Requesting '" + message + "' from '" + topic + "'") response = self.client.sync_request(request) dxl_message = JavaDxlMessage() dxl_message.setMessageVersion(response.version) dxl_message.setMessageId(response.message_id) dxl_message.setClientId(response.source_client_id) dxl_message.setBrokerId(response.source_broker_id) dxl_message.setMessageType(response.message_type) dxl_message.setBrokerIdList(response.broker_ids) dxl_message.setClientIdList(response.client_ids) dxl_message.setRequestMessageId(response.request_message_id) # Check that the Response is not an Error Response, then extract if response.message_type != Message.MESSAGE_TYPE_ERROR: dxl_message.setServiceId(response.service_id) dxl_message.setPayload(response.payload.decode()) else: dxl_message.setErrorCode(response.error_code) dxl_message.setErrorMessage(response.error_message) return dxl_message except Exception as e: logger.info("Exception: " + e.message) raise DxlJythonException(1010, "Unable to communicate with a DXL broker") def disconnect(self): if not self.isConnected(): return self.client.disconnect() def isConnected(self): if self.client is None: return False; return self.client.connected