class FunctionComponent(ResilientComponent):
"""Component that implements Resilient function 'mcafee_publish_to_dxl"""
config_file = "dxlclient_config"
def __init__(self, opts):
"""constructor provides access to the configuration options"""
super(FunctionComponent, self).__init__(opts)
try:
self.config = opts.get("fn_mcafee_opendxl").get(self.config_file)
if self.config is None:
log.error(
self.config_file +
" is not set. You must set this path to run this function")
raise ValueError(
self.config_file +
" is not set. You must set this path to run this function")
# Create configuration from file for DxlClient
config = DxlClientConfig.create_dxl_config_from_file(self.config)
# Create client
self.client = DxlClient(config)
self.client.connect()
except AttributeError:
log.error(
"There is no [fn_mcafee_opendxl] section in the config file,"
"please set that by running resilient-circuits config -u")
raise AttributeError(
"[fn_mcafee_opendxl] section is not set in the config file")
@handler("reload")
def _reload(self, event, opts):
"""Configuration options have changed, save new values"""
self.config = opts.get("fn_mcafee_opendxl").get(self.config_file)
@function("mcafee_publish_to_dxl")
def _mcafee_publish_to_dxl_function(self, event, *args, **kwargs):
"""Function: A function which takes 3 inputs:
mcafee_topic_name: String of the topic name. ie: /mcafee/service/epo/remote/epo1.
mcafee_dxl_payload: The text of the payload to publish to the topic.
mcafee_return_request: Specify whether or not to wait for and return the response.
The function will publish the provided payload to the provided topic.
Indicate whether acknowledgment response should be returned."""
try:
yield StatusMessage("Starting...")
# Get the function parameters:
mcafee_topic_name = kwargs.get("mcafee_topic_name") # text
if not mcafee_topic_name:
yield FunctionError("mcafee_topic_name is required")
mcafee_dxl_payload = kwargs.get("mcafee_dxl_payload") # text
if not mcafee_dxl_payload:
yield FunctionError("mcafee_dxl_payload is required")
mcafee_publish_method = self.get_select_param(
kwargs.get("mcafee_publish_method")
) # select, values: "Event", "Service"
if not mcafee_publish_method:
yield FunctionError("mcafee_publish_method is required")
mcafee_wait_for_response = self.get_select_param(
kwargs.get(
"mcafee_wait_for_response")) # select, values: "Yes", "No"
log.info("mcafee_topic_name: %s", mcafee_topic_name)
log.info("mcafee_dxl_payload: %s", mcafee_dxl_payload)
log.info("mcafee_publish_method: %s", mcafee_publish_method)
log.info("mcafee_wait_for_response: %s", mcafee_wait_for_response)
response = None
# Publish Event
if mcafee_publish_method == "Event":
event = Event(mcafee_topic_name)
event.payload = mcafee_dxl_payload
yield StatusMessage("Publishing Event...")
self.client.send_event(event)
# Invoke Service
else:
req = Request(mcafee_topic_name)
req.payload = mcafee_dxl_payload
yield StatusMessage("Invoking Service...")
if mcafee_wait_for_response == "No":
self.client.async_request(req)
else:
response = Response(
self.client.sync_request(req, timeout=300))
yield StatusMessage("Done...")
r = {
"mcafee_topic_name": mcafee_topic_name,
"mcafee_dxl_payload": mcafee_dxl_payload,
"mcafee_publish_method": mcafee_publish_method,
"mcafee_wait_for_response": mcafee_wait_for_response
}
# Return response from publishing to topic
if response is not None:
r["response"] = vars(response)
yield FunctionResult(r)
else:
yield FunctionResult(r)
except Exception as e:
yield FunctionError(e)
class MonitorModule(Module):
# Request topic for service registry queries
SERVICE_REGISTRY_QUERY_TOPIC = '/mcafee/service/dxl/svcregistry/query'
# Event topics for service registry changes
SERVICE_REGISTRY_REGISTER_EVENT_TOPIC = '/mcafee/event/dxl/svcregistry/register'
SERVICE_REGISTRY_UNREGISTER_EVENT_TOPIC = '/mcafee/event/dxl/svcregistry/unregister'
# How often(in seconds) to refresh the service list
SERVICE_UPDATE_INTERVAL = 60
# How long to retain clients without any keep alive before evicting them
CLIENT_RETENTION_MINUTES = 30
# A default SmartClient JSON response to show no results
NO_RESULT_JSON = u"""{response:{status:0,startRow:0,endRow:0,totalRows:0,data:[]}}"""
# Locks for the different dictionaries shared between Monitor Handlers
_service_dict_lock = threading.Lock()
_client_dict_lock = threading.Lock()
_web_socket_dict_lock = threading.Lock()
_pending_messages_lock = threading.Lock()
def __init__(self, app):
super(MonitorModule,
self).__init__(app, "monitor", "Fabric Monitor",
"/public/images/monitor.png", "monitor_layout")
# dictionary to store DXL Client instances unique to each "session"
self._client_dict = {}
# dictionary to store web sockets for each "session"
self._web_socket_dict = {}
# dictionary to store incoming messages for each "session"
self._pending_messages = {}
# dictionary to cache service state
self._services = {}
self._message_id_topics = {}
self._client_config = DxlClientConfig.create_dxl_config_from_file(
self.app.bootstrap_app.client_config_path)
# DXL Client to perform operations that are the same for all users(svc registry queries)
self._dxl_service_client = DxlClient(self._client_config)
self._dxl_service_client.connect()
self._dxl_service_client.add_event_callback(
MonitorModule.SERVICE_REGISTRY_REGISTER_EVENT_TOPIC,
_ServiceEventCallback(self))
self._dxl_service_client.add_event_callback(
MonitorModule.SERVICE_REGISTRY_UNREGISTER_EVENT_TOPIC,
_ServiceEventCallback(self))
self._refresh_all_services()
self._service_updater_thread = threading.Thread(
target=self._service_updater)
self._service_updater_thread.daemon = True
self._service_updater_thread.start()
self._dxl_client_cleanup_thread = threading.Thread(
target=self._cleanup_dxl_clients)
self._dxl_client_cleanup_thread.daemon = True
self._dxl_client_cleanup_thread.start()
@property
def handlers(self):
return [(r'/update_services', ServiceUpdateHandler, dict(module=self)),
(r'/subscriptions', SubscriptionsHandler, dict(module=self)),
(r'/messages', MessagesHandler, dict(module=self)),
(r'/send_message', SendMessageHandler, dict(module=self)),
(r'/websocket', ConsoleWebSocketHandler, dict(module=self))]
@property
def content(self):
content = pkg_resources.resource_string(__name__,
"content.html").decode("utf8")
return content.replace("@PORT@", str(self.app.bootstrap_app.port))
@property
def services(self):
with self._service_dict_lock:
return self._services.copy()
@property
def message_id_topics(self):
return self._message_id_topics
@property
def client_config(self):
return DxlClientConfig.create_dxl_config_from_file(
self.app.bootstrap_app.client_config_path)
def get_dxl_client(self, client_id):
"""
Retrieves the DxlClient for the given request. If there is not one associated with
the incoming request it creates a new one and saves the generated client_id as a cookie
:param client_id: The client identifier
:return: the DxlClient specific to this "session"
"""
if not self._client_exists_for_connection(client_id):
self._create_client_for_connection(client_id)
with self._client_dict_lock:
client = self._client_dict[client_id][0]
if not client.connected:
client.connect()
logger.debug("Returning DXL client for id: %s", client_id)
return client
def _create_client_for_connection(self, client_id):
"""
Creates a DxlClient and stores it for the give client_id
:param client_id: the client_id for the DxlClient
"""
client = DxlClient(self.client_config)
client.connect()
logger.info("Initializing new dxl client for client_id: %s", client_id)
with self._client_dict_lock:
self._client_dict[client_id] = (client, datetime.datetime.now())
def _client_exists_for_connection(self, client_id):
"""
Checks if there is already an existing DxlClient for the given client_id.
:param client_id: the ID of the DxlClient to check for
:return: whether there is an existing DxlClient for this ID
"""
with self._client_dict_lock:
return client_id in self._client_dict
@staticmethod
def create_smartclient_response_wrapper():
"""
Creates a wrapper object containing the standard fields required by SmartClient responses
:return: an initial SmartClient response wrapper
"""
response_wrapper = {"response": {}}
response = response_wrapper["response"]
response["status"] = 0
response["startRow"] = 0
response["endRow"] = 0
response["totalRows"] = 0
response["data"] = []
return response_wrapper
def create_smartclient_error_response(self, error_message):
"""
Creates an error response for the SmartClient UI with the given message
:param error_message: The error message
:return: The SmartClient response in dict form
"""
response_wrapper = self.create_smartclient_response_wrapper()
response = response_wrapper["response"]
response["status"] = -1
response["data"] = error_message
return response
def queue_message(self, message, client_id):
"""
Adds the given message to the pending messages queue for the give client.
:param message: the message to enqueue
:param client_id: the client the message is intended for
"""
with self._pending_messages_lock:
if client_id not in self._pending_messages:
self._pending_messages[client_id] = []
self._pending_messages[client_id].append(message)
def get_messages(self, client_id):
"""
Retrieves the messages pending for the given client. This does not
clear the queue after retrieving.
:param client_id: the client to retrieve messages for
:return: a List of messages for the client
"""
with self._pending_messages_lock:
if client_id in self._pending_messages:
return self._pending_messages[client_id]
return None
def clear_messages(self, client_id):
"""
Clears the pending messages for the given client.
:param client_id: the client to clear messages for
"""
with self._pending_messages_lock:
self._pending_messages[client_id] = []
def _service_updater(self):
"""
A thread target that will run forever and do a complete refresh of the
service list on an interval or if the DXL client reconnects
"""
while True:
with self._dxl_service_client._connected_lock:
self._dxl_service_client._connected_wait_condition.wait(
self.SERVICE_UPDATE_INTERVAL)
if self._dxl_service_client.connected:
logger.debug("Refreshing service list.")
self._refresh_all_services()
def _cleanup_dxl_clients(self):
"""
A thread target that will run forever and evict DXL clients if their
clients have not sent a keep-alive
"""
logger.debug("DXL client cleanup thread initialized.")
while True:
with self._client_dict_lock:
for key in list(self._client_dict):
if self._client_dict[key][1] < \
(datetime.datetime.now() - datetime.timedelta(
minutes=self.CLIENT_RETENTION_MINUTES)):
logger.debug("Evicting DXL client for client_id: %s",
key)
del self._client_dict[key]
time.sleep(5)
def _refresh_all_services(self):
"""
Queries the broker for the service list and replaces the currently stored one with the new
results. Notifies all connected web sockets that new services are available.
"""
req = Request(MonitorModule.SERVICE_REGISTRY_QUERY_TOPIC)
req.payload = "{}"
# Send the request
dxl_response = self._dxl_service_client.sync_request(req, 5)
dxl_response_dict = MessageUtils.json_payload_to_dict(dxl_response)
logger.info("Service registry response: %s", dxl_response_dict)
with self._service_dict_lock:
self._services = {}
for service_guid in dxl_response_dict["services"]:
self._services[service_guid] = dxl_response_dict["services"][
service_guid]
self.notify_web_sockets()
def update_service(self, service_event):
"""
Replaces a stored service data withe the one from the provided DXL service event
:param service_event: the DXL service event containing the service
"""
with self._service_dict_lock:
self._services[service_event['serviceGuid']] = service_event
def remove_service(self, service_event):
"""
Removes a stored service using the provided DXL service event
:param service_event: the DXL service event containing the service to be removed
"""
with self._service_dict_lock:
if service_event['serviceGuid'] in self._services:
del self._services[service_event['serviceGuid']]
def client_keep_alive(self, client_id):
logger.debug("Client keep-alive received for client id: %s", client_id)
if self._client_exists_for_connection(client_id):
with self._client_dict_lock:
self._client_dict[client_id] = (
self._client_dict[client_id][0], datetime.datetime.now())
@property
def io_loop(self):
"""
Returns the Tornado IOLoop that the web console uses
:return: The Tornado IOLoop instance
"""
return self.app.io_loop
def add_web_socket(self, client_id, web_socket):
"""
Stores a web socket associated with the given client id
:param client_id: the client id key the web socket to
:param web_socket: the web socket to store
"""
logger.debug("Adding web socket for client: %s", client_id)
with self._web_socket_dict_lock:
self._web_socket_dict[client_id] = web_socket
def remove_web_socket(self, client_id):
"""
Removes any web socket associated with the given client_id
:param client_id: The client ID
"""
logger.debug("Removing web socket for client: %s", client_id)
with self._web_socket_dict_lock:
self._web_socket_dict.pop(client_id, None)
def notify_web_sockets(self):
"""
Notifies all web sockets that there are pending service updates
"""
with self._web_socket_dict_lock:
for key in self._web_socket_dict:
try:
self.io_loop.add_callback(
self._web_socket_dict[key].write_message,
u"serviceUpdates")
except Exception:
pass
def get_message_topic(self, message):
"""
Determines the topic for the provided message. Replaces the response channel in
responses with the topic of the original request
:param message: The DXL message
:return: The topic to use
"""
if (message.message_type == Message.MESSAGE_TYPE_RESPONSE or
message.message_type == Message.MESSAGE_TYPE_ERROR) and \
message.request_message_id in self.message_id_topics:
topic = self.message_id_topics[message.request_message_id]
del self.message_id_topics[message.request_message_id]
else:
topic = message.destination_topic
return topic
class ServiceRequester(DxlRequesterInterface):
def __init__(self):
self.client = None
def connect(self, config_file="./dxlclient.config"):
if self.isConnected():
raise DxlJythonException(1100, "Already connected to the OpenDXL broker")
try:
logger.info("Reading configuration file from '%s'", config_file)
config = DxlClientConfig.create_dxl_config_from_file(config_file)
# Initialize DXL client using our configuration
self.client = DxlClient(config)
# Connect to DXL Broker
self.client.connect()
return
except Exception as e:
logger.info("Exception: " + e.message)
raise DxlJythonException(1000, "Unable to establish a connection with the DXL broker")
def sendMessage(self, topic="/dsa/dxl/test/event2", message="Default message"):
if not self.isConnected():
raise DxlJythonException(1200, "Not connected to a OpenDXL broker")
try:
request = Request(topic)
# Encode string payload as UTF-8
request.payload = message.encode()
# Send Synchronous Request with default timeout and wait for Response
logger.info("Requesting '" + message + "' from '" + topic + "'")
response = self.client.sync_request(request)
dxl_message = JavaDxlMessage()
dxl_message.setMessageVersion(response.version)
dxl_message.setMessageId(response.message_id)
dxl_message.setClientId(response.source_client_id)
dxl_message.setBrokerId(response.source_broker_id)
dxl_message.setMessageType(response.message_type)
dxl_message.setBrokerIdList(response.broker_ids)
dxl_message.setClientIdList(response.client_ids)
dxl_message.setRequestMessageId(response.request_message_id)
# Check that the Response is not an Error Response, then extract
if response.message_type != Message.MESSAGE_TYPE_ERROR:
dxl_message.setServiceId(response.service_id)
dxl_message.setPayload(response.payload.decode())
else:
dxl_message.setErrorCode(response.error_code)
dxl_message.setErrorMessage(response.error_message)
return dxl_message
except Exception as e:
logger.info("Exception: " + e.message)
raise DxlJythonException(1010, "Unable to communicate with a DXL broker")
def disconnect(self):
if not self.isConnected():
return
self.client.disconnect()
def isConnected(self):
if self.client is None:
return False;
return self.client.connected
