class Application(object):
"""
Base class used for DXL applications.
"""
# The name of the DXL client configuration file
DXL_CLIENT_CONFIG_FILE = "dxlclient.config"
# The timeout used when registering/unregistering the service
DXL_SERVICE_REGISTRATION_TIMEOUT = 60
# The name of the "IncomingMessagePool" section within the configuration file
INCOMING_MESSAGE_POOL_CONFIG_SECTION = "IncomingMessagePool"
# The name of the "MessageCallbackPool" section within the configuration file
MESSAGE_CALLBACK_POOL_CONFIG_SECTION = "MessageCallbackPool"
# The property used to specify a queue size
QUEUE_SIZE_CONFIG_PROP = "queueSize"
# The property used to specify a thread count
THREAD_COUNT_CONFIG_PROP = "threadCount"
# The default thread count for the incoming message pool
DEFAULT_THREAD_COUNT = 10
# The default queue size for the incoming message pool
DEFAULT_QUEUE_SIZE = 1000
def __init__(self, config_dir, app_config_file_name):
"""
Constructs the application
:param config_dir: The directory containing the application configuration files
:param app_config_file_name: The name of the application-specific configuration file
"""
self._config_dir = config_dir
self._dxlclient_config_path = os.path.join(config_dir,
self.DXL_CLIENT_CONFIG_FILE)
self._app_config_path = os.path.join(config_dir, app_config_file_name)
self._epo_by_topic = {}
self._dxl_client = None
self._running = False
self._destroyed = False
self._services = []
self._incoming_thread_count = self.DEFAULT_THREAD_COUNT
self._incoming_queue_size = self.DEFAULT_QUEUE_SIZE
self._callbacks_pool = None
self._callbacks_thread_count = self.DEFAULT_THREAD_COUNT
self._callbacks_queue_size = self.DEFAULT_QUEUE_SIZE
self._config = None
self._lock = RLock()
def __del__(self):
"""destructor"""
self.destroy()
def __enter__(self):
"""Enter with"""
return self
def __exit__(self, exc_type, exc_value, traceback):
"""Exit with"""
self.destroy()
def _validate_config_files(self):
"""
Validates the configuration files necessary for the application. An exception is thrown
if any of the required files are inaccessible.
"""
if not os.access(self._dxlclient_config_path, os.R_OK):
raise Exception(
"Unable to access client configuration file: {0}".format(
self._dxlclient_config_path))
if not os.access(self._app_config_path, os.R_OK):
raise Exception(
"Unable to access application configuration file: {0}".format(
self._app_config_path))
def _load_configuration(self):
"""
Loads the configuration settings from the application-specific configuration file
"""
config = ConfigParser()
self._config = config
read_files = config.read(self._app_config_path)
if len(read_files) is not 1:
raise Exception(
"Error attempting to read application configuration file: {0}".
format(self._app_config_path))
#
# Load incoming pool settings
#
try:
self._incoming_queue_size = config.getint(
self.INCOMING_MESSAGE_POOL_CONFIG_SECTION,
self.QUEUE_SIZE_CONFIG_PROP)
except:
pass
try:
self._incoming_thread_count = config.getint(
self.INCOMING_MESSAGE_POOL_CONFIG_SECTION,
self.THREAD_COUNT_CONFIG_PROP)
except:
pass
#
# Load callback pool settings
#
try:
self._callbacks_queue_size = config.getint(
self.MESSAGE_CALLBACK_POOL_CONFIG_SECTION,
self.QUEUE_SIZE_CONFIG_PROP)
except:
pass
try:
self._callbacks_thread_count = config.getint(
self.MESSAGE_CALLBACK_POOL_CONFIG_SECTION,
self.THREAD_COUNT_CONFIG_PROP)
except:
pass
self.on_load_configuration(config)
def _dxl_connect(self):
"""
Attempts to connect to the DXL fabric
"""
# Connect to fabric
config = DxlClientConfig.create_dxl_config_from_file(
self._dxlclient_config_path)
config.incoming_message_thread_pool_size = self._incoming_thread_count
config.incoming_message_queue_size = self._incoming_queue_size
logger.info(
"Incoming message configuration: queueSize={0}, threadCount={1}".
format(config.incoming_message_queue_size,
config.incoming_message_thread_pool_size))
logger.info(
"Message callback configuration: queueSize={0}, threadCount={1}".
format(self._callbacks_queue_size, self._callbacks_thread_count))
self._dxl_client = DxlClient(config)
logger.info("Attempting to connect to DXL fabric ...")
self._dxl_client.connect()
logger.info("Connected to DXL fabric.")
self.on_register_event_handlers()
self.on_register_services()
self.on_dxl_connect()
def run(self):
"""
Runs the application
"""
with self._lock:
if self._running:
raise Exception("The application is already running")
self._running = True
logger.info("Running application ...")
self.on_run()
self._validate_config_files()
self._load_configuration()
self._dxl_connect()
def destroy(self):
"""
Destroys the application (disconnects from fabric, frees resources, etc.)
"""
with self._lock:
if self._running and not self._destroyed:
logger.info("Destroying application ...")
if self._callbacks_pool is not None:
self._callbacks_pool.shutdown()
if self._dxl_client is not None:
self._unregister_services()
self._dxl_client.destroy()
self._dxl_client = None
self._destroyed = True
def _get_path(self, in_path):
"""
Returns an absolute path for a file specified in the configuration file (supports
files relative to the configuration file).
:param in_path: The specified path
:return: An absolute path for a file specified in the configuration file
"""
if not os.path.isfile(in_path) and not os.path.isabs(in_path):
config_rel_path = os.path.join(self._config_dir, in_path)
if os.path.isfile(config_rel_path):
in_path = config_rel_path
return in_path
def _unregister_services(self):
"""
Unregisters the services associated with the Application from the fabric
"""
for service in self._services:
self._dxl_client.unregister_service_sync(
service, self.DXL_SERVICE_REGISTRATION_TIMEOUT)
def _get_callbacks_pool(self):
"""
Returns the thread pool used to invoke application-specific message callbacks
:return: The thread pool used to invoke application-specific message callbacks
"""
with self._lock:
if self._callbacks_pool is None:
self._callbacks_pool = ThreadPool(self._callbacks_queue_size,
self._callbacks_thread_count,
"CallbacksPool")
return self._callbacks_pool
def add_event_callback(self, topic, callback, separate_thread):
"""
Adds a DXL event message callback to the application.
:param topic: The topic to associate with the callback
:param callback: The event callback
:param separate_thread: Whether to invoke the callback on a thread other than the incoming message
thread (this is necessary if synchronous requests are made via DXL in this callback).
"""
if separate_thread:
callback = _ThreadedEventCallback(self._get_callbacks_pool(),
callback)
self._dxl_client.add_event_callback(topic, callback)
def add_request_callback(self, service, topic, callback, separate_thread):
"""
Adds a DXL request message callback to the application.
:param service: The service to associate the request callback with
:param topic: The topic to associate with the callback
:param callback: The request callback
:param separate_thread: Whether to invoke the callback on a thread other than the incoming message
thread (this is necessary if synchronous requests are made via DXL in this callback).
"""
if separate_thread:
callback = _ThreadedRequestCallback(self._get_callbacks_pool(),
callback)
service.add_topic(topic, callback)
def register_service(self, service):
"""
Registers the specified service with the fabric
:param service: The service to register with the fabric
"""
self._dxl_client.register_service_sync(
service, self.DXL_SERVICE_REGISTRATION_TIMEOUT)
self._services.append(service)
def on_run(self):
"""
Invoked when the application has started running.
"""
pass
def on_load_configuration(self, config):
"""
Invoked after the application-specific configuration has been loaded
:param config: The application-specific configuration
"""
pass
def on_dxl_connect(self):
"""
Invoked after the client associated with the application has connected
to the DXL fabric.
"""
pass
def on_register_event_handlers(self):
"""
Invoked when event handlers should be registered with the application
"""
pass
def on_register_services(self):
"""
Invoked when services should be registered with the application
"""
pass
class MonitorModule(Module):
# Request topic for service registry queries
SERVICE_REGISTRY_QUERY_TOPIC = '/mcafee/service/dxl/svcregistry/query'
# Event topics for service registry changes
SERVICE_REGISTRY_REGISTER_EVENT_TOPIC = '/mcafee/event/dxl/svcregistry/register'
SERVICE_REGISTRY_UNREGISTER_EVENT_TOPIC = '/mcafee/event/dxl/svcregistry/unregister'
# How often(in seconds) to refresh the service list
SERVICE_UPDATE_INTERVAL = 60
# How long to retain clients without any keep alive before evicting them
CLIENT_RETENTION_MINUTES = 30
# A default SmartClient JSON response to show no results
NO_RESULT_JSON = u"""{response:{status:0,startRow:0,endRow:0,totalRows:0,data:[]}}"""
# Locks for the different dictionaries shared between Monitor Handlers
_service_dict_lock = threading.Lock()
_client_dict_lock = threading.Lock()
_web_socket_dict_lock = threading.Lock()
_pending_messages_lock = threading.Lock()
def __init__(self, app):
super(MonitorModule,
self).__init__(app, "monitor", "Fabric Monitor",
"/public/images/monitor.png", "monitor_layout")
# dictionary to store DXL Client instances unique to each "session"
self._client_dict = {}
# dictionary to store web sockets for each "session"
self._web_socket_dict = {}
# dictionary to store incoming messages for each "session"
self._pending_messages = {}
# dictionary to cache service state
self._services = {}
self._message_id_topics = {}
self._client_config = DxlClientConfig.create_dxl_config_from_file(
self.app.bootstrap_app.client_config_path)
# DXL Client to perform operations that are the same for all users(svc registry queries)
self._dxl_service_client = DxlClient(self._client_config)
self._dxl_service_client.connect()
self._dxl_service_client.add_event_callback(
MonitorModule.SERVICE_REGISTRY_REGISTER_EVENT_TOPIC,
_ServiceEventCallback(self))
self._dxl_service_client.add_event_callback(
MonitorModule.SERVICE_REGISTRY_UNREGISTER_EVENT_TOPIC,
_ServiceEventCallback(self))
self._refresh_all_services()
self._service_updater_thread = threading.Thread(
target=self._service_updater)
self._service_updater_thread.daemon = True
self._service_updater_thread.start()
self._dxl_client_cleanup_thread = threading.Thread(
target=self._cleanup_dxl_clients)
self._dxl_client_cleanup_thread.daemon = True
self._dxl_client_cleanup_thread.start()
@property
def handlers(self):
return [(r'/update_services', ServiceUpdateHandler, dict(module=self)),
(r'/subscriptions', SubscriptionsHandler, dict(module=self)),
(r'/messages', MessagesHandler, dict(module=self)),
(r'/send_message', SendMessageHandler, dict(module=self)),
(r'/websocket', ConsoleWebSocketHandler, dict(module=self))]
@property
def content(self):
content = pkg_resources.resource_string(__name__,
"content.html").decode("utf8")
return content.replace("@PORT@", str(self.app.bootstrap_app.port))
@property
def services(self):
with self._service_dict_lock:
return self._services.copy()
@property
def message_id_topics(self):
return self._message_id_topics
@property
def client_config(self):
return DxlClientConfig.create_dxl_config_from_file(
self.app.bootstrap_app.client_config_path)
def get_dxl_client(self, client_id):
"""
Retrieves the DxlClient for the given request. If there is not one associated with
the incoming request it creates a new one and saves the generated client_id as a cookie
:param client_id: The client identifier
:return: the DxlClient specific to this "session"
"""
if not self._client_exists_for_connection(client_id):
self._create_client_for_connection(client_id)
with self._client_dict_lock:
client = self._client_dict[client_id][0]
if not client.connected:
client.connect()
logger.debug("Returning DXL client for id: %s", client_id)
return client
def _create_client_for_connection(self, client_id):
"""
Creates a DxlClient and stores it for the give client_id
:param client_id: the client_id for the DxlClient
"""
client = DxlClient(self.client_config)
client.connect()
logger.info("Initializing new dxl client for client_id: %s", client_id)
with self._client_dict_lock:
self._client_dict[client_id] = (client, datetime.datetime.now())
def _client_exists_for_connection(self, client_id):
"""
Checks if there is already an existing DxlClient for the given client_id.
:param client_id: the ID of the DxlClient to check for
:return: whether there is an existing DxlClient for this ID
"""
with self._client_dict_lock:
return client_id in self._client_dict
@staticmethod
def create_smartclient_response_wrapper():
"""
Creates a wrapper object containing the standard fields required by SmartClient responses
:return: an initial SmartClient response wrapper
"""
response_wrapper = {"response": {}}
response = response_wrapper["response"]
response["status"] = 0
response["startRow"] = 0
response["endRow"] = 0
response["totalRows"] = 0
response["data"] = []
return response_wrapper
def create_smartclient_error_response(self, error_message):
"""
Creates an error response for the SmartClient UI with the given message
:param error_message: The error message
:return: The SmartClient response in dict form
"""
response_wrapper = self.create_smartclient_response_wrapper()
response = response_wrapper["response"]
response["status"] = -1
response["data"] = error_message
return response
def queue_message(self, message, client_id):
"""
Adds the given message to the pending messages queue for the give client.
:param message: the message to enqueue
:param client_id: the client the message is intended for
"""
with self._pending_messages_lock:
if client_id not in self._pending_messages:
self._pending_messages[client_id] = []
self._pending_messages[client_id].append(message)
def get_messages(self, client_id):
"""
Retrieves the messages pending for the given client. This does not
clear the queue after retrieving.
:param client_id: the client to retrieve messages for
:return: a List of messages for the client
"""
with self._pending_messages_lock:
if client_id in self._pending_messages:
return self._pending_messages[client_id]
return None
def clear_messages(self, client_id):
"""
Clears the pending messages for the given client.
:param client_id: the client to clear messages for
"""
with self._pending_messages_lock:
self._pending_messages[client_id] = []
def _service_updater(self):
"""
A thread target that will run forever and do a complete refresh of the
service list on an interval or if the DXL client reconnects
"""
while True:
with self._dxl_service_client._connected_lock:
self._dxl_service_client._connected_wait_condition.wait(
self.SERVICE_UPDATE_INTERVAL)
if self._dxl_service_client.connected:
logger.debug("Refreshing service list.")
self._refresh_all_services()
def _cleanup_dxl_clients(self):
"""
A thread target that will run forever and evict DXL clients if their
clients have not sent a keep-alive
"""
logger.debug("DXL client cleanup thread initialized.")
while True:
with self._client_dict_lock:
for key in list(self._client_dict):
if self._client_dict[key][1] < \
(datetime.datetime.now() - datetime.timedelta(
minutes=self.CLIENT_RETENTION_MINUTES)):
logger.debug("Evicting DXL client for client_id: %s",
key)
del self._client_dict[key]
time.sleep(5)
def _refresh_all_services(self):
"""
Queries the broker for the service list and replaces the currently stored one with the new
results. Notifies all connected web sockets that new services are available.
"""
req = Request(MonitorModule.SERVICE_REGISTRY_QUERY_TOPIC)
req.payload = "{}"
# Send the request
dxl_response = self._dxl_service_client.sync_request(req, 5)
dxl_response_dict = MessageUtils.json_payload_to_dict(dxl_response)
logger.info("Service registry response: %s", dxl_response_dict)
with self._service_dict_lock:
self._services = {}
for service_guid in dxl_response_dict["services"]:
self._services[service_guid] = dxl_response_dict["services"][
service_guid]
self.notify_web_sockets()
def update_service(self, service_event):
"""
Replaces a stored service data withe the one from the provided DXL service event
:param service_event: the DXL service event containing the service
"""
with self._service_dict_lock:
self._services[service_event['serviceGuid']] = service_event
def remove_service(self, service_event):
"""
Removes a stored service using the provided DXL service event
:param service_event: the DXL service event containing the service to be removed
"""
with self._service_dict_lock:
if service_event['serviceGuid'] in self._services:
del self._services[service_event['serviceGuid']]
def client_keep_alive(self, client_id):
logger.debug("Client keep-alive received for client id: %s", client_id)
if self._client_exists_for_connection(client_id):
with self._client_dict_lock:
self._client_dict[client_id] = (
self._client_dict[client_id][0], datetime.datetime.now())
@property
def io_loop(self):
"""
Returns the Tornado IOLoop that the web console uses
:return: The Tornado IOLoop instance
"""
return self.app.io_loop
def add_web_socket(self, client_id, web_socket):
"""
Stores a web socket associated with the given client id
:param client_id: the client id key the web socket to
:param web_socket: the web socket to store
"""
logger.debug("Adding web socket for client: %s", client_id)
with self._web_socket_dict_lock:
self._web_socket_dict[client_id] = web_socket
def remove_web_socket(self, client_id):
"""
Removes any web socket associated with the given client_id
:param client_id: The client ID
"""
logger.debug("Removing web socket for client: %s", client_id)
with self._web_socket_dict_lock:
self._web_socket_dict.pop(client_id, None)
def notify_web_sockets(self):
"""
Notifies all web sockets that there are pending service updates
"""
with self._web_socket_dict_lock:
for key in self._web_socket_dict:
try:
self.io_loop.add_callback(
self._web_socket_dict[key].write_message,
u"serviceUpdates")
except Exception:
pass
def get_message_topic(self, message):
"""
Determines the topic for the provided message. Replaces the response channel in
responses with the topic of the original request
:param message: The DXL message
:return: The topic to use
"""
if (message.message_type == Message.MESSAGE_TYPE_RESPONSE or
message.message_type == Message.MESSAGE_TYPE_ERROR) and \
message.request_message_id in self.message_id_topics:
topic = self.message_id_topics[message.request_message_id]
del self.message_id_topics[message.request_message_id]
else:
topic = message.destination_topic
return topic
class DXLBroker(object):
class MyEventCallback(EventCallback):
def __init__(self, broker):
EventCallback.__init__(self)
self.broker = broker
def on_event(self, event):
self.broker.logger.threaddebug(
f"{self.broker.device.name}: Message {event.message_id} ({event.message_type}), received: {event.destination_topic}, payload: {event.payload}"
)
indigo.activePlugin.processReceivedMessage(self.broker.device.id,
event.destination_topic,
event.payload)
def __init__(self, device):
self.logger = logging.getLogger("Plugin.DXLBroker")
self.deviceID = device.id
address = device.pluginProps.get(u'address', "")
port = device.pluginProps.get(u'port', "")
ca_bundle = indigo.server.getInstallFolderPath(
) + '/' + device.pluginProps.get(u'ca_bundle', "")
cert_file = indigo.server.getInstallFolderPath(
) + '/' + device.pluginProps.get(u'cert_file', "")
private_key = indigo.server.getInstallFolderPath(
) + '/' + device.pluginProps.get(u'private_key', "")
self.logger.debug(
f"{device.name}: Broker __init__ address = {address}, ca_bundle = {ca_bundle}, cert_file = {cert_file}, private_key = {private_key}"
)
device.updateStateOnServer(key="status", value="Not Connected")
device.updateStateImageOnServer(indigo.kStateImageSel.SensorOff)
# Create the client configuration
broker = Broker.parse(f"ssl://{address}:{port}")
config = DxlClientConfig(broker_ca_bundle=ca_bundle,
cert_file=cert_file,
private_key=private_key,
brokers=[broker])
# Create the DXL client
self.dxl_client = DxlClient(config)
# Connect to the fabric
self.dxl_client.connect()
device.updateStateOnServer(key="status", value="Connected")
device.updateStateImageOnServer(indigo.kStateImageSel.SensorOn)
subs = device.pluginProps.get(u'subscriptions', None)
if subs:
for topic in subs:
self.dxl_client.add_event_callback(topic,
self.MyEventCallback(self))
self.logger.info(u"{}: Subscribing to: {}".format(
device.name, topic))
def disconnect(self):
device = indigo.devices[self.deviceID]
self.dxl_client.disconnect()
self.dxl_client.destroy()
device.updateStateOnServer(key="status", value="Not Connected")
device.updateStateImageOnServer(indigo.kStateImageSel.SensorOff)
def publish(self, topic, payload=None, qos=0, retain=False):
event = Event(topic)
event.payload = payload
self.dxl_client.send_event(event)
def subscribe(self, topic):
device = indigo.devices[self.deviceID]
self.logger.info(f"{device.name}: Subscribing to: {topic}")
self.dxl_client.add_event_callback(topic, self.MyEventCallback(self))
def unsubscribe(self, topic):
device = indigo.devices[self.deviceID]
self.logger.info(f"{device.name}: Unsubscribing from: {topic}")
self.dxl_client.unsubscribe(topic)
class DxlComponentSubscriber(ResilientComponent):
"""Component that Subscribes to DXL Topics and maps data back into the Resilient Platform"""
def __init__(self, opts):
super(DxlComponentSubscriber, self).__init__(opts)
self.config = verify_config(opts)
add_methods_to_global()
# Create and connect DXL client
config_client_file = self.config.get("config_client")
dxl_config = DxlClientConfig.create_dxl_config_from_file(
config_client_file)
self.client = DxlClient(dxl_config)
self.client.connect()
# This gets run once to tell the subscriber to listen on defined topics
self.main()
def main(self):
if self.config["topic_listener_on"].lower() == "true":
log.info("Service Listener called")
self.event_subscriber(self.config)
else:
log.info(
"Event subscriber not listening. To turn on set topic_listener_on to True"
)
def event_subscriber(self, config):
try:
topic_template_dict = get_topic_template_dict(
config.get("custom_template_dir"))
class ResilientEventSubscriber(EventCallback):
def __init__(self, template):
super(ResilientEventSubscriber, self).__init__()
self.temp = template
def on_event(self, event):
message = event.payload.decode(encoding="UTF-8")
log.info("Event received payload: " + message)
message_dict = json.loads(message)
# Map values from topic to incident template to create new incident
inc_data = map_values(self.temp, message_dict)
# Create new Incident in Resilient
response = create_incident(
get_connected_resilient_client(config), inc_data)
log.info("Created incident {}".format(
str(response.get("id"))))
# Python 2.x solution
try:
for event_topic, template in topic_template_dict.iteritems():
self.client.add_event_callback(
event_topic.encode('ascii', 'ignore'),
ResilientEventSubscriber(
template.encode('ascii', 'ignore')))
log.info(
"Resilient DXL Subscriber listening on {} ...".format(
event_topic))
# Python 3.6 solution
except AttributeError:
for event_topic, template in topic_template_dict.items():
# Ensure unicode and bytes are converted to String
self.client.add_event_callback(
event_topic.encode('ascii', 'ignore').decode("utf-8"),
ResilientEventSubscriber(
template.encode('ascii',
'ignore').decode("utf-8")))
log.info(
"Resilient DXL Subscriber listening on {} ...".format(
event_topic))
except Exception as e:
log.error(e)
self.client.destroy()
class Application(object):
"""
Base class used for DXL applications.
"""
# The name of the DXL client configuration file
DXL_CLIENT_CONFIG_FILE = "dxlclient.config"
# The location of the logging configuration file (optional)
LOGGING_CONFIG_FILE = "logging.config"
# The timeout used when registering/unregistering the service
DXL_SERVICE_REGISTRATION_TIMEOUT = 60
# The name of the "IncomingMessagePool" section within the configuration file
INCOMING_MESSAGE_POOL_CONFIG_SECTION = "IncomingMessagePool"
# The name of the "MessageCallbackPool" section within the configuration file
MESSAGE_CALLBACK_POOL_CONFIG_SECTION = "MessageCallbackPool"
# The property used to specify a queue size
QUEUE_SIZE_CONFIG_PROP = "queueSize"
# The property used to specify a thread count
THREAD_COUNT_CONFIG_PROP = "threadCount"
# The default thread count for the incoming message pool
DEFAULT_THREAD_COUNT = 10
# The default queue size for the incoming message pool
DEFAULT_QUEUE_SIZE = 1000
# The directory containing the configuration files (in the Python library)
LIB_CONFIG_DIR = "_config"
# The directory containing the application configuration files (in the Python library)
LIB_APP_CONFIG_DIR = LIB_CONFIG_DIR + "/app"
def __init__(self, config_dir, app_config_file_name):
"""
Constructs the application
:param config_dir: The directory containing the application configuration files
:param app_config_file_name: The name of the application-specific configuration file
"""
self._config_dir = config_dir
self._dxlclient_config_path = os.path.join(config_dir,
self.DXL_CLIENT_CONFIG_FILE)
self._app_config_path = os.path.join(config_dir, app_config_file_name)
self._epo_by_topic = {}
self._dxl_client = None
self._running = False
self._destroyed = False
self._services = []
self._incoming_thread_count = self.DEFAULT_THREAD_COUNT
self._incoming_queue_size = self.DEFAULT_QUEUE_SIZE
self._callbacks_pool = None
self._callbacks_thread_count = self.DEFAULT_THREAD_COUNT
self._callbacks_queue_size = self.DEFAULT_QUEUE_SIZE
self._config = None
self._lock = RLock()
def __del__(self):
"""destructor"""
self.destroy()
def __enter__(self):
"""Enter with"""
return self
def __exit__(self, exc_type, exc_value, traceback):
"""Exit with"""
self.destroy()
def _validate_config_files(self):
"""
Validates the configuration files necessary for the application. An exception is thrown
if any of the required files are inaccessible.
"""
# Determine the module of the derived class
mod = self.__class__.__module__
# If the configuration directory exists in the library, create config files as necessary
# This check also provides backwards compatibility for projects that don't have the
# configuration files in the library.
if pkg_resources.resource_exists(mod, self.LIB_CONFIG_DIR):
# Create configuration directory if not found
if not os.access(self._config_dir, os.R_OK):
logger.info(
"Configuration directory '%s' not found, creating...",
self._config_dir)
os.makedirs(self._config_dir)
# Count of current configuration files
config_files_count = len([
name for name in os.listdir(self._config_dir)
if os.path.isfile(os.path.join(self._config_dir, name))
])
# Create configuration files if not found
files = pkg_resources.resource_listdir(mod,
self.LIB_APP_CONFIG_DIR)
for file_name in files:
config_path = os.path.join(self._config_dir, file_name)
if not os.access(config_path, os.R_OK):
resource_filename = pkg_resources.resource_filename(
mod, self.LIB_APP_CONFIG_DIR + "/" + file_name)
f_lower = file_name.lower()
# Copy configuration file. Only copy logging file if the
# directory was empty
if not os.path.isdir(resource_filename) and \
not(f_lower.endswith(".py")) and \
not(f_lower.endswith(".pyc")) and \
(f_lower != Application.LOGGING_CONFIG_FILE or
config_files_count == 0):
logger.info(
"Configuration file '%s' not found, creating...",
file_name)
shutil.copyfile(
pkg_resources.resource_filename(
mod,
self.LIB_APP_CONFIG_DIR + "/" + file_name),
config_path)
if not os.access(self._dxlclient_config_path, os.R_OK):
raise Exception(
"Unable to access client configuration file: {0}".format(
self._dxlclient_config_path))
if not os.access(self._app_config_path, os.R_OK):
raise Exception(
"Unable to access application configuration file: {0}".format(
self._app_config_path))
def _load_configuration(self):
"""
Loads the configuration settings from the application-specific configuration file
"""
config = ConfigParser()
self._config = config
read_files = config.read(self._app_config_path)
if len(read_files) is not 1:
raise Exception(
"Error attempting to read application configuration file: {0}".
format(self._app_config_path))
#
# Load incoming pool settings
#
# pylint: disable=bare-except
try:
self._incoming_queue_size = config.getint(
self.INCOMING_MESSAGE_POOL_CONFIG_SECTION,
self.QUEUE_SIZE_CONFIG_PROP)
except:
pass
try:
self._incoming_thread_count = config.getint(
self.INCOMING_MESSAGE_POOL_CONFIG_SECTION,
self.THREAD_COUNT_CONFIG_PROP)
except:
pass
#
# Load callback pool settings
#
try:
self._callbacks_queue_size = config.getint(
self.MESSAGE_CALLBACK_POOL_CONFIG_SECTION,
self.QUEUE_SIZE_CONFIG_PROP)
except:
pass
try:
self._callbacks_thread_count = config.getint(
self.MESSAGE_CALLBACK_POOL_CONFIG_SECTION,
self.THREAD_COUNT_CONFIG_PROP)
except:
pass
self.on_load_configuration(config)
def _dxl_connect(self):
"""
Attempts to connect to the DXL fabric
"""
# Connect to fabric
config = DxlClientConfig.create_dxl_config_from_file(
self._dxlclient_config_path)
config.incoming_message_thread_pool_size = self._incoming_thread_count
config.incoming_message_queue_size = self._incoming_queue_size
logger.info(
"Incoming message configuration: queueSize=%d, threadCount=%d",
config.incoming_message_queue_size,
config.incoming_message_thread_pool_size)
logger.info(
"Message callback configuration: queueSize=%d, threadCount=%d",
self._callbacks_queue_size, self._callbacks_thread_count)
self._dxl_client = DxlClient(config)
logger.info("Attempting to connect to DXL fabric ...")
self._dxl_client.connect()
logger.info("Connected to DXL fabric.")
self.on_register_event_handlers()
self.on_register_services()
self.on_dxl_connect()
def run(self):
"""
Runs the application
"""
with self._lock:
if self._running:
raise Exception("The application is already running")
self._running = True
logger.info("Running application ...")
self.on_run()
self._validate_config_files()
self._load_configuration()
self._dxl_connect()
def destroy(self):
"""
Destroys the application (disconnects from fabric, frees resources, etc.)
"""
with self._lock:
if self._running and not self._destroyed:
logger.info("Destroying application ...")
if self._callbacks_pool is not None:
self._callbacks_pool.shutdown()
if self._dxl_client is not None:
self._unregister_services()
self._dxl_client.destroy()
self._dxl_client = None
self._destroyed = True
def _get_path(self, in_path):
"""
Returns an absolute path for a file specified in the configuration file (supports
files relative to the configuration file).
:param in_path: The specified path
:return: An absolute path for a file specified in the configuration file
"""
if not os.path.isfile(in_path) and not os.path.isabs(in_path):
config_rel_path = os.path.join(self._config_dir, in_path)
if os.path.isfile(config_rel_path):
in_path = config_rel_path
return in_path
def _unregister_services(self):
"""
Unregisters the services associated with the Application from the fabric
"""
for service in self._services:
self._dxl_client.unregister_service_sync(
service, self.DXL_SERVICE_REGISTRATION_TIMEOUT)
def _get_callbacks_pool(self):
"""
Returns the thread pool used to invoke application-specific message callbacks
:return: The thread pool used to invoke application-specific message callbacks
"""
with self._lock:
if self._callbacks_pool is None:
self._callbacks_pool = ThreadPool(self._callbacks_queue_size,
self._callbacks_thread_count,
"CallbacksPool")
return self._callbacks_pool
def add_event_callback(self, topic, callback, separate_thread):
"""
Adds a DXL event message callback to the application.
:param topic: The topic to associate with the callback
:param callback: The event callback
:param separate_thread: Whether to invoke the callback on a thread other than the incoming message
thread (this is necessary if synchronous requests are made via DXL in this callback).
"""
if separate_thread:
callback = _ThreadedEventCallback(self._get_callbacks_pool(),
callback)
self._dxl_client.add_event_callback(topic, callback)
def add_request_callback(self, service, topic, callback, separate_thread):
"""
Adds a DXL request message callback to the application.
:param service: The service to associate the request callback with
:param topic: The topic to associate with the callback
:param callback: The request callback
:param separate_thread: Whether to invoke the callback on a thread other than the incoming message
thread (this is necessary if synchronous requests are made via DXL in this callback).
"""
if separate_thread:
callback = _ThreadedRequestCallback(self._get_callbacks_pool(),
callback)
service.add_topic(topic, callback)
def register_service(self, service):
"""
Registers the specified service with the fabric
:param service: The service to register with the fabric
"""
self._dxl_client.register_service_sync(
service, self.DXL_SERVICE_REGISTRATION_TIMEOUT)
self._services.append(service)
def on_run(self):
"""
Invoked when the application has started running.
"""
pass
def on_load_configuration(self, config):
"""
Invoked after the application-specific configuration has been loaded
:param config: The application-specific configuration
"""
pass
def on_dxl_connect(self):
"""
Invoked after the client associated with the application has connected
to the DXL fabric.
"""
pass
def on_register_event_handlers(self):
"""
Invoked when event handlers should be registered with the application
"""
pass
def on_register_services(self):
"""
Invoked when services should be registered with the application
"""
pass
