def _dxl_connect(self):
"""
Attempts to connect to the DXL fabric and register the ePO DXL service
"""
# Connect to fabric
config = DxlClientConfig.create_dxl_config_from_file(
self._dxlclient_config_path)
config.incoming_message_thread_pool_size = self._incoming_thread_count
config.incoming_message_queue_size = self._incoming_queue_size
logger.info(
"Incoming message configuration: queueSize={0}, threadCount={1}".
format(config.incoming_message_queue_size,
config.incoming_message_thread_pool_size))
client = DxlClient(config)
logger.info("Attempting to connect to DXL fabric ...")
client.connect()
logger.info("Connected to DXL fabric.")
try:
# Register service
service = ServiceRegistrationInfo(client, self.DXL_SERVICE_TYPE)
for request_topic in self._epo_by_topic:
service.add_topic(
str(request_topic),
_EpoRequestCallback(client, self._epo_by_topic))
logger.info("Registering service ...")
client.register_service_sync(service,
self.DXL_SERVICE_REGISTRATION_TIMEOUT)
logger.info("Service registration succeeded.")
except:
client.destroy()
raise
self._dxl_client = client
self._dxl_service = service
class Application(object):
"""
Base class used for DXL applications.
"""
# The name of the DXL client configuration file
DXL_CLIENT_CONFIG_FILE = "dxlclient.config"
# The timeout used when registering/unregistering the service
DXL_SERVICE_REGISTRATION_TIMEOUT = 60
# The name of the "IncomingMessagePool" section within the configuration file
INCOMING_MESSAGE_POOL_CONFIG_SECTION = "IncomingMessagePool"
# The name of the "MessageCallbackPool" section within the configuration file
MESSAGE_CALLBACK_POOL_CONFIG_SECTION = "MessageCallbackPool"
# The property used to specify a queue size
QUEUE_SIZE_CONFIG_PROP = "queueSize"
# The property used to specify a thread count
THREAD_COUNT_CONFIG_PROP = "threadCount"
# The default thread count for the incoming message pool
DEFAULT_THREAD_COUNT = 10
# The default queue size for the incoming message pool
DEFAULT_QUEUE_SIZE = 1000
def __init__(self, config_dir, app_config_file_name):
"""
Constructs the application
:param config_dir: The directory containing the application configuration files
:param app_config_file_name: The name of the application-specific configuration file
"""
self._config_dir = config_dir
self._dxlclient_config_path = os.path.join(config_dir,
self.DXL_CLIENT_CONFIG_FILE)
self._app_config_path = os.path.join(config_dir, app_config_file_name)
self._epo_by_topic = {}
self._dxl_client = None
self._running = False
self._destroyed = False
self._services = []
self._incoming_thread_count = self.DEFAULT_THREAD_COUNT
self._incoming_queue_size = self.DEFAULT_QUEUE_SIZE
self._callbacks_pool = None
self._callbacks_thread_count = self.DEFAULT_THREAD_COUNT
self._callbacks_queue_size = self.DEFAULT_QUEUE_SIZE
self._config = None
self._lock = RLock()
def __del__(self):
"""destructor"""
self.destroy()
def __enter__(self):
"""Enter with"""
return self
def __exit__(self, exc_type, exc_value, traceback):
"""Exit with"""
self.destroy()
def _validate_config_files(self):
"""
Validates the configuration files necessary for the application. An exception is thrown
if any of the required files are inaccessible.
"""
if not os.access(self._dxlclient_config_path, os.R_OK):
raise Exception(
"Unable to access client configuration file: {0}".format(
self._dxlclient_config_path))
if not os.access(self._app_config_path, os.R_OK):
raise Exception(
"Unable to access application configuration file: {0}".format(
self._app_config_path))
def _load_configuration(self):
"""
Loads the configuration settings from the application-specific configuration file
"""
config = ConfigParser()
self._config = config
read_files = config.read(self._app_config_path)
if len(read_files) is not 1:
raise Exception(
"Error attempting to read application configuration file: {0}".
format(self._app_config_path))
#
# Load incoming pool settings
#
try:
self._incoming_queue_size = config.getint(
self.INCOMING_MESSAGE_POOL_CONFIG_SECTION,
self.QUEUE_SIZE_CONFIG_PROP)
except:
pass
try:
self._incoming_thread_count = config.getint(
self.INCOMING_MESSAGE_POOL_CONFIG_SECTION,
self.THREAD_COUNT_CONFIG_PROP)
except:
pass
#
# Load callback pool settings
#
try:
self._callbacks_queue_size = config.getint(
self.MESSAGE_CALLBACK_POOL_CONFIG_SECTION,
self.QUEUE_SIZE_CONFIG_PROP)
except:
pass
try:
self._callbacks_thread_count = config.getint(
self.MESSAGE_CALLBACK_POOL_CONFIG_SECTION,
self.THREAD_COUNT_CONFIG_PROP)
except:
pass
self.on_load_configuration(config)
def _dxl_connect(self):
"""
Attempts to connect to the DXL fabric
"""
# Connect to fabric
config = DxlClientConfig.create_dxl_config_from_file(
self._dxlclient_config_path)
config.incoming_message_thread_pool_size = self._incoming_thread_count
config.incoming_message_queue_size = self._incoming_queue_size
logger.info(
"Incoming message configuration: queueSize={0}, threadCount={1}".
format(config.incoming_message_queue_size,
config.incoming_message_thread_pool_size))
logger.info(
"Message callback configuration: queueSize={0}, threadCount={1}".
format(self._callbacks_queue_size, self._callbacks_thread_count))
self._dxl_client = DxlClient(config)
logger.info("Attempting to connect to DXL fabric ...")
self._dxl_client.connect()
logger.info("Connected to DXL fabric.")
self.on_register_event_handlers()
self.on_register_services()
self.on_dxl_connect()
def run(self):
"""
Runs the application
"""
with self._lock:
if self._running:
raise Exception("The application is already running")
self._running = True
logger.info("Running application ...")
self.on_run()
self._validate_config_files()
self._load_configuration()
self._dxl_connect()
def destroy(self):
"""
Destroys the application (disconnects from fabric, frees resources, etc.)
"""
with self._lock:
if self._running and not self._destroyed:
logger.info("Destroying application ...")
if self._callbacks_pool is not None:
self._callbacks_pool.shutdown()
if self._dxl_client is not None:
self._unregister_services()
self._dxl_client.destroy()
self._dxl_client = None
self._destroyed = True
def _get_path(self, in_path):
"""
Returns an absolute path for a file specified in the configuration file (supports
files relative to the configuration file).
:param in_path: The specified path
:return: An absolute path for a file specified in the configuration file
"""
if not os.path.isfile(in_path) and not os.path.isabs(in_path):
config_rel_path = os.path.join(self._config_dir, in_path)
if os.path.isfile(config_rel_path):
in_path = config_rel_path
return in_path
def _unregister_services(self):
"""
Unregisters the services associated with the Application from the fabric
"""
for service in self._services:
self._dxl_client.unregister_service_sync(
service, self.DXL_SERVICE_REGISTRATION_TIMEOUT)
def _get_callbacks_pool(self):
"""
Returns the thread pool used to invoke application-specific message callbacks
:return: The thread pool used to invoke application-specific message callbacks
"""
with self._lock:
if self._callbacks_pool is None:
self._callbacks_pool = ThreadPool(self._callbacks_queue_size,
self._callbacks_thread_count,
"CallbacksPool")
return self._callbacks_pool
def add_event_callback(self, topic, callback, separate_thread):
"""
Adds a DXL event message callback to the application.
:param topic: The topic to associate with the callback
:param callback: The event callback
:param separate_thread: Whether to invoke the callback on a thread other than the incoming message
thread (this is necessary if synchronous requests are made via DXL in this callback).
"""
if separate_thread:
callback = _ThreadedEventCallback(self._get_callbacks_pool(),
callback)
self._dxl_client.add_event_callback(topic, callback)
def add_request_callback(self, service, topic, callback, separate_thread):
"""
Adds a DXL request message callback to the application.
:param service: The service to associate the request callback with
:param topic: The topic to associate with the callback
:param callback: The request callback
:param separate_thread: Whether to invoke the callback on a thread other than the incoming message
thread (this is necessary if synchronous requests are made via DXL in this callback).
"""
if separate_thread:
callback = _ThreadedRequestCallback(self._get_callbacks_pool(),
callback)
service.add_topic(topic, callback)
def register_service(self, service):
"""
Registers the specified service with the fabric
:param service: The service to register with the fabric
"""
self._dxl_client.register_service_sync(
service, self.DXL_SERVICE_REGISTRATION_TIMEOUT)
self._services.append(service)
def on_run(self):
"""
Invoked when the application has started running.
"""
pass
def on_load_configuration(self, config):
"""
Invoked after the application-specific configuration has been loaded
:param config: The application-specific configuration
"""
pass
def on_dxl_connect(self):
"""
Invoked after the client associated with the application has connected
to the DXL fabric.
"""
pass
def on_register_event_handlers(self):
"""
Invoked when event handlers should be registered with the application
"""
pass
def on_register_services(self):
"""
Invoked when services should be registered with the application
"""
pass
class DxlComponentSubscriber(ResilientComponent):
"""Component that Subscribes to DXL Topics and maps data back into the Resilient Platform"""
def __init__(self, opts):
super(DxlComponentSubscriber, self).__init__(opts)
self.config = verify_config(opts)
add_methods_to_global()
# Create and connect DXL client
config_client_file = self.config.get("config_client")
dxl_config = DxlClientConfig.create_dxl_config_from_file(
config_client_file)
self.client = DxlClient(dxl_config)
self.client.connect()
# This gets run once to tell the subscriber to listen on defined topics
self.main()
def main(self):
if self.config["topic_listener_on"].lower() == "true":
log.info("Service Listener called")
self.event_subscriber(self.config)
else:
log.info(
"Event subscriber not listening. To turn on set topic_listener_on to True"
)
def event_subscriber(self, config):
try:
topic_template_dict = get_topic_template_dict(
config.get("custom_template_dir"))
class ResilientEventSubscriber(EventCallback):
def __init__(self, template):
super(ResilientEventSubscriber, self).__init__()
self.temp = template
def on_event(self, event):
message = event.payload.decode(encoding="UTF-8")
log.info("Event received payload: " + message)
message_dict = json.loads(message)
# Map values from topic to incident template to create new incident
inc_data = map_values(self.temp, message_dict)
# Create new Incident in Resilient
response = create_incident(
get_connected_resilient_client(config), inc_data)
log.info("Created incident {}".format(
str(response.get("id"))))
# Python 2.x solution
try:
for event_topic, template in topic_template_dict.iteritems():
self.client.add_event_callback(
event_topic.encode('ascii', 'ignore'),
ResilientEventSubscriber(
template.encode('ascii', 'ignore')))
log.info(
"Resilient DXL Subscriber listening on {} ...".format(
event_topic))
# Python 3.6 solution
except AttributeError:
for event_topic, template in topic_template_dict.items():
# Ensure unicode and bytes are converted to String
self.client.add_event_callback(
event_topic.encode('ascii', 'ignore').decode("utf-8"),
ResilientEventSubscriber(
template.encode('ascii',
'ignore').decode("utf-8")))
log.info(
"Resilient DXL Subscriber listening on {} ...".format(
event_topic))
except Exception as e:
log.error(e)
self.client.destroy()
class DXLBroker(object):
class MyEventCallback(EventCallback):
def __init__(self, broker):
EventCallback.__init__(self)
self.broker = broker
def on_event(self, event):
self.broker.logger.threaddebug(
f"{self.broker.device.name}: Message {event.message_id} ({event.message_type}), received: {event.destination_topic}, payload: {event.payload}"
)
indigo.activePlugin.processReceivedMessage(self.broker.device.id,
event.destination_topic,
event.payload)
def __init__(self, device):
self.logger = logging.getLogger("Plugin.DXLBroker")
self.deviceID = device.id
address = device.pluginProps.get(u'address', "")
port = device.pluginProps.get(u'port', "")
ca_bundle = indigo.server.getInstallFolderPath(
) + '/' + device.pluginProps.get(u'ca_bundle', "")
cert_file = indigo.server.getInstallFolderPath(
) + '/' + device.pluginProps.get(u'cert_file', "")
private_key = indigo.server.getInstallFolderPath(
) + '/' + device.pluginProps.get(u'private_key', "")
self.logger.debug(
f"{device.name}: Broker __init__ address = {address}, ca_bundle = {ca_bundle}, cert_file = {cert_file}, private_key = {private_key}"
)
device.updateStateOnServer(key="status", value="Not Connected")
device.updateStateImageOnServer(indigo.kStateImageSel.SensorOff)
# Create the client configuration
broker = Broker.parse(f"ssl://{address}:{port}")
config = DxlClientConfig(broker_ca_bundle=ca_bundle,
cert_file=cert_file,
private_key=private_key,
brokers=[broker])
# Create the DXL client
self.dxl_client = DxlClient(config)
# Connect to the fabric
self.dxl_client.connect()
device.updateStateOnServer(key="status", value="Connected")
device.updateStateImageOnServer(indigo.kStateImageSel.SensorOn)
subs = device.pluginProps.get(u'subscriptions', None)
if subs:
for topic in subs:
self.dxl_client.add_event_callback(topic,
self.MyEventCallback(self))
self.logger.info(u"{}: Subscribing to: {}".format(
device.name, topic))
def disconnect(self):
device = indigo.devices[self.deviceID]
self.dxl_client.disconnect()
self.dxl_client.destroy()
device.updateStateOnServer(key="status", value="Not Connected")
device.updateStateImageOnServer(indigo.kStateImageSel.SensorOff)
def publish(self, topic, payload=None, qos=0, retain=False):
event = Event(topic)
event.payload = payload
self.dxl_client.send_event(event)
def subscribe(self, topic):
device = indigo.devices[self.deviceID]
self.logger.info(f"{device.name}: Subscribing to: {topic}")
self.dxl_client.add_event_callback(topic, self.MyEventCallback(self))
def unsubscribe(self, topic):
device = indigo.devices[self.deviceID]
self.logger.info(f"{device.name}: Unsubscribing from: {topic}")
self.dxl_client.unsubscribe(topic)
class Application(object):
"""
Base class used for DXL applications.
"""
# The name of the DXL client configuration file
DXL_CLIENT_CONFIG_FILE = "dxlclient.config"
# The location of the logging configuration file (optional)
LOGGING_CONFIG_FILE = "logging.config"
# The timeout used when registering/unregistering the service
DXL_SERVICE_REGISTRATION_TIMEOUT = 60
# The name of the "IncomingMessagePool" section within the configuration file
INCOMING_MESSAGE_POOL_CONFIG_SECTION = "IncomingMessagePool"
# The name of the "MessageCallbackPool" section within the configuration file
MESSAGE_CALLBACK_POOL_CONFIG_SECTION = "MessageCallbackPool"
# The property used to specify a queue size
QUEUE_SIZE_CONFIG_PROP = "queueSize"
# The property used to specify a thread count
THREAD_COUNT_CONFIG_PROP = "threadCount"
# The default thread count for the incoming message pool
DEFAULT_THREAD_COUNT = 10
# The default queue size for the incoming message pool
DEFAULT_QUEUE_SIZE = 1000
# The directory containing the configuration files (in the Python library)
LIB_CONFIG_DIR = "_config"
# The directory containing the application configuration files (in the Python library)
LIB_APP_CONFIG_DIR = LIB_CONFIG_DIR + "/app"
def __init__(self, config_dir, app_config_file_name):
"""
Constructs the application
:param config_dir: The directory containing the application configuration files
:param app_config_file_name: The name of the application-specific configuration file
"""
self._config_dir = config_dir
self._dxlclient_config_path = os.path.join(config_dir,
self.DXL_CLIENT_CONFIG_FILE)
self._app_config_path = os.path.join(config_dir, app_config_file_name)
self._epo_by_topic = {}
self._dxl_client = None
self._running = False
self._destroyed = False
self._services = []
self._incoming_thread_count = self.DEFAULT_THREAD_COUNT
self._incoming_queue_size = self.DEFAULT_QUEUE_SIZE
self._callbacks_pool = None
self._callbacks_thread_count = self.DEFAULT_THREAD_COUNT
self._callbacks_queue_size = self.DEFAULT_QUEUE_SIZE
self._config = None
self._lock = RLock()
def __del__(self):
"""destructor"""
self.destroy()
def __enter__(self):
"""Enter with"""
return self
def __exit__(self, exc_type, exc_value, traceback):
"""Exit with"""
self.destroy()
def _validate_config_files(self):
"""
Validates the configuration files necessary for the application. An exception is thrown
if any of the required files are inaccessible.
"""
# Determine the module of the derived class
mod = self.__class__.__module__
# If the configuration directory exists in the library, create config files as necessary
# This check also provides backwards compatibility for projects that don't have the
# configuration files in the library.
if pkg_resources.resource_exists(mod, self.LIB_CONFIG_DIR):
# Create configuration directory if not found
if not os.access(self._config_dir, os.R_OK):
logger.info(
"Configuration directory '%s' not found, creating...",
self._config_dir)
os.makedirs(self._config_dir)
# Count of current configuration files
config_files_count = len([
name for name in os.listdir(self._config_dir)
if os.path.isfile(os.path.join(self._config_dir, name))
])
# Create configuration files if not found
files = pkg_resources.resource_listdir(mod,
self.LIB_APP_CONFIG_DIR)
for file_name in files:
config_path = os.path.join(self._config_dir, file_name)
if not os.access(config_path, os.R_OK):
resource_filename = pkg_resources.resource_filename(
mod, self.LIB_APP_CONFIG_DIR + "/" + file_name)
f_lower = file_name.lower()
# Copy configuration file. Only copy logging file if the
# directory was empty
if not os.path.isdir(resource_filename) and \
not(f_lower.endswith(".py")) and \
not(f_lower.endswith(".pyc")) and \
(f_lower != Application.LOGGING_CONFIG_FILE or
config_files_count == 0):
logger.info(
"Configuration file '%s' not found, creating...",
file_name)
shutil.copyfile(
pkg_resources.resource_filename(
mod,
self.LIB_APP_CONFIG_DIR + "/" + file_name),
config_path)
if not os.access(self._dxlclient_config_path, os.R_OK):
raise Exception(
"Unable to access client configuration file: {0}".format(
self._dxlclient_config_path))
if not os.access(self._app_config_path, os.R_OK):
raise Exception(
"Unable to access application configuration file: {0}".format(
self._app_config_path))
def _load_configuration(self):
"""
Loads the configuration settings from the application-specific configuration file
"""
config = ConfigParser()
self._config = config
read_files = config.read(self._app_config_path)
if len(read_files) is not 1:
raise Exception(
"Error attempting to read application configuration file: {0}".
format(self._app_config_path))
#
# Load incoming pool settings
#
# pylint: disable=bare-except
try:
self._incoming_queue_size = config.getint(
self.INCOMING_MESSAGE_POOL_CONFIG_SECTION,
self.QUEUE_SIZE_CONFIG_PROP)
except:
pass
try:
self._incoming_thread_count = config.getint(
self.INCOMING_MESSAGE_POOL_CONFIG_SECTION,
self.THREAD_COUNT_CONFIG_PROP)
except:
pass
#
# Load callback pool settings
#
try:
self._callbacks_queue_size = config.getint(
self.MESSAGE_CALLBACK_POOL_CONFIG_SECTION,
self.QUEUE_SIZE_CONFIG_PROP)
except:
pass
try:
self._callbacks_thread_count = config.getint(
self.MESSAGE_CALLBACK_POOL_CONFIG_SECTION,
self.THREAD_COUNT_CONFIG_PROP)
except:
pass
self.on_load_configuration(config)
def _dxl_connect(self):
"""
Attempts to connect to the DXL fabric
"""
# Connect to fabric
config = DxlClientConfig.create_dxl_config_from_file(
self._dxlclient_config_path)
config.incoming_message_thread_pool_size = self._incoming_thread_count
config.incoming_message_queue_size = self._incoming_queue_size
logger.info(
"Incoming message configuration: queueSize=%d, threadCount=%d",
config.incoming_message_queue_size,
config.incoming_message_thread_pool_size)
logger.info(
"Message callback configuration: queueSize=%d, threadCount=%d",
self._callbacks_queue_size, self._callbacks_thread_count)
self._dxl_client = DxlClient(config)
logger.info("Attempting to connect to DXL fabric ...")
self._dxl_client.connect()
logger.info("Connected to DXL fabric.")
self.on_register_event_handlers()
self.on_register_services()
self.on_dxl_connect()
def run(self):
"""
Runs the application
"""
with self._lock:
if self._running:
raise Exception("The application is already running")
self._running = True
logger.info("Running application ...")
self.on_run()
self._validate_config_files()
self._load_configuration()
self._dxl_connect()
def destroy(self):
"""
Destroys the application (disconnects from fabric, frees resources, etc.)
"""
with self._lock:
if self._running and not self._destroyed:
logger.info("Destroying application ...")
if self._callbacks_pool is not None:
self._callbacks_pool.shutdown()
if self._dxl_client is not None:
self._unregister_services()
self._dxl_client.destroy()
self._dxl_client = None
self._destroyed = True
def _get_path(self, in_path):
"""
Returns an absolute path for a file specified in the configuration file (supports
files relative to the configuration file).
:param in_path: The specified path
:return: An absolute path for a file specified in the configuration file
"""
if not os.path.isfile(in_path) and not os.path.isabs(in_path):
config_rel_path = os.path.join(self._config_dir, in_path)
if os.path.isfile(config_rel_path):
in_path = config_rel_path
return in_path
def _unregister_services(self):
"""
Unregisters the services associated with the Application from the fabric
"""
for service in self._services:
self._dxl_client.unregister_service_sync(
service, self.DXL_SERVICE_REGISTRATION_TIMEOUT)
def _get_callbacks_pool(self):
"""
Returns the thread pool used to invoke application-specific message callbacks
:return: The thread pool used to invoke application-specific message callbacks
"""
with self._lock:
if self._callbacks_pool is None:
self._callbacks_pool = ThreadPool(self._callbacks_queue_size,
self._callbacks_thread_count,
"CallbacksPool")
return self._callbacks_pool
def add_event_callback(self, topic, callback, separate_thread):
"""
Adds a DXL event message callback to the application.
:param topic: The topic to associate with the callback
:param callback: The event callback
:param separate_thread: Whether to invoke the callback on a thread other than the incoming message
thread (this is necessary if synchronous requests are made via DXL in this callback).
"""
if separate_thread:
callback = _ThreadedEventCallback(self._get_callbacks_pool(),
callback)
self._dxl_client.add_event_callback(topic, callback)
def add_request_callback(self, service, topic, callback, separate_thread):
"""
Adds a DXL request message callback to the application.
:param service: The service to associate the request callback with
:param topic: The topic to associate with the callback
:param callback: The request callback
:param separate_thread: Whether to invoke the callback on a thread other than the incoming message
thread (this is necessary if synchronous requests are made via DXL in this callback).
"""
if separate_thread:
callback = _ThreadedRequestCallback(self._get_callbacks_pool(),
callback)
service.add_topic(topic, callback)
def register_service(self, service):
"""
Registers the specified service with the fabric
:param service: The service to register with the fabric
"""
self._dxl_client.register_service_sync(
service, self.DXL_SERVICE_REGISTRATION_TIMEOUT)
self._services.append(service)
def on_run(self):
"""
Invoked when the application has started running.
"""
pass
def on_load_configuration(self, config):
"""
Invoked after the application-specific configuration has been loaded
:param config: The application-specific configuration
"""
pass
def on_dxl_connect(self):
"""
Invoked after the client associated with the application has connected
to the DXL fabric.
"""
pass
def on_register_event_handlers(self):
"""
Invoked when event handlers should be registered with the application
"""
pass
def on_register_services(self):
"""
Invoked when services should be registered with the application
"""
pass