class TIEOutputBot(Bot):
def init(self):
if DxlClient is None:
raise ValueError(
"Could not import 'dxlclient'. Please install it.")
self.config = DxlClientConfig.create_dxl_config_from_file(
self.parameters.dxl_config_file)
self.dxlclient = DxlClient(self.config)
def process(self):
event = self.receive_message()
payload = json.dumps(event)
self.dxlclient.connect()
tie_client = TieClient(self.dxlclient)
tie_client.set_file_reputation(
TrustLevel.MOST_LIKELY_MALICIOUS, {
HashType.SHA256: event.get("malware.hash.sha256"),
HashType.SHA1: event.get("malware.hash.sha1"),
HashType.MD5: event.get("malware.hash.md5")
},
filename=event.get("malware.name"),
comment=self.parameters.comment)
self.dxlclient.disconnect()
self.logger.info("Event successfully sent.")
self.acknowledge_message()
class EventPublisher(DxlPublisherInterface):
def __init__(self):
self.client = None
def connect(self, config_file="./dxlclient.config"):
if self.isConnected():
raise DxlJythonException(
1100, "Already connected to the OpenDXL broker")
try:
logger.info("Reading configuration file from '%s'", config_file)
config = DxlClientConfig.create_dxl_config_from_file(config_file)
# Initialize DXL client using our configuration
self.client = DxlClient(config)
# Connect to DXL Broker
self.client.connect()
return
except Exception as e:
logger.info("Exception: " + e.message)
raise DxlJythonException(
1000, "Unable to establish a connection with the DXL broker")
def sendMessage(self,
topic="/dsa/dxl/test/event2",
message="Default message"):
if not self.isConnected():
raise DxlJythonException(1200, "Not connected to a OpenDXL broker")
try:
event = Event(topic)
# Encode string payload as UTF-8
event.payload = message.encode()
# Send event on DXL
logger.info("Sending '" + message + "' to '" + topic + "'")
self.client.send_event(event)
return "Event successfully posted to topic '%s'" % topic
except Exception as e:
logger.info("Exception: " + e.message)
raise DxlJythonException(
1010, "Unable to communicate with a DXL broker")
def disconnect(self):
if not self.isConnected():
return
self.client.disconnect()
def isConnected(self):
if self.client is None:
return False
return self.client.connected
class CuckooDXLClient(object):
"""
Wrapper class for the OpenDXL Python Client.
"""
def __init__(self):
"""
Constructor.
"""
self.client = None
self.config = None
self.enabled = None
def init(self):
"""
Initialization method to determine if the OpenDXL Cuckoo Reporting Module is enabled and to get the OpenDXL
Python Client config file location from the dxl_client_config_file setting under the [dxleventreporting]
section of the reporting.conf file.
:return: A boolean indicating if the OpenDXL Cuckoo Reporting Module is enabled or not.
"""
self.enabled = config("reporting:dxleventreporting:enabled")
dxl_client_config_file = config(
"reporting:dxleventreporting:dxl_client_config_file")
if dxl_client_config_file is None:
raise CuckooOperationalError(
"Missing dxl_client_config_file setting under the "
"[dxleventreporting] section in the report.conf file.")
self.config = DxlClientConfig.create_dxl_config_from_file(
dxl_client_config_file)
return self.enabled
def connect(self):
"""
A method to have the OpenDXL Python Client connect to a DXL Broker listed in the OpenDXL Python Client
configuration client.
"""
if not self.enabled:
return
try:
log.info("Creating DXL Client")
if not self.client:
self.client = DxlClient(self.config)
self.client.connect()
elif not self.client.connected:
self.client.connect()
log.info("Connected to a broker")
except Exception as ex:
log.exception(
"Error creating DXL Client and connecting to a DXL Broker.")
raise CuckooOperationalError(
"Error creating DXL Client and connecting to a DXL Broker: %s"
% ex)
def _create_client_for_connection(self, client_id):
"""
Creates a DxlClient and stores it for the give client_id
:param client_id: the client_id for the DxlClient
"""
client = DxlClient(self.client_config)
client.connect()
logger.info("Initializing new dxl client for client_id: %s", client_id)
with self._client_dict_lock:
self._client_dict[client_id] = (client, datetime.datetime.now())
def selftest_function(opts):
options = opts.get(CONFIG_DATA_SECTION, {})
try:
try:
config_file = options.get("dxlclient_config")
if config_file is None:
log.error(
"dxlclient_config is not set. You must set this path to run this function"
)
raise ValueError(
"dxlclient_config is not set. You must set this path to run this function"
)
# Create configuration from file for DxlClient
dxlclient_config = DxlClientConfig.create_dxl_config_from_file(
config_file)
except AttributeError:
log.error(
"There is no [fn_mcafee_tie] section in the config file, "
"please set that by running resilient-circuits config -u")
raise AttributeError(
"[fn_mcafee_tie] section is not set in the config file")
dxlclient = DxlClient(dxlclient_config)
dxlclient.connect()
tie_client = TieClient(dxlclient)
if dxlclient.connected and tie_client:
state = 'success'
reason = None
else:
state = 'failure'
reason = 'authorization failure'
return {'state': state, 'reason': reason}
except Exception as exc:
return {'state': 'failure', 'reason': exc}
def _dxl_connect(self):
"""
Attempts to connect to the DXL fabric and register the ePO DXL service
"""
# Connect to fabric
config = DxlClientConfig.create_dxl_config_from_file(
self._dxlclient_config_path)
config.incoming_message_thread_pool_size = self._incoming_thread_count
config.incoming_message_queue_size = self._incoming_queue_size
logger.info(
"Incoming message configuration: queueSize={0}, threadCount={1}".
format(config.incoming_message_queue_size,
config.incoming_message_thread_pool_size))
client = DxlClient(config)
logger.info("Attempting to connect to DXL fabric ...")
client.connect()
logger.info("Connected to DXL fabric.")
try:
# Register service
service = ServiceRegistrationInfo(client, self.DXL_SERVICE_TYPE)
for request_topic in self._epo_by_topic:
service.add_topic(
str(request_topic),
_EpoRequestCallback(client, self._epo_by_topic))
logger.info("Registering service ...")
client.register_service_sync(service,
self.DXL_SERVICE_REGISTRATION_TIMEOUT)
logger.info("Service registration succeeded.")
except:
client.destroy()
raise
self._dxl_client = client
self._dxl_service = service
class MonitorModule(Module):
# Request topic for service registry queries
SERVICE_REGISTRY_QUERY_TOPIC = '/mcafee/service/dxl/svcregistry/query'
# Event topics for service registry changes
SERVICE_REGISTRY_REGISTER_EVENT_TOPIC = '/mcafee/event/dxl/svcregistry/register'
SERVICE_REGISTRY_UNREGISTER_EVENT_TOPIC = '/mcafee/event/dxl/svcregistry/unregister'
# How often(in seconds) to refresh the service list
SERVICE_UPDATE_INTERVAL = 60
# How long to retain clients without any keep alive before evicting them
CLIENT_RETENTION_MINUTES = 30
# A default SmartClient JSON response to show no results
NO_RESULT_JSON = u"""{response:{status:0,startRow:0,endRow:0,totalRows:0,data:[]}}"""
# Locks for the different dictionaries shared between Monitor Handlers
_service_dict_lock = threading.Lock()
_client_dict_lock = threading.Lock()
_web_socket_dict_lock = threading.Lock()
_pending_messages_lock = threading.Lock()
def __init__(self, app):
super(MonitorModule,
self).__init__(app, "monitor", "Fabric Monitor",
"/public/images/monitor.png", "monitor_layout")
# dictionary to store DXL Client instances unique to each "session"
self._client_dict = {}
# dictionary to store web sockets for each "session"
self._web_socket_dict = {}
# dictionary to store incoming messages for each "session"
self._pending_messages = {}
# dictionary to cache service state
self._services = {}
self._message_id_topics = {}
self._client_config = DxlClientConfig.create_dxl_config_from_file(
self.app.bootstrap_app.client_config_path)
# DXL Client to perform operations that are the same for all users(svc registry queries)
self._dxl_service_client = DxlClient(self._client_config)
self._dxl_service_client.connect()
self._dxl_service_client.add_event_callback(
MonitorModule.SERVICE_REGISTRY_REGISTER_EVENT_TOPIC,
_ServiceEventCallback(self))
self._dxl_service_client.add_event_callback(
MonitorModule.SERVICE_REGISTRY_UNREGISTER_EVENT_TOPIC,
_ServiceEventCallback(self))
self._refresh_all_services()
self._service_updater_thread = threading.Thread(
target=self._service_updater)
self._service_updater_thread.daemon = True
self._service_updater_thread.start()
self._dxl_client_cleanup_thread = threading.Thread(
target=self._cleanup_dxl_clients)
self._dxl_client_cleanup_thread.daemon = True
self._dxl_client_cleanup_thread.start()
@property
def handlers(self):
return [(r'/update_services', ServiceUpdateHandler, dict(module=self)),
(r'/subscriptions', SubscriptionsHandler, dict(module=self)),
(r'/messages', MessagesHandler, dict(module=self)),
(r'/send_message', SendMessageHandler, dict(module=self)),
(r'/websocket', ConsoleWebSocketHandler, dict(module=self))]
@property
def content(self):
content = pkg_resources.resource_string(__name__,
"content.html").decode("utf8")
return content.replace("@PORT@", str(self.app.bootstrap_app.port))
@property
def services(self):
with self._service_dict_lock:
return self._services.copy()
@property
def message_id_topics(self):
return self._message_id_topics
@property
def client_config(self):
return DxlClientConfig.create_dxl_config_from_file(
self.app.bootstrap_app.client_config_path)
def get_dxl_client(self, client_id):
"""
Retrieves the DxlClient for the given request. If there is not one associated with
the incoming request it creates a new one and saves the generated client_id as a cookie
:param client_id: The client identifier
:return: the DxlClient specific to this "session"
"""
if not self._client_exists_for_connection(client_id):
self._create_client_for_connection(client_id)
with self._client_dict_lock:
client = self._client_dict[client_id][0]
if not client.connected:
client.connect()
logger.debug("Returning DXL client for id: %s", client_id)
return client
def _create_client_for_connection(self, client_id):
"""
Creates a DxlClient and stores it for the give client_id
:param client_id: the client_id for the DxlClient
"""
client = DxlClient(self.client_config)
client.connect()
logger.info("Initializing new dxl client for client_id: %s", client_id)
with self._client_dict_lock:
self._client_dict[client_id] = (client, datetime.datetime.now())
def _client_exists_for_connection(self, client_id):
"""
Checks if there is already an existing DxlClient for the given client_id.
:param client_id: the ID of the DxlClient to check for
:return: whether there is an existing DxlClient for this ID
"""
with self._client_dict_lock:
return client_id in self._client_dict
@staticmethod
def create_smartclient_response_wrapper():
"""
Creates a wrapper object containing the standard fields required by SmartClient responses
:return: an initial SmartClient response wrapper
"""
response_wrapper = {"response": {}}
response = response_wrapper["response"]
response["status"] = 0
response["startRow"] = 0
response["endRow"] = 0
response["totalRows"] = 0
response["data"] = []
return response_wrapper
def create_smartclient_error_response(self, error_message):
"""
Creates an error response for the SmartClient UI with the given message
:param error_message: The error message
:return: The SmartClient response in dict form
"""
response_wrapper = self.create_smartclient_response_wrapper()
response = response_wrapper["response"]
response["status"] = -1
response["data"] = error_message
return response
def queue_message(self, message, client_id):
"""
Adds the given message to the pending messages queue for the give client.
:param message: the message to enqueue
:param client_id: the client the message is intended for
"""
with self._pending_messages_lock:
if client_id not in self._pending_messages:
self._pending_messages[client_id] = []
self._pending_messages[client_id].append(message)
def get_messages(self, client_id):
"""
Retrieves the messages pending for the given client. This does not
clear the queue after retrieving.
:param client_id: the client to retrieve messages for
:return: a List of messages for the client
"""
with self._pending_messages_lock:
if client_id in self._pending_messages:
return self._pending_messages[client_id]
return None
def clear_messages(self, client_id):
"""
Clears the pending messages for the given client.
:param client_id: the client to clear messages for
"""
with self._pending_messages_lock:
self._pending_messages[client_id] = []
def _service_updater(self):
"""
A thread target that will run forever and do a complete refresh of the
service list on an interval or if the DXL client reconnects
"""
while True:
with self._dxl_service_client._connected_lock:
self._dxl_service_client._connected_wait_condition.wait(
self.SERVICE_UPDATE_INTERVAL)
if self._dxl_service_client.connected:
logger.debug("Refreshing service list.")
self._refresh_all_services()
def _cleanup_dxl_clients(self):
"""
A thread target that will run forever and evict DXL clients if their
clients have not sent a keep-alive
"""
logger.debug("DXL client cleanup thread initialized.")
while True:
with self._client_dict_lock:
for key in list(self._client_dict):
if self._client_dict[key][1] < \
(datetime.datetime.now() - datetime.timedelta(
minutes=self.CLIENT_RETENTION_MINUTES)):
logger.debug("Evicting DXL client for client_id: %s",
key)
del self._client_dict[key]
time.sleep(5)
def _refresh_all_services(self):
"""
Queries the broker for the service list and replaces the currently stored one with the new
results. Notifies all connected web sockets that new services are available.
"""
req = Request(MonitorModule.SERVICE_REGISTRY_QUERY_TOPIC)
req.payload = "{}"
# Send the request
dxl_response = self._dxl_service_client.sync_request(req, 5)
dxl_response_dict = MessageUtils.json_payload_to_dict(dxl_response)
logger.info("Service registry response: %s", dxl_response_dict)
with self._service_dict_lock:
self._services = {}
for service_guid in dxl_response_dict["services"]:
self._services[service_guid] = dxl_response_dict["services"][
service_guid]
self.notify_web_sockets()
def update_service(self, service_event):
"""
Replaces a stored service data withe the one from the provided DXL service event
:param service_event: the DXL service event containing the service
"""
with self._service_dict_lock:
self._services[service_event['serviceGuid']] = service_event
def remove_service(self, service_event):
"""
Removes a stored service using the provided DXL service event
:param service_event: the DXL service event containing the service to be removed
"""
with self._service_dict_lock:
if service_event['serviceGuid'] in self._services:
del self._services[service_event['serviceGuid']]
def client_keep_alive(self, client_id):
logger.debug("Client keep-alive received for client id: %s", client_id)
if self._client_exists_for_connection(client_id):
with self._client_dict_lock:
self._client_dict[client_id] = (
self._client_dict[client_id][0], datetime.datetime.now())
@property
def io_loop(self):
"""
Returns the Tornado IOLoop that the web console uses
:return: The Tornado IOLoop instance
"""
return self.app.io_loop
def add_web_socket(self, client_id, web_socket):
"""
Stores a web socket associated with the given client id
:param client_id: the client id key the web socket to
:param web_socket: the web socket to store
"""
logger.debug("Adding web socket for client: %s", client_id)
with self._web_socket_dict_lock:
self._web_socket_dict[client_id] = web_socket
def remove_web_socket(self, client_id):
"""
Removes any web socket associated with the given client_id
:param client_id: The client ID
"""
logger.debug("Removing web socket for client: %s", client_id)
with self._web_socket_dict_lock:
self._web_socket_dict.pop(client_id, None)
def notify_web_sockets(self):
"""
Notifies all web sockets that there are pending service updates
"""
with self._web_socket_dict_lock:
for key in self._web_socket_dict:
try:
self.io_loop.add_callback(
self._web_socket_dict[key].write_message,
u"serviceUpdates")
except Exception:
pass
def get_message_topic(self, message):
"""
Determines the topic for the provided message. Replaces the response channel in
responses with the topic of the original request
:param message: The DXL message
:return: The topic to use
"""
if (message.message_type == Message.MESSAGE_TYPE_RESPONSE or
message.message_type == Message.MESSAGE_TYPE_ERROR) and \
message.request_message_id in self.message_id_topics:
topic = self.message_id_topics[message.request_message_id]
del self.message_id_topics[message.request_message_id]
else:
topic = message.destination_topic
return topic
class FunctionComponent(ResilientComponent):
"""Component that implements Resilient function 'mcafee_publish_to_dxl"""
config_file = "dxlclient_config"
def __init__(self, opts):
"""constructor provides access to the configuration options"""
super(FunctionComponent, self).__init__(opts)
try:
self.config = opts.get("fn_mcafee_opendxl").get(self.config_file)
if self.config is None:
log.error(
self.config_file +
" is not set. You must set this path to run this function")
raise ValueError(
self.config_file +
" is not set. You must set this path to run this function")
# Create configuration from file for DxlClient
config = DxlClientConfig.create_dxl_config_from_file(self.config)
# Create client
self.client = DxlClient(config)
self.client.connect()
except AttributeError:
log.error(
"There is no [fn_mcafee_opendxl] section in the config file,"
"please set that by running resilient-circuits config -u")
raise AttributeError(
"[fn_mcafee_opendxl] section is not set in the config file")
@handler("reload")
def _reload(self, event, opts):
"""Configuration options have changed, save new values"""
self.config = opts.get("fn_mcafee_opendxl").get(self.config_file)
@function("mcafee_publish_to_dxl")
def _mcafee_publish_to_dxl_function(self, event, *args, **kwargs):
"""Function: A function which takes 3 inputs:
mcafee_topic_name: String of the topic name. ie: /mcafee/service/epo/remote/epo1.
mcafee_dxl_payload: The text of the payload to publish to the topic.
mcafee_return_request: Specify whether or not to wait for and return the response.
The function will publish the provided payload to the provided topic.
Indicate whether acknowledgment response should be returned."""
try:
yield StatusMessage("Starting...")
# Get the function parameters:
mcafee_topic_name = kwargs.get("mcafee_topic_name") # text
if not mcafee_topic_name:
yield FunctionError("mcafee_topic_name is required")
mcafee_dxl_payload = kwargs.get("mcafee_dxl_payload") # text
if not mcafee_dxl_payload:
yield FunctionError("mcafee_dxl_payload is required")
mcafee_publish_method = self.get_select_param(
kwargs.get("mcafee_publish_method")
) # select, values: "Event", "Service"
if not mcafee_publish_method:
yield FunctionError("mcafee_publish_method is required")
mcafee_wait_for_response = self.get_select_param(
kwargs.get(
"mcafee_wait_for_response")) # select, values: "Yes", "No"
log.info("mcafee_topic_name: %s", mcafee_topic_name)
log.info("mcafee_dxl_payload: %s", mcafee_dxl_payload)
log.info("mcafee_publish_method: %s", mcafee_publish_method)
log.info("mcafee_wait_for_response: %s", mcafee_wait_for_response)
response = None
# Publish Event
if mcafee_publish_method == "Event":
event = Event(mcafee_topic_name)
event.payload = mcafee_dxl_payload
yield StatusMessage("Publishing Event...")
self.client.send_event(event)
# Invoke Service
else:
req = Request(mcafee_topic_name)
req.payload = mcafee_dxl_payload
yield StatusMessage("Invoking Service...")
if mcafee_wait_for_response == "No":
self.client.async_request(req)
else:
response = Response(
self.client.sync_request(req, timeout=300))
yield StatusMessage("Done...")
r = {
"mcafee_topic_name": mcafee_topic_name,
"mcafee_dxl_payload": mcafee_dxl_payload,
"mcafee_publish_method": mcafee_publish_method,
"mcafee_wait_for_response": mcafee_wait_for_response
}
# Return response from publishing to topic
if response is not None:
r["response"] = vars(response)
yield FunctionResult(r)
else:
yield FunctionResult(r)
except Exception as e:
yield FunctionError(e)
class FunctionComponent(ResilientComponent):
"""Component that implements Resilient function 'mcafee_tie_search_hash"""
config_file = "dxlclient_config"
def __init__(self, opts):
"""constructor provides access to the configuration options"""
super(FunctionComponent, self).__init__(opts)
try:
config = opts.get("fn_mcafee_tie").get(self.config_file)
if config is None:
LOG.error(self.config_file + " is not set. You must set this path to run this function")
raise ValueError(self.config_file + " is not set. You must set this path to run this function")
LOG.info("Using %s to create configuration for DxlClient", config)
# Create configuration from file for DxlClient
self.config = DxlClientConfig.create_dxl_config_from_file(config)
except AttributeError:
LOG.error("There is no [fn_mcafee_tie] section in the config file, "
"please set that by running resilient-circuits config -u")
raise AttributeError("[fn_mcafee_tie] section is not set in the config file")
self.client = DxlClient(self.config)
self._connect_client()
def _connect_client(self):
# Connect client
if not self.client.connected:
self.client.connect()
self.tie_client = TieClient(self.client)
@handler("reload")
def _reload(self, event, opts):
"""Configuration options have changed, save new values"""
self.options = opts.get("fn_mcafee_tie", {})
@function("mcafee_tie_search_hash")
def _mcafee_tie_search_hash_function(self, event, *args, **kwargs):
"""Function: """
yield StatusMessage("Searching Hash...")
try:
response_dict = {}
# Get the function parameters:
mcafee_tie_hash_type = kwargs.get("mcafee_tie_hash_type") # text
if not mcafee_tie_hash_type:
yield FunctionError("mcafee_tie_hash_type is required")
mcafee_tie_hash = kwargs.get("mcafee_tie_hash") # text
if not mcafee_tie_hash:
yield FunctionError("mcafee_tie_hash is required")
LOG.debug("_lookup_hash started for Artifact Type {0} - Artifact Value {1}".format(
mcafee_tie_hash_type, mcafee_tie_hash))
if mcafee_tie_hash_type == "md5":
resilient_hash = {HashType.MD5: mcafee_tie_hash}
elif mcafee_tie_hash_type == "sha1":
resilient_hash = {HashType.SHA1: mcafee_tie_hash}
elif mcafee_tie_hash_type == "sha256":
resilient_hash = {HashType.SHA256: mcafee_tie_hash}
else:
yield FunctionError("Something went wrong setting the hash value")
# Make sure client is connected
self._connect_client()
reputations_dict = \
self.tie_client.get_file_reputation(
resilient_hash
)
system_list = self.tie_client.get_file_first_references(
resilient_hash
)
response_dict["Enterprise"] = self._get_enterprise_info(reputations_dict)
response_dict["GTI"] = self._get_gti_info(reputations_dict)
response_dict["ATD"] = self._get_atd_info(reputations_dict)
response_dict["MWG"] = self._get_mwg_info(reputations_dict)
response_dict["system_list"] = system_list
yield StatusMessage("Done...")
# Produce a FunctionResult with the return value
yield FunctionResult(response_dict)
except Exception as err:
yield FunctionError(err)
def _get_enterprise_info(self, reputations_dict):
ent_dict = {}
# Information for Enterprise file provider
if FileProvider.ENTERPRISE in reputations_dict:
ent_rep = reputations_dict[FileProvider.ENTERPRISE]
ent_dict["File Provider"] = "Enterprise"
ent_dict["Create Date"] = EpochMixin.to_localtime_string(ent_rep[ReputationProp.CREATE_DATE])
trust_level = self._get_trust_level(ent_rep[ReputationProp.TRUST_LEVEL])
if trust_level:
ent_dict["Trust Level"] = trust_level
# Retrieve the enterprise reputation attributes
if ReputationProp.ATTRIBUTES in ent_rep:
ent_rep_attribs = ent_rep[ReputationProp.ATTRIBUTES]
attribs_dict = {}
if FileEnterpriseAttrib.PREVALENCE in ent_rep_attribs:
attribs_dict["Prevalence"] = ent_rep_attribs[FileEnterpriseAttrib.PREVALENCE]
if FileEnterpriseAttrib.ENTERPRISE_SIZE in ent_rep_attribs:
attribs_dict["Enterprise Size"] = ent_rep_attribs[FileEnterpriseAttrib.ENTERPRISE_SIZE]
if FileEnterpriseAttrib.FIRST_CONTACT in ent_rep_attribs:
attribs_dict["First Contact"] = FileEnterpriseAttrib.to_localtime_string(
ent_rep_attribs[FileEnterpriseAttrib.FIRST_CONTACT])
if FileEnterpriseAttrib.PARENT_AVG_LOCAL_REP in ent_rep_attribs:
attribs_dict["Parent Avg Local Rep"] = self._get_trust_level(int(ent_rep_attribs[
FileEnterpriseAttrib.PARENT_AVG_LOCAL_REP]))
if FileEnterpriseAttrib.PARENT_FILE_REPS in ent_rep_attribs:
attribs_dict["Parent File Reps"] = FileEnterpriseAttrib.to_aggregate_tuple(ent_rep_attribs[
FileEnterpriseAttrib.PARENT_FILE_REPS])
if FileEnterpriseAttrib.CHILD_FILE_REPS in ent_rep_attribs:
attribs_dict["Child File Reps"] = FileEnterpriseAttrib.to_aggregate_tuple(ent_rep_attribs[
FileEnterpriseAttrib.CHILD_FILE_REPS])
if FileEnterpriseAttrib.AVG_LOCAL_REP in ent_rep_attribs:
attribs_dict["Average Local Rep"] = self._get_trust_level(int(ent_rep_attribs[
FileEnterpriseAttrib.AVG_LOCAL_REP]))
if FileEnterpriseAttrib.MAX_LOCAL_REP in ent_rep_attribs:
attribs_dict["Max Local Rep"] = self._get_trust_level(int(ent_rep_attribs[
FileEnterpriseAttrib.MAX_LOCAL_REP]))
if FileEnterpriseAttrib.MIN_LOCAL_REP in ent_rep_attribs:
attribs_dict["Min Local Rep"] = self._get_trust_level(int(ent_rep_attribs[
FileEnterpriseAttrib.MIN_LOCAL_REP]))
if FileEnterpriseAttrib.DETECTION_COUNT in ent_rep_attribs:
attribs_dict["Detection Count"] = ent_rep_attribs[FileEnterpriseAttrib.DETECTION_COUNT]
if FileEnterpriseAttrib.FILE_NAME_COUNT in ent_rep_attribs:
attribs_dict["File Name Count"] = ent_rep_attribs[FileEnterpriseAttrib.FILE_NAME_COUNT]
if FileEnterpriseAttrib.LAST_DETECTION_TIME in ent_rep_attribs:
attribs_dict["Last Detection Time"] = FileEnterpriseAttrib.to_localtime_string(
ent_rep_attribs[FileEnterpriseAttrib.FIRST_CONTACT])
if FileEnterpriseAttrib.IS_PREVALENT in ent_rep_attribs:
attribs_dict["Is Prevalent"] = ent_rep_attribs[FileEnterpriseAttrib.IS_PREVALENT]
if FileEnterpriseAttrib.PARENT_MIN_LOCAL_REP in ent_rep_attribs:
attribs_dict["Parent Min Local Rep"] = self._get_trust_level(int(ent_rep_attribs[
FileEnterpriseAttrib.PARENT_MIN_LOCAL_REP]))
if FileEnterpriseAttrib.PARENT_MAX_LOCAL_REP in ent_rep_attribs:
attribs_dict["Parent Max Local Rep"] = self._get_trust_level(int(ent_rep_attribs[
FileEnterpriseAttrib.PARENT_MAX_LOCAL_REP]))
ent_dict["Attributes"] = attribs_dict
return ent_dict
def _get_gti_info(self, reputations_dict):
gti_dict = {}
# Information for GTI file provider
if FileProvider.GTI in reputations_dict:
gti_rep = reputations_dict[FileProvider.GTI]
gti_dict["File Provider"] = "GTI"
gti_dict["Create Date"] = EpochMixin.to_localtime_string(gti_rep[ReputationProp.CREATE_DATE])
trust_level = self._get_trust_level(gti_rep[ReputationProp.TRUST_LEVEL])
if trust_level:
gti_dict["Trust Level"] = trust_level
# Retrieve the GTI reputation attributes
if ReputationProp.ATTRIBUTES in gti_rep:
gti_rep_attribs = gti_rep[ReputationProp.ATTRIBUTES]
attribs_dict = {}
# Get prevalence (if it exists)
if FileGtiAttrib.PREVALENCE in gti_rep_attribs:
attribs_dict["Prevalence"] = gti_rep_attribs[FileGtiAttrib.PREVALENCE]
# Get First Contact Date (if it exists)
if FileGtiAttrib.FIRST_CONTACT in gti_rep_attribs:
attribs_dict["First Contact"] = EpochMixin.to_localtime_string(gti_rep_attribs[
FileGtiAttrib.FIRST_CONTACT])
gti_dict["Attributes"] = attribs_dict
return gti_dict
def _get_atd_info(self, reputations_dict):
atd_dict = {}
# Information for Advanced Threat Defense file provider
if FileProvider.ATD in reputations_dict:
atd_rep = reputations_dict[FileProvider.ATD]
atd_dict["File Provider"] = "ATD"
atd_dict["Create Date"] = EpochMixin.to_localtime_string(atd_rep[ReputationProp.CREATE_DATE])
trust_level = self._get_trust_level(atd_rep[ReputationProp.TRUST_LEVEL])
if trust_level:
atd_dict["Trust Level"] = trust_level
# Retrieve the ATD reputation attributes
if ReputationProp.ATTRIBUTES in atd_rep:
atd_rep_attribs = atd_rep[ReputationProp.ATTRIBUTES]
attribs_dict = {}
# Get Trust scores
if AtdAttrib.GAM_SCORE in atd_rep_attribs:
attribs_dict["GAM Score"] = self._get_atd_trust_level(atd_rep, AtdAttrib.GAM_SCORE)
if AtdAttrib.AV_ENGINE_SCORE in atd_rep_attribs:
attribs_dict["AV Engine Score"] = self._get_atd_trust_level(atd_rep, AtdAttrib.AV_ENGINE_SCORE)
if AtdAttrib.GAM_SCORE in atd_rep_attribs:
attribs_dict["Sandbox Score"] = self._get_atd_trust_level(atd_rep, AtdAttrib.SANDBOX_SCORE)
if AtdAttrib.GAM_SCORE in atd_rep_attribs:
attribs_dict["Verdict"] = self._get_atd_trust_level(atd_rep, AtdAttrib.VERDICT)
if AtdAttrib.BEHAVIORS in atd_rep_attribs:
attribs_dict["Behaviors"] = atd_rep_attribs[AtdAttrib.BEHAVIORS]
atd_dict["Attributes"] = attribs_dict
return atd_dict
def _get_mwg_info(self, reputations_dict):
mwg_dict = {}
# Information for file provider
if FileProvider.MWG in reputations_dict:
mwg_rep = reputations_dict[FileProvider.MWG]
mwg_dict["File Provider"] = "MWG"
mwg_dict["Create Date"] = EpochMixin.to_localtime_string(mwg_rep[ReputationProp.CREATE_DATE])
trust_level = self._get_trust_level(mwg_rep[ReputationProp.TRUST_LEVEL])
if trust_level:
mwg_dict["Trust Level"] = trust_level
return mwg_dict
@staticmethod
def _get_trust_level(trust_level_number):
trust_level = ""
if TrustLevel.KNOWN_TRUSTED_INSTALLER is trust_level_number:
trust_level = "Known Trusted Installer"
elif TrustLevel.KNOWN_TRUSTED is trust_level_number:
trust_level = "Known Trusted"
elif TrustLevel.MOST_LIKELY_TRUSTED is trust_level_number:
trust_level = "Most Likely Trusted"
elif TrustLevel.MIGHT_BE_TRUSTED is trust_level_number:
trust_level = "Might Be Trusted"
elif TrustLevel.UNKNOWN is trust_level_number:
trust_level = "Unknown"
elif TrustLevel.MIGHT_BE_MALICIOUS is trust_level_number:
trust_level = "Might be Malicious"
elif TrustLevel.MOST_LIKELY_MALICIOUS is trust_level_number:
trust_level = "Most Likely Malicious"
elif TrustLevel.KNOWN_MALICIOUS is trust_level_number:
trust_level = "Known Malicious"
elif TrustLevel.NOT_SET is trust_level_number:
trust_level = "Not Set"
return trust_level
@staticmethod
def _get_atd_trust_level(atd_rep, rep_provider):
trust_level = ""
if AtdTrustLevel.KNOWN_TRUSTED is atd_rep[rep_provider]:
trust_level = "Known Trusted"
elif AtdTrustLevel.MOST_LIKELY_TRUSTED is atd_rep[rep_provider]:
trust_level = "Most Likely Trusted"
elif AtdTrustLevel.MIGHT_BE_TRUSTED is atd_rep[rep_provider]:
trust_level = "Might Be Trusted"
elif AtdTrustLevel.UNKNOWN is atd_rep[rep_provider]:
trust_level = "Unknown"
elif AtdTrustLevel.MIGHT_BE_MALICIOUS is atd_rep[rep_provider]:
trust_level = "Might be Malicious"
elif AtdTrustLevel.MOST_LIKELY_MALICIOUS is atd_rep[rep_provider]:
trust_level = "Most Likely Malicious"
elif AtdTrustLevel.KNOWN_MALICIOUS is atd_rep[rep_provider]:
trust_level = "Known Malicious"
elif AtdTrustLevel.NOT_SET is atd_rep[rep_provider]:
trust_level = "Not Set"
return trust_level
class DxlComponentSubscriber(ResilientComponent):
"""Component that Subscribes to DXL Topics and maps data back into the Resilient Platform"""
def __init__(self, opts):
super(DxlComponentSubscriber, self).__init__(opts)
self.config = verify_config(opts)
add_methods_to_global()
# Create and connect DXL client
config_client_file = self.config.get("config_client")
dxl_config = DxlClientConfig.create_dxl_config_from_file(
config_client_file)
self.client = DxlClient(dxl_config)
self.client.connect()
# This gets run once to tell the subscriber to listen on defined topics
self.main()
def main(self):
if self.config["topic_listener_on"].lower() == "true":
log.info("Service Listener called")
self.event_subscriber(self.config)
else:
log.info(
"Event subscriber not listening. To turn on set topic_listener_on to True"
)
def event_subscriber(self, config):
try:
topic_template_dict = get_topic_template_dict(
config.get("custom_template_dir"))
class ResilientEventSubscriber(EventCallback):
def __init__(self, template):
super(ResilientEventSubscriber, self).__init__()
self.temp = template
def on_event(self, event):
message = event.payload.decode(encoding="UTF-8")
log.info("Event received payload: " + message)
message_dict = json.loads(message)
# Map values from topic to incident template to create new incident
inc_data = map_values(self.temp, message_dict)
# Create new Incident in Resilient
response = create_incident(
get_connected_resilient_client(config), inc_data)
log.info("Created incident {}".format(
str(response.get("id"))))
# Python 2.x solution
try:
for event_topic, template in topic_template_dict.iteritems():
self.client.add_event_callback(
event_topic.encode('ascii', 'ignore'),
ResilientEventSubscriber(
template.encode('ascii', 'ignore')))
log.info(
"Resilient DXL Subscriber listening on {} ...".format(
event_topic))
# Python 3.6 solution
except AttributeError:
for event_topic, template in topic_template_dict.items():
# Ensure unicode and bytes are converted to String
self.client.add_event_callback(
event_topic.encode('ascii', 'ignore').decode("utf-8"),
ResilientEventSubscriber(
template.encode('ascii',
'ignore').decode("utf-8")))
log.info(
"Resilient DXL Subscriber listening on {} ...".format(
event_topic))
except Exception as e:
log.error(e)
self.client.destroy()
class Application(object):
"""
Base class used for DXL applications.
"""
# The name of the DXL client configuration file
DXL_CLIENT_CONFIG_FILE = "dxlclient.config"
# The location of the logging configuration file (optional)
LOGGING_CONFIG_FILE = "logging.config"
# The timeout used when registering/unregistering the service
DXL_SERVICE_REGISTRATION_TIMEOUT = 60
# The name of the "IncomingMessagePool" section within the configuration file
INCOMING_MESSAGE_POOL_CONFIG_SECTION = "IncomingMessagePool"
# The name of the "MessageCallbackPool" section within the configuration file
MESSAGE_CALLBACK_POOL_CONFIG_SECTION = "MessageCallbackPool"
# The property used to specify a queue size
QUEUE_SIZE_CONFIG_PROP = "queueSize"
# The property used to specify a thread count
THREAD_COUNT_CONFIG_PROP = "threadCount"
# The default thread count for the incoming message pool
DEFAULT_THREAD_COUNT = 10
# The default queue size for the incoming message pool
DEFAULT_QUEUE_SIZE = 1000
# The directory containing the configuration files (in the Python library)
LIB_CONFIG_DIR = "_config"
# The directory containing the application configuration files (in the Python library)
LIB_APP_CONFIG_DIR = LIB_CONFIG_DIR + "/app"
def __init__(self, config_dir, app_config_file_name):
"""
Constructs the application
:param config_dir: The directory containing the application configuration files
:param app_config_file_name: The name of the application-specific configuration file
"""
self._config_dir = config_dir
self._dxlclient_config_path = os.path.join(config_dir,
self.DXL_CLIENT_CONFIG_FILE)
self._app_config_path = os.path.join(config_dir, app_config_file_name)
self._epo_by_topic = {}
self._dxl_client = None
self._running = False
self._destroyed = False
self._services = []
self._incoming_thread_count = self.DEFAULT_THREAD_COUNT
self._incoming_queue_size = self.DEFAULT_QUEUE_SIZE
self._callbacks_pool = None
self._callbacks_thread_count = self.DEFAULT_THREAD_COUNT
self._callbacks_queue_size = self.DEFAULT_QUEUE_SIZE
self._config = None
self._lock = RLock()
def __del__(self):
"""destructor"""
self.destroy()
def __enter__(self):
"""Enter with"""
return self
def __exit__(self, exc_type, exc_value, traceback):
"""Exit with"""
self.destroy()
def _validate_config_files(self):
"""
Validates the configuration files necessary for the application. An exception is thrown
if any of the required files are inaccessible.
"""
# Determine the module of the derived class
mod = self.__class__.__module__
# If the configuration directory exists in the library, create config files as necessary
# This check also provides backwards compatibility for projects that don't have the
# configuration files in the library.
if pkg_resources.resource_exists(mod, self.LIB_CONFIG_DIR):
# Create configuration directory if not found
if not os.access(self._config_dir, os.R_OK):
logger.info(
"Configuration directory '%s' not found, creating...",
self._config_dir)
os.makedirs(self._config_dir)
# Count of current configuration files
config_files_count = len([
name for name in os.listdir(self._config_dir)
if os.path.isfile(os.path.join(self._config_dir, name))
])
# Create configuration files if not found
files = pkg_resources.resource_listdir(mod,
self.LIB_APP_CONFIG_DIR)
for file_name in files:
config_path = os.path.join(self._config_dir, file_name)
if not os.access(config_path, os.R_OK):
resource_filename = pkg_resources.resource_filename(
mod, self.LIB_APP_CONFIG_DIR + "/" + file_name)
f_lower = file_name.lower()
# Copy configuration file. Only copy logging file if the
# directory was empty
if not os.path.isdir(resource_filename) and \
not(f_lower.endswith(".py")) and \
not(f_lower.endswith(".pyc")) and \
(f_lower != Application.LOGGING_CONFIG_FILE or
config_files_count == 0):
logger.info(
"Configuration file '%s' not found, creating...",
file_name)
shutil.copyfile(
pkg_resources.resource_filename(
mod,
self.LIB_APP_CONFIG_DIR + "/" + file_name),
config_path)
if not os.access(self._dxlclient_config_path, os.R_OK):
raise Exception(
"Unable to access client configuration file: {0}".format(
self._dxlclient_config_path))
if not os.access(self._app_config_path, os.R_OK):
raise Exception(
"Unable to access application configuration file: {0}".format(
self._app_config_path))
def _load_configuration(self):
"""
Loads the configuration settings from the application-specific configuration file
"""
config = ConfigParser()
self._config = config
read_files = config.read(self._app_config_path)
if len(read_files) is not 1:
raise Exception(
"Error attempting to read application configuration file: {0}".
format(self._app_config_path))
#
# Load incoming pool settings
#
# pylint: disable=bare-except
try:
self._incoming_queue_size = config.getint(
self.INCOMING_MESSAGE_POOL_CONFIG_SECTION,
self.QUEUE_SIZE_CONFIG_PROP)
except:
pass
try:
self._incoming_thread_count = config.getint(
self.INCOMING_MESSAGE_POOL_CONFIG_SECTION,
self.THREAD_COUNT_CONFIG_PROP)
except:
pass
#
# Load callback pool settings
#
try:
self._callbacks_queue_size = config.getint(
self.MESSAGE_CALLBACK_POOL_CONFIG_SECTION,
self.QUEUE_SIZE_CONFIG_PROP)
except:
pass
try:
self._callbacks_thread_count = config.getint(
self.MESSAGE_CALLBACK_POOL_CONFIG_SECTION,
self.THREAD_COUNT_CONFIG_PROP)
except:
pass
self.on_load_configuration(config)
def _dxl_connect(self):
"""
Attempts to connect to the DXL fabric
"""
# Connect to fabric
config = DxlClientConfig.create_dxl_config_from_file(
self._dxlclient_config_path)
config.incoming_message_thread_pool_size = self._incoming_thread_count
config.incoming_message_queue_size = self._incoming_queue_size
logger.info(
"Incoming message configuration: queueSize=%d, threadCount=%d",
config.incoming_message_queue_size,
config.incoming_message_thread_pool_size)
logger.info(
"Message callback configuration: queueSize=%d, threadCount=%d",
self._callbacks_queue_size, self._callbacks_thread_count)
self._dxl_client = DxlClient(config)
logger.info("Attempting to connect to DXL fabric ...")
self._dxl_client.connect()
logger.info("Connected to DXL fabric.")
self.on_register_event_handlers()
self.on_register_services()
self.on_dxl_connect()
def run(self):
"""
Runs the application
"""
with self._lock:
if self._running:
raise Exception("The application is already running")
self._running = True
logger.info("Running application ...")
self.on_run()
self._validate_config_files()
self._load_configuration()
self._dxl_connect()
def destroy(self):
"""
Destroys the application (disconnects from fabric, frees resources, etc.)
"""
with self._lock:
if self._running and not self._destroyed:
logger.info("Destroying application ...")
if self._callbacks_pool is not None:
self._callbacks_pool.shutdown()
if self._dxl_client is not None:
self._unregister_services()
self._dxl_client.destroy()
self._dxl_client = None
self._destroyed = True
def _get_path(self, in_path):
"""
Returns an absolute path for a file specified in the configuration file (supports
files relative to the configuration file).
:param in_path: The specified path
:return: An absolute path for a file specified in the configuration file
"""
if not os.path.isfile(in_path) and not os.path.isabs(in_path):
config_rel_path = os.path.join(self._config_dir, in_path)
if os.path.isfile(config_rel_path):
in_path = config_rel_path
return in_path
def _unregister_services(self):
"""
Unregisters the services associated with the Application from the fabric
"""
for service in self._services:
self._dxl_client.unregister_service_sync(
service, self.DXL_SERVICE_REGISTRATION_TIMEOUT)
def _get_callbacks_pool(self):
"""
Returns the thread pool used to invoke application-specific message callbacks
:return: The thread pool used to invoke application-specific message callbacks
"""
with self._lock:
if self._callbacks_pool is None:
self._callbacks_pool = ThreadPool(self._callbacks_queue_size,
self._callbacks_thread_count,
"CallbacksPool")
return self._callbacks_pool
def add_event_callback(self, topic, callback, separate_thread):
"""
Adds a DXL event message callback to the application.
:param topic: The topic to associate with the callback
:param callback: The event callback
:param separate_thread: Whether to invoke the callback on a thread other than the incoming message
thread (this is necessary if synchronous requests are made via DXL in this callback).
"""
if separate_thread:
callback = _ThreadedEventCallback(self._get_callbacks_pool(),
callback)
self._dxl_client.add_event_callback(topic, callback)
def add_request_callback(self, service, topic, callback, separate_thread):
"""
Adds a DXL request message callback to the application.
:param service: The service to associate the request callback with
:param topic: The topic to associate with the callback
:param callback: The request callback
:param separate_thread: Whether to invoke the callback on a thread other than the incoming message
thread (this is necessary if synchronous requests are made via DXL in this callback).
"""
if separate_thread:
callback = _ThreadedRequestCallback(self._get_callbacks_pool(),
callback)
service.add_topic(topic, callback)
def register_service(self, service):
"""
Registers the specified service with the fabric
:param service: The service to register with the fabric
"""
self._dxl_client.register_service_sync(
service, self.DXL_SERVICE_REGISTRATION_TIMEOUT)
self._services.append(service)
def on_run(self):
"""
Invoked when the application has started running.
"""
pass
def on_load_configuration(self, config):
"""
Invoked after the application-specific configuration has been loaded
:param config: The application-specific configuration
"""
pass
def on_dxl_connect(self):
"""
Invoked after the client associated with the application has connected
to the DXL fabric.
"""
pass
def on_register_event_handlers(self):
"""
Invoked when event handlers should be registered with the application
"""
pass
def on_register_services(self):
"""
Invoked when services should be registered with the application
"""
pass
# Dump the dictionary
print json.dumps(first_instance_dict,
sort_keys=True, indent=4, separators=(',', ': '))
agentGuid = json.dumps(first_instance_dict['agentGuid'])[1:-1]
fileName = json.dumps(first_instance_dict['name'])[1:-1]
sha1 = json.dumps(first_instance_dict['hashes']['sha1'])[1:-1]
print agentGuid, fileName, sha1
searchHASH(sha1,agentGuid,fileName)
# Create the client
client1 = DxlClient(config)
client2 = DxlClient(config)
# Connect to the fabric
client1.connect()
client2.connect()
# Create the McAfee Active Response (MAR) client
mar_client = MarClient(client1)
# Create the McAfee Threat Intelligence Exchange (TIE) client
tie_client = TieClient(client2)
# Create first instance callback
first_instance_callback = MyFirstInstanceCallback()
# Register first instance callback with the client
tie_client.add_file_first_instance_callback(first_instance_callback)
def searchHASH(sha1,agentGuid,fileName):
class EventSender:
TRUST_LEVEL = {
'NOT_SET': '0',
'KNOWN_MALICIOUS': '1',
'MOST_LIKELY_MALICIOUS': '15',
'MIGHT_BE_MALICIOUS': '30',
'UNKNOWN': '50',
'MIGHT_BE_TRUSTED': '70',
'MOST_LIKELY_TRUSTED': '85',
'KNOWN_TRUSTED': '99',
'KNOWN_TRUSTED_INSTALLER': '100'
}
broker_ca_bundle = tempfile.NamedTemporaryFile().name
cert_file = tempfile.NamedTemporaryFile().name
private_key = tempfile.NamedTemporaryFile().name
def __init__(self, params: Dict):
with open(self.broker_ca_bundle, "w") as text_file:
text_file.write(params['broker_ca_bundle'])
with open(self.cert_file, "w") as text_file:
text_file.write(params['cert_file'])
with open(self.private_key, "w") as text_file:
text_file.write(params['private_key'])
if 'broker_urls' in params:
self.broker_urls = params['broker_urls'].split(',')
self.push_ip_topic = params.get('push_ip_topic')
self.push_url_topic = params.get('push_url_topic')
self.push_domain_topic = params.get('push_domain_topic')
self.push_hash_topic = params.get('push_hash_topic')
self.client = DxlClient(self.get_client_config())
self.client.connect()
def __del__(self):
self.client.disconnect()
def push_ip(self, ip, trust_level, topic):
if not is_ip_valid(ip):
raise ValueError(f'argument ip {ip} is not a valid IP')
trust_level_key = self.TRUST_LEVEL[trust_level]
if topic:
self.push_ip_topic = topic
self.send_event(self.push_ip_topic,
f'ip:{ip};trust_level:{trust_level_key}')
return f'Successfully pushed ip {ip} with trust level {trust_level}'
def push_url(self, url, trust_level, topic):
trust_level_key = self.TRUST_LEVEL[trust_level]
if topic:
self.push_url_topic = topic
self.send_event(self.push_url_topic,
f'url:{url};trust_level:{trust_level_key}')
return f'Successfully pushed url {url} with trust level {trust_level}'
def push_domain(self, domain, trust_level, topic):
trust_level_key = self.TRUST_LEVEL[trust_level]
if topic:
self.push_domain_topic = topic
self.send_event(self.push_domain_topic,
f'domain:{domain};trust_level:{trust_level_key}')
return f'Successfully pushed domain {domain} with trust level {trust_level}'
def push_hash(self, hash_obj, trust_level, topic):
trust_level_key = self.TRUST_LEVEL[trust_level]
if topic:
self.push_ip_topic = topic
self.send_event(self.push_hash_topic,
f'hash:{hash_obj};trust_level:{trust_level_key}')
return f'Successfully pushed hash {hash_obj} with trust level {trust_level}'
def get_client_config(self):
config = DxlClientConfig(
broker_ca_bundle=self.broker_ca_bundle,
cert_file=self.cert_file,
private_key=self.private_key,
brokers=[Broker.parse(url) for url in self.broker_urls])
config.connect_retries = CONNECT_RETRIES
config.reconnect_delay = RECONNECT_DELAY
config.reconnect_delay_max = RECONNECT_DELAY_MAX
return config
def send_event(self, topic, payload):
if not topic:
raise Exception(
f'Error in {demisto.command()} topic field is required')
event = Event(topic)
event.payload = str(payload).encode()
self.client.send_event(event)
def send_event_wrapper(self, topic, payload):
self.send_event(topic, payload)
return 'Successfully sent event'
class ServiceRequester(DxlRequesterInterface):
def __init__(self):
self.client = None
def connect(self, config_file="./dxlclient.config"):
if self.isConnected():
raise DxlJythonException(1100, "Already connected to the OpenDXL broker")
try:
logger.info("Reading configuration file from '%s'", config_file)
config = DxlClientConfig.create_dxl_config_from_file(config_file)
# Initialize DXL client using our configuration
self.client = DxlClient(config)
# Connect to DXL Broker
self.client.connect()
return
except Exception as e:
logger.info("Exception: " + e.message)
raise DxlJythonException(1000, "Unable to establish a connection with the DXL broker")
def sendMessage(self, topic="/dsa/dxl/test/event2", message="Default message"):
if not self.isConnected():
raise DxlJythonException(1200, "Not connected to a OpenDXL broker")
try:
request = Request(topic)
# Encode string payload as UTF-8
request.payload = message.encode()
# Send Synchronous Request with default timeout and wait for Response
logger.info("Requesting '" + message + "' from '" + topic + "'")
response = self.client.sync_request(request)
dxl_message = JavaDxlMessage()
dxl_message.setMessageVersion(response.version)
dxl_message.setMessageId(response.message_id)
dxl_message.setClientId(response.source_client_id)
dxl_message.setBrokerId(response.source_broker_id)
dxl_message.setMessageType(response.message_type)
dxl_message.setBrokerIdList(response.broker_ids)
dxl_message.setClientIdList(response.client_ids)
dxl_message.setRequestMessageId(response.request_message_id)
# Check that the Response is not an Error Response, then extract
if response.message_type != Message.MESSAGE_TYPE_ERROR:
dxl_message.setServiceId(response.service_id)
dxl_message.setPayload(response.payload.decode())
else:
dxl_message.setErrorCode(response.error_code)
dxl_message.setErrorMessage(response.error_message)
return dxl_message
except Exception as e:
logger.info("Exception: " + e.message)
raise DxlJythonException(1010, "Unable to communicate with a DXL broker")
def disconnect(self):
if not self.isConnected():
return
self.client.disconnect()
def isConnected(self):
if self.client is None:
return False;
return self.client.connected
import sys
import argparse
import json
from socket import gethostname
from pygtail import Pygtail
from time import sleep
from IPy import IP
from dxlclient.client import DxlClient
from dxlclient.client_config import DxlClientConfig
from common import *
from dxlclient.message import Message, Event
config = DxlClientConfig.create_dxl_config_from_file(CONFIG_FILE)
client = DxlClient(config)
client.connect()
def sendData(data):
try:
v = IP(data).version()
except Exception as e:
print("Error in '%s': %s" % (data, str(e)))
return
if v == 4:
evt = Event('/feed/bad/ipv4')
if v == 6:
evt = Event('/feed/bad/ipv6')
evt.payload = str(data).encode()
client.send_event(evt)
class McAfeeTieSearcher(BaseComponent):
"""
McAfee TIE custom threat searcher component
Test using 'curl':
curl -v -X OPTIONS 'http://127.0.0.1:9000/cts/mcafee_tie_searcher'
curl -v -k --header "Content-Type: application/json" --data-binary '{"type":"hash.md5",
"value":"1CF5B6CC0E6B742F6DEF8BF96D847A25"}' 'http://127.0.0.1:9000/cts/mcafee_tie_searcher'
curl -v -k --header "Content-Type: application/json" --data-binary '{"type":"hash.sha1",
"value":"19A82049C4336E8A5A30426FEC3F560358FAB6ED"}' 'http://127.0.0.1:9000/cts/mcafee_tie_searcher'
curl -v -k --header "Content-Type: application/json" --data-binary '{"type":"hash.sha256",
"value":"A8B03AD33BC6D7A7F376B943F763EEDD1CDAF125D012F9018F2F56678AE67EA4"}'
'http://127.0.0.1:9000/cts/mcafee_tie_searcher'
"""
# Register this as an async searcher for the URL /<root>/mcafee_tie_searcher
channel = searcher_channel("mcafee_tie_searcher")
config_file = "dxlclient_config"
def __init__(self, opts):
super(McAfeeTieSearcher, self).__init__(opts)
try:
config = opts.get("mcafee").get(self.config_file)
if config is None:
LOG.error(self.config_file + " is not set. You must set this path to run this threat service")
raise ValueError(self.config_file + " is not set. You must set this path to run this threat service")
# Create configuration from file for DxlClient
self.config = DxlClientConfig.create_dxl_config_from_file(config)
except AttributeError:
LOG.error("There is no [mcafee] section in the config file,"
"please set that by running resilient-circuits config -u")
raise AttributeError("[mcafee] section is not set in the config file")
# Create client
self.client = DxlClient(self.config)
self.client.connect()
# Handle lookup for artifacts of type md5, sha1, and sha256 hashes
@handler("hash.md5", "hash.sha1", "hash.sha256")
def _lookup_hash(self, event, *args, **kwargs):
artifact_type = event.artifact["type"]
artifact_value = event.artifact["value"]
LOG.debug("_lookup_hash started for Artifact Type {0} - Artifact Value {1}".format(
artifact_type, artifact_value))
tie_client = TieClient(self.client)
if artifact_type == "hash.md5":
resilient_hash = {HashType.MD5: artifact_value}
elif artifact_type == "hash.sha1":
resilient_hash = {HashType.SHA1: artifact_value}
elif artifact_type == "hash.sha256":
resilient_hash = {HashType.SHA256: artifact_value}
else:
raise ValueError("Something went wrong setting the hash value")
reputations_dict = \
tie_client.get_file_reputation(
resilient_hash
)
hits = self._query_mcafee_tie(reputations_dict)
yield hits
def _query_mcafee_tie(self, reputations_dict):
hit = Hit()
# Check Enterprise File Provider
hit = self._get_enterprise_info(reputations_dict, hit)
# Check GTI File Provider
hit = self._get_gti_info(reputations_dict, hit)
# Check ATD File Provider
hit = self._get_atd_info(reputations_dict, hit)
# Check MWG File Provider
hit = self._get_mwg_info(reputations_dict, hit)
# Verifies a trust level was set before returning a hit
for prop in hit["props"]:
if fnmatch(prop["name"], "*Trust Level"):
return hit
return []
def _get_enterprise_info(self, reputations_dict, hit):
# Information for Enterprise file provider
if FileProvider.ENTERPRISE in reputations_dict:
ent_rep = reputations_dict[FileProvider.ENTERPRISE]
trust_level = self._get_trust_level(ent_rep[ReputationProp.TRUST_LEVEL])
if trust_level:
# Not a hit until trust level has been verified to less than or equal to MIGHT BE MALICIOUS
hit.append(StringProp(name="Enterprise Trust Level", value=trust_level))
# Retrieve the enterprise reputation attributes
ent_rep_attribs = ent_rep[ReputationProp.ATTRIBUTES]
# Get Average Local Rep
if FileEnterpriseAttrib.AVG_LOCAL_REP in ent_rep_attribs:
local_rep = self._get_trust_level(int(ent_rep_attribs[FileEnterpriseAttrib.AVG_LOCAL_REP]))
if local_rep:
hit.append(StringProp(name="Enterprise Avg Local Trust Level", value=local_rep))
# Get prevalence (if it exists)
if FileEnterpriseAttrib.PREVALENCE in ent_rep_attribs:
hit.append(StringProp(name="Enterprise Prevalence",
value=ent_rep_attribs[FileEnterpriseAttrib.PREVALENCE]))
# Get Enterprise Size (if it exists)
if FileEnterpriseAttrib.ENTERPRISE_SIZE in ent_rep_attribs:
hit.append(StringProp(name="Enterprise Size",
value=ent_rep_attribs[FileEnterpriseAttrib.ENTERPRISE_SIZE]))
# Get First Contact Date (if it exists)
if FileEnterpriseAttrib.FIRST_CONTACT in ent_rep_attribs:
hit.append(StringProp(name="Enterprise First Contact",
value=FileEnterpriseAttrib.to_localtime_string(
ent_rep_attribs[FileEnterpriseAttrib.FIRST_CONTACT])))
return hit
def _get_gti_info(self, reputations_dict, hit):
# Information for GTI file provider
if FileProvider.GTI in reputations_dict:
gti_rep = reputations_dict[FileProvider.GTI]
trust_level = self._get_trust_level(gti_rep[ReputationProp.TRUST_LEVEL])
if trust_level:
# Not a hit until trust level has been verified to less than or equal to MIGHT BE MALICIOUS
hit.append(StringProp(name="GTI Trust Level", value=trust_level))
# Retrieve the GTI reputation attributes
gti_rep_attribs = gti_rep[ReputationProp.ATTRIBUTES]
# Get prevalence (if it exists)
if FileGtiAttrib.PREVALENCE in gti_rep_attribs:
hit.append(StringProp(name="GTI Prevalence", value=gti_rep_attribs[FileGtiAttrib.PREVALENCE]))
# Get First Contact Date (if it exists)
if FileGtiAttrib.FIRST_CONTACT in gti_rep_attribs:
hit.append(StringProp(name="GTI First Contact", value=EpochMixin.to_localtime_string(
gti_rep_attribs[FileGtiAttrib.FIRST_CONTACT])))
return hit
def _get_atd_info(self, reputations_dict, hit):
# Information for Advanced Threat Defense file provider
if FileProvider.ATD in reputations_dict:
atd_rep = reputations_dict[FileProvider.ATD]
trust_level = self._get_trust_level(atd_rep[ReputationProp.TRUST_LEVEL])
if trust_level:
# Not a hit until trust level has been verified to less than or equal to MIGHT BE MALICIOUS
hit.append(StringProp(name="ATD Trust Level", value=trust_level))
return hit
def _get_mwg_info(self, reputations_dict, hit):
# Information for file provider
if FileProvider.MWG in reputations_dict:
mwg_rep = reputations_dict[FileProvider.MWG]
trust_level = self._get_trust_level(mwg_rep[ReputationProp.TRUST_LEVEL])
if trust_level:
# Not a hit until trust level has been verified to less than or equal to MIGHT BE MALICIOUS
hit.append(StringProp(name="MWG Trust Level", value=trust_level))
return hit
@staticmethod
def _get_trust_level(trust_level_number):
trust_level = ""
if TrustLevel.MIGHT_BE_MALICIOUS is trust_level_number:
trust_level = "Might be Malicious"
elif TrustLevel.MOST_LIKELY_MALICIOUS is trust_level_number:
trust_level = "Most Likely Malicious"
elif TrustLevel.KNOWN_MALICIOUS is trust_level_number:
trust_level = "Known Malicious"
return trust_level
class Application(object):
"""
Base class used for DXL applications.
"""
# The name of the DXL client configuration file
DXL_CLIENT_CONFIG_FILE = "dxlclient.config"
# The timeout used when registering/unregistering the service
DXL_SERVICE_REGISTRATION_TIMEOUT = 60
# The name of the "IncomingMessagePool" section within the configuration file
INCOMING_MESSAGE_POOL_CONFIG_SECTION = "IncomingMessagePool"
# The name of the "MessageCallbackPool" section within the configuration file
MESSAGE_CALLBACK_POOL_CONFIG_SECTION = "MessageCallbackPool"
# The property used to specify a queue size
QUEUE_SIZE_CONFIG_PROP = "queueSize"
# The property used to specify a thread count
THREAD_COUNT_CONFIG_PROP = "threadCount"
# The default thread count for the incoming message pool
DEFAULT_THREAD_COUNT = 10
# The default queue size for the incoming message pool
DEFAULT_QUEUE_SIZE = 1000
def __init__(self, config_dir, app_config_file_name):
"""
Constructs the application
:param config_dir: The directory containing the application configuration files
:param app_config_file_name: The name of the application-specific configuration file
"""
self._config_dir = config_dir
self._dxlclient_config_path = os.path.join(config_dir,
self.DXL_CLIENT_CONFIG_FILE)
self._app_config_path = os.path.join(config_dir, app_config_file_name)
self._epo_by_topic = {}
self._dxl_client = None
self._running = False
self._destroyed = False
self._services = []
self._incoming_thread_count = self.DEFAULT_THREAD_COUNT
self._incoming_queue_size = self.DEFAULT_QUEUE_SIZE
self._callbacks_pool = None
self._callbacks_thread_count = self.DEFAULT_THREAD_COUNT
self._callbacks_queue_size = self.DEFAULT_QUEUE_SIZE
self._config = None
self._lock = RLock()
def __del__(self):
"""destructor"""
self.destroy()
def __enter__(self):
"""Enter with"""
return self
def __exit__(self, exc_type, exc_value, traceback):
"""Exit with"""
self.destroy()
def _validate_config_files(self):
"""
Validates the configuration files necessary for the application. An exception is thrown
if any of the required files are inaccessible.
"""
if not os.access(self._dxlclient_config_path, os.R_OK):
raise Exception(
"Unable to access client configuration file: {0}".format(
self._dxlclient_config_path))
if not os.access(self._app_config_path, os.R_OK):
raise Exception(
"Unable to access application configuration file: {0}".format(
self._app_config_path))
def _load_configuration(self):
"""
Loads the configuration settings from the application-specific configuration file
"""
config = ConfigParser()
self._config = config
read_files = config.read(self._app_config_path)
if len(read_files) is not 1:
raise Exception(
"Error attempting to read application configuration file: {0}".
format(self._app_config_path))
#
# Load incoming pool settings
#
try:
self._incoming_queue_size = config.getint(
self.INCOMING_MESSAGE_POOL_CONFIG_SECTION,
self.QUEUE_SIZE_CONFIG_PROP)
except:
pass
try:
self._incoming_thread_count = config.getint(
self.INCOMING_MESSAGE_POOL_CONFIG_SECTION,
self.THREAD_COUNT_CONFIG_PROP)
except:
pass
#
# Load callback pool settings
#
try:
self._callbacks_queue_size = config.getint(
self.MESSAGE_CALLBACK_POOL_CONFIG_SECTION,
self.QUEUE_SIZE_CONFIG_PROP)
except:
pass
try:
self._callbacks_thread_count = config.getint(
self.MESSAGE_CALLBACK_POOL_CONFIG_SECTION,
self.THREAD_COUNT_CONFIG_PROP)
except:
pass
self.on_load_configuration(config)
def _dxl_connect(self):
"""
Attempts to connect to the DXL fabric
"""
# Connect to fabric
config = DxlClientConfig.create_dxl_config_from_file(
self._dxlclient_config_path)
config.incoming_message_thread_pool_size = self._incoming_thread_count
config.incoming_message_queue_size = self._incoming_queue_size
logger.info(
"Incoming message configuration: queueSize={0}, threadCount={1}".
format(config.incoming_message_queue_size,
config.incoming_message_thread_pool_size))
logger.info(
"Message callback configuration: queueSize={0}, threadCount={1}".
format(self._callbacks_queue_size, self._callbacks_thread_count))
self._dxl_client = DxlClient(config)
logger.info("Attempting to connect to DXL fabric ...")
self._dxl_client.connect()
logger.info("Connected to DXL fabric.")
self.on_register_event_handlers()
self.on_register_services()
self.on_dxl_connect()
def run(self):
"""
Runs the application
"""
with self._lock:
if self._running:
raise Exception("The application is already running")
self._running = True
logger.info("Running application ...")
self.on_run()
self._validate_config_files()
self._load_configuration()
self._dxl_connect()
def destroy(self):
"""
Destroys the application (disconnects from fabric, frees resources, etc.)
"""
with self._lock:
if self._running and not self._destroyed:
logger.info("Destroying application ...")
if self._callbacks_pool is not None:
self._callbacks_pool.shutdown()
if self._dxl_client is not None:
self._unregister_services()
self._dxl_client.destroy()
self._dxl_client = None
self._destroyed = True
def _get_path(self, in_path):
"""
Returns an absolute path for a file specified in the configuration file (supports
files relative to the configuration file).
:param in_path: The specified path
:return: An absolute path for a file specified in the configuration file
"""
if not os.path.isfile(in_path) and not os.path.isabs(in_path):
config_rel_path = os.path.join(self._config_dir, in_path)
if os.path.isfile(config_rel_path):
in_path = config_rel_path
return in_path
def _unregister_services(self):
"""
Unregisters the services associated with the Application from the fabric
"""
for service in self._services:
self._dxl_client.unregister_service_sync(
service, self.DXL_SERVICE_REGISTRATION_TIMEOUT)
def _get_callbacks_pool(self):
"""
Returns the thread pool used to invoke application-specific message callbacks
:return: The thread pool used to invoke application-specific message callbacks
"""
with self._lock:
if self._callbacks_pool is None:
self._callbacks_pool = ThreadPool(self._callbacks_queue_size,
self._callbacks_thread_count,
"CallbacksPool")
return self._callbacks_pool
def add_event_callback(self, topic, callback, separate_thread):
"""
Adds a DXL event message callback to the application.
:param topic: The topic to associate with the callback
:param callback: The event callback
:param separate_thread: Whether to invoke the callback on a thread other than the incoming message
thread (this is necessary if synchronous requests are made via DXL in this callback).
"""
if separate_thread:
callback = _ThreadedEventCallback(self._get_callbacks_pool(),
callback)
self._dxl_client.add_event_callback(topic, callback)
def add_request_callback(self, service, topic, callback, separate_thread):
"""
Adds a DXL request message callback to the application.
:param service: The service to associate the request callback with
:param topic: The topic to associate with the callback
:param callback: The request callback
:param separate_thread: Whether to invoke the callback on a thread other than the incoming message
thread (this is necessary if synchronous requests are made via DXL in this callback).
"""
if separate_thread:
callback = _ThreadedRequestCallback(self._get_callbacks_pool(),
callback)
service.add_topic(topic, callback)
def register_service(self, service):
"""
Registers the specified service with the fabric
:param service: The service to register with the fabric
"""
self._dxl_client.register_service_sync(
service, self.DXL_SERVICE_REGISTRATION_TIMEOUT)
self._services.append(service)
def on_run(self):
"""
Invoked when the application has started running.
"""
pass
def on_load_configuration(self, config):
"""
Invoked after the application-specific configuration has been loaded
:param config: The application-specific configuration
"""
pass
def on_dxl_connect(self):
"""
Invoked after the client associated with the application has connected
to the DXL fabric.
"""
pass
def on_register_event_handlers(self):
"""
Invoked when event handlers should be registered with the application
"""
pass
def on_register_services(self):
"""
Invoked when services should be registered with the application
"""
pass
class DXLBroker(object):
class MyEventCallback(EventCallback):
def __init__(self, broker):
EventCallback.__init__(self)
self.broker = broker
def on_event(self, event):
self.broker.logger.threaddebug(
f"{self.broker.device.name}: Message {event.message_id} ({event.message_type}), received: {event.destination_topic}, payload: {event.payload}"
)
indigo.activePlugin.processReceivedMessage(self.broker.device.id,
event.destination_topic,
event.payload)
def __init__(self, device):
self.logger = logging.getLogger("Plugin.DXLBroker")
self.deviceID = device.id
address = device.pluginProps.get(u'address', "")
port = device.pluginProps.get(u'port', "")
ca_bundle = indigo.server.getInstallFolderPath(
) + '/' + device.pluginProps.get(u'ca_bundle', "")
cert_file = indigo.server.getInstallFolderPath(
) + '/' + device.pluginProps.get(u'cert_file', "")
private_key = indigo.server.getInstallFolderPath(
) + '/' + device.pluginProps.get(u'private_key', "")
self.logger.debug(
f"{device.name}: Broker __init__ address = {address}, ca_bundle = {ca_bundle}, cert_file = {cert_file}, private_key = {private_key}"
)
device.updateStateOnServer(key="status", value="Not Connected")
device.updateStateImageOnServer(indigo.kStateImageSel.SensorOff)
# Create the client configuration
broker = Broker.parse(f"ssl://{address}:{port}")
config = DxlClientConfig(broker_ca_bundle=ca_bundle,
cert_file=cert_file,
private_key=private_key,
brokers=[broker])
# Create the DXL client
self.dxl_client = DxlClient(config)
# Connect to the fabric
self.dxl_client.connect()
device.updateStateOnServer(key="status", value="Connected")
device.updateStateImageOnServer(indigo.kStateImageSel.SensorOn)
subs = device.pluginProps.get(u'subscriptions', None)
if subs:
for topic in subs:
self.dxl_client.add_event_callback(topic,
self.MyEventCallback(self))
self.logger.info(u"{}: Subscribing to: {}".format(
device.name, topic))
def disconnect(self):
device = indigo.devices[self.deviceID]
self.dxl_client.disconnect()
self.dxl_client.destroy()
device.updateStateOnServer(key="status", value="Not Connected")
device.updateStateImageOnServer(indigo.kStateImageSel.SensorOff)
def publish(self, topic, payload=None, qos=0, retain=False):
event = Event(topic)
event.payload = payload
self.dxl_client.send_event(event)
def subscribe(self, topic):
device = indigo.devices[self.deviceID]
self.logger.info(f"{device.name}: Subscribing to: {topic}")
self.dxl_client.add_event_callback(topic, self.MyEventCallback(self))
def unsubscribe(self, topic):
device = indigo.devices[self.deviceID]
self.logger.info(f"{device.name}: Unsubscribing from: {topic}")
self.dxl_client.unsubscribe(topic)
