def on_request(self, request):
"""
Invoked when a request message is received.
:param request: The request message
"""
# Handle request
logger.info("Request received on topic: '{0}' with payload: '{1}'".format(
request.destination_topic, MessageUtils.decode_payload(request)))
try:
# Parameters
nmap_params = MessageUtils.json_payload_to_dict(request)
logger.info("[1/2] Requested NMAP action ({0}) for request {1} is under processing...".format(
MessageUtils.decode_payload(request), request.message_id))
# Create a report from Nmap tool execution
nmap_response = self._do_nmap_scan(nmap_params, request)
logger.info("[2/2] Requested NMAP action was processed successfully for request {0}."
" Preparing response...".format(request.message_id))
# Create response
res = Response(request)
# Set payload
MessageUtils.dict_to_json_payload(res, nmap_response)
# Send response
self._app.client.send_response(res)
logger.info("Sending response for request {0}".format(request.message_id))
except Exception as ex:
logger.exception("Error handling request")
err_res = ErrorResponse(request, MessageUtils.encode(str(ex)))
self._app.client.send_response(err_res)
def on_request(self, request):
"""
Invoked when a request message is received.
:param request: The request message
"""
# Handle request
logger.info("Request received on topic: '%s' with payload: '%s'",
request.destination_topic,
MessageUtils.decode_payload(request))
try:
res = Response(request)
request_dict = MessageUtils.json_payload_to_dict(request) \
if request.payload else {}
# Ensure required parameters are present
if self._required_params:
for name in self._required_params:
if name not in request_dict:
raise Exception(
"Required parameter not found: '{}'".format(name))
if "format" not in request_dict:
request_dict["format"] = "json"
elif request_dict["format"] not in ("json", "xml"):
raise Exception(
"Unsupported format requested: '{}'. {}".format(
request_dict["format"],
"Only 'json' and 'xml' are supported."))
# Invoke DomainTools API via client
dt_response = \
getattr(self._app.domaintools_api,
self._func_name)(**request_dict)
# Set response payload
response_data = dt_response.data()
if isinstance(response_data, dict):
MessageUtils.dict_to_json_payload(res, response_data)
else:
MessageUtils.encode_payload(res, response_data)
except ServiceException as ex:
logger.exception("Error handling request")
msg = "%s: %s" % (ex.__class__.__name__, ex.reason)
res = ErrorResponse(request,
error_message=MessageUtils.encode(msg))
except Exception as ex:
logger.exception("Error handling request")
msg = str(ex)
if not msg:
msg = ex.__class__.__name__
res = ErrorResponse(request,
error_message=MessageUtils.encode(msg))
# Send response
self._app.client.send_response(res)
def _sync_request(dxl_client, request, response_timeout, payload_dict):
"""
Performs a synchronous DXL request and returns the payload
:param dxl_client: The DXL client with which to perform the request
:param request: The DXL request to send
:param response_timeout: The maximum amount of time to wait for a response
:param payload_dict: The dictionary (``dict``) to use as the payload of the DXL request
:return: The result of the remote command execution (resulting payload)
"""
# Set the payload
MessageUtils.dict_to_json_payload(request, payload_dict)
# Display the request that is going to be sent
logger.debug(
"Request:\n%s",
MessageUtils.dict_to_json(payload_dict, pretty_print=True))
# Send the request and wait for a response (synchronous)
res = dxl_client.sync_request(request, timeout=response_timeout)
if res.message_type == Message.MESSAGE_TYPE_ERROR:
raise Exception("Error: " + res.error_message + " (" +
str(res.error_code) + ")")
# Return a dictionary corresponding to the response payload
ret_val = MessageUtils.decode_payload(res)
# Display the response
logger.debug("Response:\n%s", ret_val)
return ret_val
def on_request(self, request):
"""
Invoked when a request message is received.
:param request: The request message
"""
# Handle request
logger.info(
"Request received on topic: '{0}' with payload: '{1}'".format(
request.destination_topic,
MessageUtils.decode_payload(request)))
try:
# Create response
res = Response(request)
# Read DXL request payload into dictionary
params = MessageUtils.json_payload_to_dict(request)
# Invoke API call
if 'data' in params.keys() and params['data'] != "":
tide_data = params['data']
if 'type' in params.keys(): tide_type = params['type']
else: tide_type = get_ioc_type(tide_data, 'tide')
if 'rlimit' in params.keys(
) and params['max_rec'] < self._app.tide_max_rec:
tide_max_rec = params['max_rec']
else:
tide_max_rec = self._app.tide_max_rec #Error data is required
if 'format' in params.keys(): tide_format = params['format']
else:
tide_format = self._app.tide_format #Error data is required
http_res = requests.get(
"https://api.activetrust.net:8000/api/data/threats",
params={
"type": tide_type,
tide_type: tide_data,
"data_format": tide_format,
"rlimit": tide_max_rec
},
auth=(self._app.api_key, ''))
content = unicode(http_res.content, "utf-8")
else:
content = "{'status':'error','errorMessage':'The data field is required'}" #Error data is required
# Add web service response to DXL response payload
MessageUtils.encode_payload(res, content, enc='utf-8')
# Send response
self._app.client.send_response(res)
except Exception as ex:
logger.exception("Error handling request")
err_res = ErrorResponse(request,
error_code=0,
error_message=MessageUtils.encode(str(ex)))
self._app.client.send_response(err_res)
def on_request(self, request):
"""
Invoked when a request message is received.
:param dxlclient.message.Request request: The request message
"""
logger.info("Request received on topic: '%s' with payload: '%s'",
request.destination_topic,
MessageUtils.decode_payload(request))
def on_request(self, request):
"""
Invoked when a request message is received.
:param request: The request message
"""
# Handle request
logger.info("Request received on topic: '%s' with payload: '%s'",
request.destination_topic,
MessageUtils.decode_payload(request))
try:
# API URL
api_url = self._app.URL_VOID_API_URL_FORMAT.format(self._app.api_key)
command = request.destination_topic[self._app.SERVICE_TYPE_LENGTH + 1:]
params = {}
if request.payload:
params = MessageUtils.json_payload_to_dict(request)
if self._required_params:
self._validate(params)
if command == self._app.CMD_HOST_INFO:
host = params[self.PARAM_HOST]
api_url = "{0}{1}/{2}".format(api_url, self.PARAM_HOST, host)
elif command == self._app.CMD_HOST_RESCAN:
host = params[self.PARAM_HOST]
api_url = "{0}{1}/{2}/rescan".format(api_url, self.PARAM_HOST, host)
elif command == self._app.CMD_HOST_SCAN:
host = params[self.PARAM_HOST]
api_url = "{0}{1}/{2}/scan".format(api_url, self.PARAM_HOST, host)
elif command == self._app.CMD_STATS_REMAINED:
api_url = "{0}{1}".format(api_url, command)
# Invoke URLVoid API
url_void_api_response = requests.get(api_url)
# Check HTTP response code
url_void_api_response.raise_for_status()
# Create response
res = Response(request)
# Set payload
MessageUtils.encode_payload(res, url_void_api_response.text)
# Send response
self._app.client.send_response(res)
except Exception as ex:
logger.exception("Error handling request")
err_res = ErrorResponse(request, error_message=MessageUtils.encode(str(ex)))
self._app.client.send_response(err_res)
def on_request(self, request):
"""
Invoked when a request message is received.
:param request: The request message
"""
# Handle request
logger.info(
"Request received on topic: '{0}' with payload: '{1}'".format(
request.destination_topic,
MessageUtils.decode_payload(request)))
try:
# Create response
res = Response(request)
# Read DXL request payload into dictionary
params = MessageUtils.decode_payload(request)
headers = {'Content-Type': 'application/json'}
#print(params)
# Invoke API call
http_res = requests.post(
"https://api.activetrust.net:8000/api/services/intel/lookup/jobs?wait=true",
headers=headers,
data=params,
auth=(self._app.api_key, ''))
content = unicode(http_res.content, "utf-8")
# Add web service response to DXL response payload
MessageUtils.encode_payload(res, content, enc='utf-8')
# Send response
self._app.client.send_response(res)
except Exception as ex:
logger.exception("Error handling request")
err_res = ErrorResponse(request,
error_code=0,
error_message=MessageUtils.encode(str(ex)))
self._app.client.send_response(err_res)
def _decode_response(res):
"""
Decodes the payload from DXL Response object into a string.
:param res: The DXL Response object to decode.
:return: The decoded payload.
:raise Exception: If ``res`` is an ErrorResponse.
"""
if res.message_type == Message.MESSAGE_TYPE_ERROR:
raise Exception("Error: " + res.error_message + " (" + str(
res.error_code) + ")")
# Return a dictionary corresponding to the response payload
ret_val = MessageUtils.decode_payload(res)
# Display the response
logger.debug("Response:\n%s", ret_val)
return ret_val
def on_request(self, request):
"""
Invoked when a request message is received.
:param request: The request message
"""
# Handle request
logger.info("Request received on topic: '%s' with payload: '%s'",
request.destination_topic,
MessageUtils.decode_payload(request))
try:
# Retrieve the parameters from the request
params = MessageUtils.json_payload_to_dict(request)
if self._PARAM_HOST not in params:
raise Exception(
"Required parameter not specified: '{0}'".format(
self._PARAM_HOST))
target = params[self._PARAM_HOST]
ipaddr = socket.gethostbyname(target)
logger.debug("IP to be located: %s", ipaddr)
# Lookup the IP/host
results = self._app.database.lookup_ip(ipaddr)
logger.debug(results)
# Create response
res = Response(request)
# Set payload
MessageUtils.dict_to_json_payload(res, results)
# Send response
self._app.client.send_response(res)
except Exception as ex:
logger.exception("Error handling request")
err_res = ErrorResponse(request,
error_code=0,
error_message=MessageUtils.encode(str(ex)))
self._app.client.send_response(err_res)
logger = logging.getLogger(__name__)
# Create DXL configuration from file
config = DxlClientConfig.create_dxl_config_from_file(CONFIG_FILE)
# Create the client
with DxlClient(config) as client:
# Connect to the fabric
client.connect()
logger.info("Connected to DXL fabric.")
# Invoke 'host scan' method
request_topic = "/opendxl-urlvoid/service/urlvapi/host/scan"
req = Request(request_topic)
MessageUtils.dict_to_json_payload(req, {"host": "027.ru"})
res = client.sync_request(req, timeout=60)
if res.message_type != Message.MESSAGE_TYPE_ERROR:
payload = MessageUtils.decode_payload(res)
xml = xml.dom.minidom.parseString(payload)
print("Response for URLVoid host scan:")
print(
xml.toprettyxml(indent=' ', newl='',
encoding="UTF-8").decode("UTF-8"))
else:
print("Error invoking service with topic '{0}': {1} ({2})".format(
request_topic, res.error_message, res.error_code))
# Create DXL configuration from file
config = DxlClientConfig.create_dxl_config_from_file(CONFIG_FILE)
# The ePO unique identifier
EPO_UNIQUE_ID = "<specify-ePO-unique-identifier>"
# Create the client
with DxlClient(config) as client:
# Connect to the fabric
client.connect()
# Create the request
req = Request("/mcafee/service/epo/remote/{0}".format(EPO_UNIQUE_ID))
# Set the payload for the request (core.help remote command)
MessageUtils.dict_to_json_payload(req, {
"command": "core.help",
"output": "verbose",
"params": {}
})
# Send the request
res = client.sync_request(req, timeout=30)
if res.message_type != Message.MESSAGE_TYPE_ERROR:
# Display resulting payload
print(MessageUtils.decode_payload(res))
else:
print("Error: {0} ({1}) ".format(res.error_message,
str(res.error_code)))
def get(self, *args, **kwargs):
"""HTTP GET"""
client_id = self.get_query_argument("clientId", "null")
if client_id == "null":
self.write(
dxlconsole.util.create_sc_error_response(
"No client ID sent with request."))
return
response_wrapper = dxlconsole.util.create_sc_response_wrapper()
response = response_wrapper["response"]
messages = self._module.get_messages(client_id)
if messages:
for message in messages:
# If we have a topic stored as a response for this message ID
# use it instead of the response topic
topic = self._module.get_message_topic(message)
if message.message_type == Message.MESSAGE_TYPE_ERROR:
payload = self.escape(message.error_message + " (" +
str(message.error_code) + ")")
original_payload = payload
else:
decoded_payload = MessageUtils.decode_payload(message)
original_payload = decoded_payload
try:
payload = "<pre><code>" + \
self.escape(
MessageUtils.dict_to_json(
MessageUtils.json_payload_to_dict(
message), True)) \
+ "</pre></code>"
except Exception:
try:
xml_payload = BeautifulSoup(
original_payload, "html.parser")
payload = "<pre lang='xml'><code>" + self.escape(
xml_payload.prettify()) + "</code></pre>"
original_payload = self.escape(original_payload)
except Exception as ex:
logger.exception(ex)
payload = original_payload
if len(payload) > self._MAX_DETAILS_PAYLOAD_LENGTH:
payload = payload[0:self._MAX_DETAILS_PAYLOAD_LENGTH] + \
" ..."
if len(payload) > self._MAX_TABLE_PAYLOAD_LENGTH:
original_payload = \
original_payload[0:self._MAX_TABLE_PAYLOAD_LENGTH] + \
" ..."
message_type = "Event" if message.message_type == Message.MESSAGE_TYPE_EVENT \
else "Response" if message.message_type == Message.MESSAGE_TYPE_RESPONSE \
else "Request" if message.message_type == Message.MESSAGE_TYPE_REQUEST \
else "Error Response" if message.message_type == Message.MESSAGE_TYPE_ERROR \
else "Unknown"
message_entry = {
'topic':
topic,
'received':
'',
'id':
message.message_id,
'type':
message_type,
'payload':
payload,
'originalPayload':
original_payload,
'sourceBroker':
message.source_broker_id,
'sourceClient':
message.source_client_id,
'otherFields':
"<pre><code>" + self.escape(
MessageUtils.dict_to_json(message.other_fields, True))
+ "</pre></code>"
}
response["data"].append(message_entry)
self._module.clear_messages(client_id)
logger.debug("Message handler response: %s",
json.dumps(response_wrapper))
self.write(response_wrapper)
def on_request(self, request):
"""
Invoked when a request message is received.
:param request: The request message
"""
# Handle request
logger.info(
"Request received on topic: '{0}' with payload: '{1}'".format(
request.destination_topic,
MessageUtils.decode_payload(request)))
try:
# API URL
api_url = self._app.VTAPI_URL_FORMAT.format(
request.destination_topic[self._app.SERVICE_TYPE_LENGTH:])
# Parameters
params = MessageUtils.json_payload_to_dict(request)
params["apikey"] = self._app.api_key
# Validate parameters
self._validate(params)
# Invoke VirusTotal API
if self._is_get:
vtapi_response = requests.get(api_url,
params=params,
headers=self._headers)
else:
vtapi_response = requests.post(api_url,
params=params,
headers=self._headers)
# Check HTTP response code
status_code = vtapi_response.status_code
if status_code != 200:
vtapi_response.raise_for_status()
http_message = self._get_http_error_message(status_code)
if http_message:
raise Exception("VirusTotal error, {0} ({1})".format(
http_message, str(status_code)))
else:
raise Exception(
"VirusTotal error, HTTP response code: {0}".format(
status_code))
# Create response
res = Response(request)
# Set payload
MessageUtils.dict_to_json_payload(res, vtapi_response.json())
# Send response
self._app.client.send_response(res)
except Exception as ex:
logger.exception("Error handling request")
err_message = str(ex)
err_message = err_message.replace(self._app.api_key, "--api-key--")
err_res = ErrorResponse(
request, error_message=MessageUtils.encode(err_message))
self._app.client.send_response(err_res)
def on_request(self, request):
"""
Invoked when a request message is received.
:param request: The request message
"""
# Handle request
logger.info(
"Request received on topic: '{0}' with payload: '{1}'".format(
request.destination_topic,
MessageUtils.decode_payload(request)))
try:
# Create response
res = Response(request)
# Read DXL request payload into dictionary
params = MessageUtils.json_payload_to_dict(request)
# Invoke API call
if 'data' in params.keys() and params['data'] != "":
dossier_data = params['data']
if 'type' in params.keys(): dossier_type = params['type']
else: dossier_type = get_ioc_type(dossier_data, 'dossier')
if 'sources' in params.keys():
payload = {
"target": {
"one": {
"type": dossier_type,
"target": dossier_data,
"sources": params['sources']
}
}
}
else:
payload = {
"target": {
"one": {
"type": dossier_type,
"target": dossier_data
}
}
}
headers = {'Content-Type': 'application/json'}
http_res = requests.post(
"https://api.activetrust.net:8000/api/services/intel/lookup/jobs?wait=true",
json=payload,
auth=(self._app.api_key, ''))
content = unicode(http_res.content, "utf-8")
else:
content = "{'status':'error','errorMessage':'The data field is required'}" #Error data is required
# Add web service response to DXL response payload
MessageUtils.encode_payload(res, content, enc='utf-8')
# Send response
self._app.client.send_response(res)
except Exception as ex:
logger.exception("Error handling request")
err_res = ErrorResponse(request,
error_code=0,
error_message=MessageUtils.encode(str(ex)))
self._app.client.send_response(err_res)
def on_request(self, request): #pylint
"""
Invoked when a request message is received.
:param request: The request message
"""
# Handle request
logger.info("Request received on topic: '%s' with payload: '%s'",
request.destination_topic,
MessageUtils.decode_payload(request))
try:
command = request.destination_topic[self._app.SERVICE_TYPE_LENGTH +
1:]
stats_remained = False
if self._app.REQ_TOPIC_SUFFIX_STATS in command:
stats_remained = True
command = command[:len(command) -
len(self._app.REQ_TOPIC_SUFFIX_STATS) - 1]
# API URL
api_url = self._app.API_VOID_URL_FORMAT.format(
command, self._app.KEY_PARAM_FORMAT.format(self._app.api_key))
if stats_remained:
api_url = "{0}&{1}".format(api_url, self.PARAM_STATS)
else:
params = {}
if request.payload:
params = MessageUtils.json_payload_to_dict(request)
if self._required_params:
self._validate(params)
if command == self._app.CMD_IP_REPUTATION:
ip_param = params[self.PARAM_IP]
api_url = "{0}&{1}={2}".format(api_url, self.PARAM_IP,
ip_param)
elif command == self._app.CMD_DOMAIN_REPUTATION:
host = params[self.PARAM_HOST]
api_url = "{0}&{1}={2}".format(api_url, self.PARAM_HOST,
host)
elif command == self._app.CMD_DNS_LOOKUP:
action = params[self.PARAM_ACTION]
host = params[self.PARAM_HOST]
api_url = "{0}&{1}={2}&{3}={4}".format(
api_url, self.PARAM_ACTION, action, self.PARAM_HOST,
host)
elif command in self._app.CMD_LIST:
host = params[self.PARAM_HOST]
api_url = "{0}&{1}={2}".format(api_url, self.PARAM_HOST,
host)
# Invoke URLVoid API
api_void_response = requests.get(api_url)
# Check HTTP response code
api_void_response.raise_for_status()
# Create response
res = Response(request)
# Set payload
MessageUtils.encode_payload(res, api_void_response.text)
# Send response
self._app.client.send_response(res)
except Exception as ex:
logger.exception("Error handling request")
err_res = ErrorResponse(request,
error_message=MessageUtils.encode(str(ex)))
self._app.client.send_response(err_res)
def on_event(event, index_operation):
"""
Callback invoked with content received for a DXL event. The callback should
return a dictionary (or list of dictionaries) with parameters for the
Elasticsearch index operation(s) that the service should perform.
The elements for each dictionary returned should correspond to parameters
in the `Elasticsearch Python Index API <https://elasticsearch-py.readthedocs.io/en/master/api.html#elasticsearch.Elasticsearch.index>`__.
For example, the value for the "doc_type" element in the dictionary will be
supplied as the "doc_type" parameter for the Elasticsearch "index"
operation.
:param dxlclient.message.Event event: The event which was received.
:param dict index_operation: A dict with a set of parameters configured
for storing the event payload into Elasticsearch. The dictionary
should include the following:
.. code-block::python
{ "index": "<documentIndex value from the application config>",
"doc_type": "<documentType value from the application config>",
"body": "<payload from the event parameter>",
"id": "<id from event payload>" }
If the event payload could be converted into a dict from JSON, the
value for the "body" element will be a dict. Otherwise, the payload
will be a str.
The value for the "id" element is pulled from the value for the key
in the event payload which corresponds to the "idFieldName" value
in the application configuration. If no value was set for "idFieldName"
in the application configuration, the value for the "id" element is
None.
:return: A dictionary (or list of dictionaries) with parameters for an
Elasticsearch "index" operation to perform. If None is returned, no
"index" operations will be performed.
:rtype: dict or list(dict)
"""
logger.info("Event payload received for transform: %s", event.payload)
logger.info("Index operation received for transform: %s", index_operation)
# Modify the "id" and "body" elements in the index operation dictionary.
event_payload = MessageUtils.decode_payload(event)
index_operation["id"] = "advanced-event-example-id-1"
# Store the event payload string in a dictionary. This allows Elasticsearch
# to serialize the document into JSON for storage.
index_operation["body"] = MessageUtils.dict_to_json({
"id":
index_operation["id"],
"message":
event_payload,
"source":
"Advanced Transform Example"
})
# Create a second index operation dictionary, using some of the values
# from the first dictionary.
another_index_operation = {
"index":
index_operation["index"],
"doc_type":
index_operation["doc_type"],
"id":
"advanced-event-example-id-2",
"body":
MessageUtils.dict_to_json({
"id": "advanced-event-example-id-2",
"message": event_payload,
"source": "Advanced Transform Example"
})
}
# Return info for two documents to store in Elasticsearch.
return [index_operation, another_index_operation]