def on_request(self, request): """ Invoked when a request message is received. :param request: The request message """ # Handle request logger.info("Request received on topic: '{0}' with payload: '{1}'".format( request.destination_topic, MessageUtils.decode_payload(request))) try: # Parameters nmap_params = MessageUtils.json_payload_to_dict(request) logger.info("[1/2] Requested NMAP action ({0}) for request {1} is under processing...".format( MessageUtils.decode_payload(request), request.message_id)) # Create a report from Nmap tool execution nmap_response = self._do_nmap_scan(nmap_params, request) logger.info("[2/2] Requested NMAP action was processed successfully for request {0}." " Preparing response...".format(request.message_id)) # Create response res = Response(request) # Set payload MessageUtils.dict_to_json_payload(res, nmap_response) # Send response self._app.client.send_response(res) logger.info("Sending response for request {0}".format(request.message_id)) except Exception as ex: logger.exception("Error handling request") err_res = ErrorResponse(request, MessageUtils.encode(str(ex))) self._app.client.send_response(err_res)
def on_request(self, request): """ Invoked when a request message is received. :param request: The request message """ # Handle request logger.info("Request received on topic: '%s' with payload: '%s'", request.destination_topic, MessageUtils.decode_payload(request)) try: res = Response(request) request_dict = MessageUtils.json_payload_to_dict(request) \ if request.payload else {} # Ensure required parameters are present if self._required_params: for name in self._required_params: if name not in request_dict: raise Exception( "Required parameter not found: '{}'".format(name)) if "format" not in request_dict: request_dict["format"] = "json" elif request_dict["format"] not in ("json", "xml"): raise Exception( "Unsupported format requested: '{}'. {}".format( request_dict["format"], "Only 'json' and 'xml' are supported.")) # Invoke DomainTools API via client dt_response = \ getattr(self._app.domaintools_api, self._func_name)(**request_dict) # Set response payload response_data = dt_response.data() if isinstance(response_data, dict): MessageUtils.dict_to_json_payload(res, response_data) else: MessageUtils.encode_payload(res, response_data) except ServiceException as ex: logger.exception("Error handling request") msg = "%s: %s" % (ex.__class__.__name__, ex.reason) res = ErrorResponse(request, error_message=MessageUtils.encode(msg)) except Exception as ex: logger.exception("Error handling request") msg = str(ex) if not msg: msg = ex.__class__.__name__ res = ErrorResponse(request, error_message=MessageUtils.encode(msg)) # Send response self._app.client.send_response(res)
def _sync_request(dxl_client, request, response_timeout, payload_dict): """ Performs a synchronous DXL request and returns the payload :param dxl_client: The DXL client with which to perform the request :param request: The DXL request to send :param response_timeout: The maximum amount of time to wait for a response :param payload_dict: The dictionary (``dict``) to use as the payload of the DXL request :return: The result of the remote command execution (resulting payload) """ # Set the payload MessageUtils.dict_to_json_payload(request, payload_dict) # Display the request that is going to be sent logger.debug( "Request:\n%s", MessageUtils.dict_to_json(payload_dict, pretty_print=True)) # Send the request and wait for a response (synchronous) res = dxl_client.sync_request(request, timeout=response_timeout) if res.message_type == Message.MESSAGE_TYPE_ERROR: raise Exception("Error: " + res.error_message + " (" + str(res.error_code) + ")") # Return a dictionary corresponding to the response payload ret_val = MessageUtils.decode_payload(res) # Display the response logger.debug("Response:\n%s", ret_val) return ret_val
def on_request(self, request): """ Invoked when a request message is received. :param request: The request message """ # Handle request logger.info( "Request received on topic: '{0}' with payload: '{1}'".format( request.destination_topic, MessageUtils.decode_payload(request))) try: # Create response res = Response(request) # Read DXL request payload into dictionary params = MessageUtils.json_payload_to_dict(request) # Invoke API call if 'data' in params.keys() and params['data'] != "": tide_data = params['data'] if 'type' in params.keys(): tide_type = params['type'] else: tide_type = get_ioc_type(tide_data, 'tide') if 'rlimit' in params.keys( ) and params['max_rec'] < self._app.tide_max_rec: tide_max_rec = params['max_rec'] else: tide_max_rec = self._app.tide_max_rec #Error data is required if 'format' in params.keys(): tide_format = params['format'] else: tide_format = self._app.tide_format #Error data is required http_res = requests.get( "https://api.activetrust.net:8000/api/data/threats", params={ "type": tide_type, tide_type: tide_data, "data_format": tide_format, "rlimit": tide_max_rec }, auth=(self._app.api_key, '')) content = unicode(http_res.content, "utf-8") else: content = "{'status':'error','errorMessage':'The data field is required'}" #Error data is required # Add web service response to DXL response payload MessageUtils.encode_payload(res, content, enc='utf-8') # Send response self._app.client.send_response(res) except Exception as ex: logger.exception("Error handling request") err_res = ErrorResponse(request, error_code=0, error_message=MessageUtils.encode(str(ex))) self._app.client.send_response(err_res)
def on_request(self, request): """ Invoked when a request message is received. :param dxlclient.message.Request request: The request message """ logger.info("Request received on topic: '%s' with payload: '%s'", request.destination_topic, MessageUtils.decode_payload(request))
def on_request(self, request): """ Invoked when a request message is received. :param request: The request message """ # Handle request logger.info("Request received on topic: '%s' with payload: '%s'", request.destination_topic, MessageUtils.decode_payload(request)) try: # API URL api_url = self._app.URL_VOID_API_URL_FORMAT.format(self._app.api_key) command = request.destination_topic[self._app.SERVICE_TYPE_LENGTH + 1:] params = {} if request.payload: params = MessageUtils.json_payload_to_dict(request) if self._required_params: self._validate(params) if command == self._app.CMD_HOST_INFO: host = params[self.PARAM_HOST] api_url = "{0}{1}/{2}".format(api_url, self.PARAM_HOST, host) elif command == self._app.CMD_HOST_RESCAN: host = params[self.PARAM_HOST] api_url = "{0}{1}/{2}/rescan".format(api_url, self.PARAM_HOST, host) elif command == self._app.CMD_HOST_SCAN: host = params[self.PARAM_HOST] api_url = "{0}{1}/{2}/scan".format(api_url, self.PARAM_HOST, host) elif command == self._app.CMD_STATS_REMAINED: api_url = "{0}{1}".format(api_url, command) # Invoke URLVoid API url_void_api_response = requests.get(api_url) # Check HTTP response code url_void_api_response.raise_for_status() # Create response res = Response(request) # Set payload MessageUtils.encode_payload(res, url_void_api_response.text) # Send response self._app.client.send_response(res) except Exception as ex: logger.exception("Error handling request") err_res = ErrorResponse(request, error_message=MessageUtils.encode(str(ex))) self._app.client.send_response(err_res)
def on_request(self, request): """ Invoked when a request message is received. :param request: The request message """ # Handle request logger.info( "Request received on topic: '{0}' with payload: '{1}'".format( request.destination_topic, MessageUtils.decode_payload(request))) try: # Create response res = Response(request) # Read DXL request payload into dictionary params = MessageUtils.decode_payload(request) headers = {'Content-Type': 'application/json'} #print(params) # Invoke API call http_res = requests.post( "https://api.activetrust.net:8000/api/services/intel/lookup/jobs?wait=true", headers=headers, data=params, auth=(self._app.api_key, '')) content = unicode(http_res.content, "utf-8") # Add web service response to DXL response payload MessageUtils.encode_payload(res, content, enc='utf-8') # Send response self._app.client.send_response(res) except Exception as ex: logger.exception("Error handling request") err_res = ErrorResponse(request, error_code=0, error_message=MessageUtils.encode(str(ex))) self._app.client.send_response(err_res)
def _decode_response(res): """ Decodes the payload from DXL Response object into a string. :param res: The DXL Response object to decode. :return: The decoded payload. :raise Exception: If ``res`` is an ErrorResponse. """ if res.message_type == Message.MESSAGE_TYPE_ERROR: raise Exception("Error: " + res.error_message + " (" + str( res.error_code) + ")") # Return a dictionary corresponding to the response payload ret_val = MessageUtils.decode_payload(res) # Display the response logger.debug("Response:\n%s", ret_val) return ret_val
def on_request(self, request): """ Invoked when a request message is received. :param request: The request message """ # Handle request logger.info("Request received on topic: '%s' with payload: '%s'", request.destination_topic, MessageUtils.decode_payload(request)) try: # Retrieve the parameters from the request params = MessageUtils.json_payload_to_dict(request) if self._PARAM_HOST not in params: raise Exception( "Required parameter not specified: '{0}'".format( self._PARAM_HOST)) target = params[self._PARAM_HOST] ipaddr = socket.gethostbyname(target) logger.debug("IP to be located: %s", ipaddr) # Lookup the IP/host results = self._app.database.lookup_ip(ipaddr) logger.debug(results) # Create response res = Response(request) # Set payload MessageUtils.dict_to_json_payload(res, results) # Send response self._app.client.send_response(res) except Exception as ex: logger.exception("Error handling request") err_res = ErrorResponse(request, error_code=0, error_message=MessageUtils.encode(str(ex))) self._app.client.send_response(err_res)
logger = logging.getLogger(__name__) # Create DXL configuration from file config = DxlClientConfig.create_dxl_config_from_file(CONFIG_FILE) # Create the client with DxlClient(config) as client: # Connect to the fabric client.connect() logger.info("Connected to DXL fabric.") # Invoke 'host scan' method request_topic = "/opendxl-urlvoid/service/urlvapi/host/scan" req = Request(request_topic) MessageUtils.dict_to_json_payload(req, {"host": "027.ru"}) res = client.sync_request(req, timeout=60) if res.message_type != Message.MESSAGE_TYPE_ERROR: payload = MessageUtils.decode_payload(res) xml = xml.dom.minidom.parseString(payload) print("Response for URLVoid host scan:") print( xml.toprettyxml(indent=' ', newl='', encoding="UTF-8").decode("UTF-8")) else: print("Error invoking service with topic '{0}': {1} ({2})".format( request_topic, res.error_message, res.error_code))
# Create DXL configuration from file config = DxlClientConfig.create_dxl_config_from_file(CONFIG_FILE) # The ePO unique identifier EPO_UNIQUE_ID = "<specify-ePO-unique-identifier>" # Create the client with DxlClient(config) as client: # Connect to the fabric client.connect() # Create the request req = Request("/mcafee/service/epo/remote/{0}".format(EPO_UNIQUE_ID)) # Set the payload for the request (core.help remote command) MessageUtils.dict_to_json_payload(req, { "command": "core.help", "output": "verbose", "params": {} }) # Send the request res = client.sync_request(req, timeout=30) if res.message_type != Message.MESSAGE_TYPE_ERROR: # Display resulting payload print(MessageUtils.decode_payload(res)) else: print("Error: {0} ({1}) ".format(res.error_message, str(res.error_code)))
def get(self, *args, **kwargs): """HTTP GET""" client_id = self.get_query_argument("clientId", "null") if client_id == "null": self.write( dxlconsole.util.create_sc_error_response( "No client ID sent with request.")) return response_wrapper = dxlconsole.util.create_sc_response_wrapper() response = response_wrapper["response"] messages = self._module.get_messages(client_id) if messages: for message in messages: # If we have a topic stored as a response for this message ID # use it instead of the response topic topic = self._module.get_message_topic(message) if message.message_type == Message.MESSAGE_TYPE_ERROR: payload = self.escape(message.error_message + " (" + str(message.error_code) + ")") original_payload = payload else: decoded_payload = MessageUtils.decode_payload(message) original_payload = decoded_payload try: payload = "<pre><code>" + \ self.escape( MessageUtils.dict_to_json( MessageUtils.json_payload_to_dict( message), True)) \ + "</pre></code>" except Exception: try: xml_payload = BeautifulSoup( original_payload, "html.parser") payload = "<pre lang='xml'><code>" + self.escape( xml_payload.prettify()) + "</code></pre>" original_payload = self.escape(original_payload) except Exception as ex: logger.exception(ex) payload = original_payload if len(payload) > self._MAX_DETAILS_PAYLOAD_LENGTH: payload = payload[0:self._MAX_DETAILS_PAYLOAD_LENGTH] + \ " ..." if len(payload) > self._MAX_TABLE_PAYLOAD_LENGTH: original_payload = \ original_payload[0:self._MAX_TABLE_PAYLOAD_LENGTH] + \ " ..." message_type = "Event" if message.message_type == Message.MESSAGE_TYPE_EVENT \ else "Response" if message.message_type == Message.MESSAGE_TYPE_RESPONSE \ else "Request" if message.message_type == Message.MESSAGE_TYPE_REQUEST \ else "Error Response" if message.message_type == Message.MESSAGE_TYPE_ERROR \ else "Unknown" message_entry = { 'topic': topic, 'received': '', 'id': message.message_id, 'type': message_type, 'payload': payload, 'originalPayload': original_payload, 'sourceBroker': message.source_broker_id, 'sourceClient': message.source_client_id, 'otherFields': "<pre><code>" + self.escape( MessageUtils.dict_to_json(message.other_fields, True)) + "</pre></code>" } response["data"].append(message_entry) self._module.clear_messages(client_id) logger.debug("Message handler response: %s", json.dumps(response_wrapper)) self.write(response_wrapper)
def on_request(self, request): """ Invoked when a request message is received. :param request: The request message """ # Handle request logger.info( "Request received on topic: '{0}' with payload: '{1}'".format( request.destination_topic, MessageUtils.decode_payload(request))) try: # API URL api_url = self._app.VTAPI_URL_FORMAT.format( request.destination_topic[self._app.SERVICE_TYPE_LENGTH:]) # Parameters params = MessageUtils.json_payload_to_dict(request) params["apikey"] = self._app.api_key # Validate parameters self._validate(params) # Invoke VirusTotal API if self._is_get: vtapi_response = requests.get(api_url, params=params, headers=self._headers) else: vtapi_response = requests.post(api_url, params=params, headers=self._headers) # Check HTTP response code status_code = vtapi_response.status_code if status_code != 200: vtapi_response.raise_for_status() http_message = self._get_http_error_message(status_code) if http_message: raise Exception("VirusTotal error, {0} ({1})".format( http_message, str(status_code))) else: raise Exception( "VirusTotal error, HTTP response code: {0}".format( status_code)) # Create response res = Response(request) # Set payload MessageUtils.dict_to_json_payload(res, vtapi_response.json()) # Send response self._app.client.send_response(res) except Exception as ex: logger.exception("Error handling request") err_message = str(ex) err_message = err_message.replace(self._app.api_key, "--api-key--") err_res = ErrorResponse( request, error_message=MessageUtils.encode(err_message)) self._app.client.send_response(err_res)
def on_request(self, request): """ Invoked when a request message is received. :param request: The request message """ # Handle request logger.info( "Request received on topic: '{0}' with payload: '{1}'".format( request.destination_topic, MessageUtils.decode_payload(request))) try: # Create response res = Response(request) # Read DXL request payload into dictionary params = MessageUtils.json_payload_to_dict(request) # Invoke API call if 'data' in params.keys() and params['data'] != "": dossier_data = params['data'] if 'type' in params.keys(): dossier_type = params['type'] else: dossier_type = get_ioc_type(dossier_data, 'dossier') if 'sources' in params.keys(): payload = { "target": { "one": { "type": dossier_type, "target": dossier_data, "sources": params['sources'] } } } else: payload = { "target": { "one": { "type": dossier_type, "target": dossier_data } } } headers = {'Content-Type': 'application/json'} http_res = requests.post( "https://api.activetrust.net:8000/api/services/intel/lookup/jobs?wait=true", json=payload, auth=(self._app.api_key, '')) content = unicode(http_res.content, "utf-8") else: content = "{'status':'error','errorMessage':'The data field is required'}" #Error data is required # Add web service response to DXL response payload MessageUtils.encode_payload(res, content, enc='utf-8') # Send response self._app.client.send_response(res) except Exception as ex: logger.exception("Error handling request") err_res = ErrorResponse(request, error_code=0, error_message=MessageUtils.encode(str(ex))) self._app.client.send_response(err_res)
def on_request(self, request): #pylint """ Invoked when a request message is received. :param request: The request message """ # Handle request logger.info("Request received on topic: '%s' with payload: '%s'", request.destination_topic, MessageUtils.decode_payload(request)) try: command = request.destination_topic[self._app.SERVICE_TYPE_LENGTH + 1:] stats_remained = False if self._app.REQ_TOPIC_SUFFIX_STATS in command: stats_remained = True command = command[:len(command) - len(self._app.REQ_TOPIC_SUFFIX_STATS) - 1] # API URL api_url = self._app.API_VOID_URL_FORMAT.format( command, self._app.KEY_PARAM_FORMAT.format(self._app.api_key)) if stats_remained: api_url = "{0}&{1}".format(api_url, self.PARAM_STATS) else: params = {} if request.payload: params = MessageUtils.json_payload_to_dict(request) if self._required_params: self._validate(params) if command == self._app.CMD_IP_REPUTATION: ip_param = params[self.PARAM_IP] api_url = "{0}&{1}={2}".format(api_url, self.PARAM_IP, ip_param) elif command == self._app.CMD_DOMAIN_REPUTATION: host = params[self.PARAM_HOST] api_url = "{0}&{1}={2}".format(api_url, self.PARAM_HOST, host) elif command == self._app.CMD_DNS_LOOKUP: action = params[self.PARAM_ACTION] host = params[self.PARAM_HOST] api_url = "{0}&{1}={2}&{3}={4}".format( api_url, self.PARAM_ACTION, action, self.PARAM_HOST, host) elif command in self._app.CMD_LIST: host = params[self.PARAM_HOST] api_url = "{0}&{1}={2}".format(api_url, self.PARAM_HOST, host) # Invoke URLVoid API api_void_response = requests.get(api_url) # Check HTTP response code api_void_response.raise_for_status() # Create response res = Response(request) # Set payload MessageUtils.encode_payload(res, api_void_response.text) # Send response self._app.client.send_response(res) except Exception as ex: logger.exception("Error handling request") err_res = ErrorResponse(request, error_message=MessageUtils.encode(str(ex))) self._app.client.send_response(err_res)
def on_event(event, index_operation): """ Callback invoked with content received for a DXL event. The callback should return a dictionary (or list of dictionaries) with parameters for the Elasticsearch index operation(s) that the service should perform. The elements for each dictionary returned should correspond to parameters in the `Elasticsearch Python Index API <https://elasticsearch-py.readthedocs.io/en/master/api.html#elasticsearch.Elasticsearch.index>`__. For example, the value for the "doc_type" element in the dictionary will be supplied as the "doc_type" parameter for the Elasticsearch "index" operation. :param dxlclient.message.Event event: The event which was received. :param dict index_operation: A dict with a set of parameters configured for storing the event payload into Elasticsearch. The dictionary should include the following: .. code-block::python { "index": "<documentIndex value from the application config>", "doc_type": "<documentType value from the application config>", "body": "<payload from the event parameter>", "id": "<id from event payload>" } If the event payload could be converted into a dict from JSON, the value for the "body" element will be a dict. Otherwise, the payload will be a str. The value for the "id" element is pulled from the value for the key in the event payload which corresponds to the "idFieldName" value in the application configuration. If no value was set for "idFieldName" in the application configuration, the value for the "id" element is None. :return: A dictionary (or list of dictionaries) with parameters for an Elasticsearch "index" operation to perform. If None is returned, no "index" operations will be performed. :rtype: dict or list(dict) """ logger.info("Event payload received for transform: %s", event.payload) logger.info("Index operation received for transform: %s", index_operation) # Modify the "id" and "body" elements in the index operation dictionary. event_payload = MessageUtils.decode_payload(event) index_operation["id"] = "advanced-event-example-id-1" # Store the event payload string in a dictionary. This allows Elasticsearch # to serialize the document into JSON for storage. index_operation["body"] = MessageUtils.dict_to_json({ "id": index_operation["id"], "message": event_payload, "source": "Advanced Transform Example" }) # Create a second index operation dictionary, using some of the values # from the first dictionary. another_index_operation = { "index": index_operation["index"], "doc_type": index_operation["doc_type"], "id": "advanced-event-example-id-2", "body": MessageUtils.dict_to_json({ "id": "advanced-event-example-id-2", "message": event_payload, "source": "Advanced Transform Example" }) } # Return info for two documents to store in Elasticsearch. return [index_operation, another_index_operation]