def on_request(self, request): """ Invoked when a request message is received. :param request: The request message """ # Handle request request_payload = MessageUtils.json_payload_to_dict(request) if request.destination_topic in self._callbacks: try: self._callbacks[request.destination_topic](request, request_payload) except Exception as ex: err_res = ErrorResponse(request, error_code=0, error_message=MessageUtils.encode( str(ex))) self._client.send_response(err_res) else: err_res = ErrorResponse( request, error_code=0, error_message=MessageUtils.encode("Unknown topic: " + request.destination_topic)) self._client.send_response(err_res)
def on_request(self, request): """ Invoked when a request message is received. :param request: The request message """ # Handle request request_payload = MessageUtils.json_payload_to_dict(request) logger.info( "Request received on topic: '{0}' with payload: '{1}'".format( request.destination_topic, request_payload)) if request.destination_topic in self._callbacks: try: self._callbacks[request.destination_topic](request, request_payload) except Exception as ex: logger.exception("Error handling request") err_res = ErrorResponse(request, error_code=0, error_message=MessageUtils.encode( str(ex))) self._app.client.send_response(err_res) else: logger.exception("Unknown topic: %s", request.destination_topic) err_res = ErrorResponse( request, error_code=0, error_message=MessageUtils.encode( "Unknown topic: " + request.destination_topic)) self._app.client.send_response(err_res)
def on_request(self, request): """ Invoked when a request message is received. :param request: The request message """ # Handle request logger.info("Request received on topic: '%s' with payload: '%s'", request.destination_topic, MessageUtils.decode_payload(request)) try: res = Response(request) request_dict = MessageUtils.json_payload_to_dict(request) \ if request.payload else {} # Ensure required parameters are present if self._required_params: for name in self._required_params: if name not in request_dict: raise Exception( "Required parameter not found: '{}'".format(name)) if "format" not in request_dict: request_dict["format"] = "json" elif request_dict["format"] not in ("json", "xml"): raise Exception( "Unsupported format requested: '{}'. {}".format( request_dict["format"], "Only 'json' and 'xml' are supported.")) # Invoke DomainTools API via client dt_response = \ getattr(self._app.domaintools_api, self._func_name)(**request_dict) # Set response payload response_data = dt_response.data() if isinstance(response_data, dict): MessageUtils.dict_to_json_payload(res, response_data) else: MessageUtils.encode_payload(res, response_data) except ServiceException as ex: logger.exception("Error handling request") msg = "%s: %s" % (ex.__class__.__name__, ex.reason) res = ErrorResponse(request, error_message=MessageUtils.encode(msg)) except Exception as ex: logger.exception("Error handling request") msg = str(ex) if not msg: msg = ex.__class__.__name__ res = ErrorResponse(request, error_message=MessageUtils.encode(msg)) # Send response self._app.client.send_response(res)
def _pop_attribute_from_request(self, request, attr_name): """ Pop the value for a named attribute from the supplied request JSON and return the remaining request body. :param request: :param attr_name: :return: :raises ValueError: if the named attribute does not appear in the request. :return: A tuple containing two elements: the value associated with the attr_name parameter and the request body (minus the attr_name attribute and its associated value), converted to a dict. :rtype: tuple """ try: request_body = MessageUtils.json_payload_to_dict(request) attr_value = request_body.pop(attr_name, None) if not attr_name: raise ValueError("Attribute {} is missing".format(attr_name)) except Exception as ex: request_body = {} attr_value = None error_str = str(ex) logger.exception("Error handling request: %s", error_str) res = ErrorResponse(request, error_message=MessageUtils.encode(error_str)) self._dxl_client.send_response(res) return (attr_value, request_body)
def on_request(self, request): """ Invoked when a request message is received. :param request: The request message """ request_payload = MessageUtils.json_payload_to_dict(request) try: res = Response(request) payload = {"code": 200, "body": {}} if request_payload["target"] == "/v1/simple": payload = self.v1_simple(request_payload, payload) else: payload = self.v1_complex(request_payload, payload) MessageUtils.dict_to_json_payload(res, payload) self._client.send_response(res) except Exception as ex: err_res = ErrorResponse(request, error_code=0, error_message=MessageUtils.encode(str(ex))) self._client.send_response(err_res)
def on_request(self, request): """ Invoked when a request message is received. :param request: The request message """ # Handle request logger.info("Request received on topic: '{0}' with payload: '{1}'".format( request.destination_topic, MessageUtils.decode_payload(request))) try: # Parameters nmap_params = MessageUtils.json_payload_to_dict(request) logger.info("[1/2] Requested NMAP action ({0}) for request {1} is under processing...".format( MessageUtils.decode_payload(request), request.message_id)) # Create a report from Nmap tool execution nmap_response = self._do_nmap_scan(nmap_params, request) logger.info("[2/2] Requested NMAP action was processed successfully for request {0}." " Preparing response...".format(request.message_id)) # Create response res = Response(request) # Set payload MessageUtils.dict_to_json_payload(res, nmap_response) # Send response self._app.client.send_response(res) logger.info("Sending response for request {0}".format(request.message_id)) except Exception as ex: logger.exception("Error handling request") err_res = ErrorResponse(request, MessageUtils.encode(str(ex))) self._app.client.send_response(err_res)
def _generate_nmap_dict_report(self, targets, options, request): """ Executes the Nmap tool to scan/analyze a list of hosts (IPs) with the specific (parameters) options. :param targets: List of hosts (IPs) to scan/analyze. :param options: Options required to execute Nmap tool. :param request: The request message. :return: The Nmap output information (formatted) based on the original Nmap XML report information. """ # Create Nmap process and run it nmproc = NmapProcess(targets, options) try: # Run the Nmap process nmproc.run() # Generate report nmap_report = NmapParser.parse(nmproc.stdout) # Parse Nmap report parsed_report = self._parse_nmap_xml_report(nmap_report) # Return the parse Nmap report return parsed_report except Exception as ex: logger.exception("Nmap scan failed: {0}".format(nmproc.stderr)) err_res = ErrorResponse(request, MessageUtils.encode(str(ex))) self._app.client.send_response(err_res)
def on_request(self, request): """ Callback invoked when a request is received. :param dxlclient.message.Request request: The request """ logger.info("Request received on topic '%s'", request.destination_topic) logger.debug("Payload for topic %s: %s", request.destination_topic, request.payload) try: request_dict = MessageUtils.json_payload_to_dict(request) \ if request.payload else {} if "event" in request_dict and \ type(request_dict["event"]).__name__ in ("str", "unicode") and \ request_dict["event"].isdigit(): request_dict["event"] = int(request_dict["event"]) response_data = self._api_method(**request_dict) if isinstance(response_data, dict) and \ response_data.get("errors", None): res = ErrorResponse(request, error_message=str( response_data["errors"][0])) else: res = Response(request) MessageUtils.dict_to_json_payload(res, response_data) except Exception as ex: error_str = str(ex) logger.exception("Error handling request: %s", error_str) res = ErrorResponse(request, error_message=MessageUtils.encode(error_str)) self._app.client.send_response(res)
def _request(self, dxl_request, path, request_fn, body=None): """ Make a request to TheHive server, delivering the response to the DXL fabric. :param dxlclient.message.Request dxl_request: DXL request containing parameters to forward along in a request to TheHive server. :param str path: URL subpath for the request to send to TheHive server. :param function request_fn: Callback which is invoked to make TheHive request. :param str body: Request body to include in the request. """ try: request_url = self._api_url + path response = request_fn(request_url, body) if 200 <= response.status_code <= 299: # TheHive request was successful so forward the response # along as-is to the DXL fabric. res = Response(dxl_request) MessageUtils.dict_to_json_payload(res, response.json()) else: # TheHive request encountered an error. Attempt to decode # an error message from the response body. res = self._build_http_error_response(dxl_request, response) except Exception as ex: error_str = str(ex) logger.exception("Error handling request: %s", error_str) res = ErrorResponse(dxl_request, error_message=MessageUtils.encode(error_str)) self._dxl_client.send_response(res)
def on_request(self, request): """ Invoked when a request message is received. :param dxlclient.message.Request request: The request message """ # Handle request logger.debug("Request received on topic: '%s'", request.destination_topic) try: # Create response res = Response(request) # Store the next segment. result = self._store_manager.store_segment(request) # Set payload MessageUtils.dict_to_json_payload(res, result.to_dict()) # Send response self._dxl_client.send_response(res) except Exception as ex: logger.exception("Error handling request") err_res = ErrorResponse(request, error_code=0, error_message=MessageUtils.encode(str(ex))) self._dxl_client.send_response(err_res)
def on_request(self, request): """ Invoked when a request message is received. :param request: The request message """ # Handle request logger.info( "Request received on topic: '{0}' with payload: '{1}'".format( request.destination_topic, MessageUtils.decode_payload(request))) try: # Create response res = Response(request) # Read DXL request payload into dictionary params = MessageUtils.json_payload_to_dict(request) # Invoke API call if 'data' in params.keys() and params['data'] != "": tide_data = params['data'] if 'type' in params.keys(): tide_type = params['type'] else: tide_type = get_ioc_type(tide_data, 'tide') if 'rlimit' in params.keys( ) and params['max_rec'] < self._app.tide_max_rec: tide_max_rec = params['max_rec'] else: tide_max_rec = self._app.tide_max_rec #Error data is required if 'format' in params.keys(): tide_format = params['format'] else: tide_format = self._app.tide_format #Error data is required http_res = requests.get( "https://api.activetrust.net:8000/api/data/threats", params={ "type": tide_type, tide_type: tide_data, "data_format": tide_format, "rlimit": tide_max_rec }, auth=(self._app.api_key, '')) content = unicode(http_res.content, "utf-8") else: content = "{'status':'error','errorMessage':'The data field is required'}" #Error data is required # Add web service response to DXL response payload MessageUtils.encode_payload(res, content, enc='utf-8') # Send response self._app.client.send_response(res) except Exception as ex: logger.exception("Error handling request") err_res = ErrorResponse(request, error_code=0, error_message=MessageUtils.encode(str(ex))) self._app.client.send_response(err_res)
def on_request(self, request): try: res = Response(request) result = self._store_manager.store_segment(request) MessageUtils.dict_to_json_payload(res, result.to_dict()) self._dxl_client.send_response(res) except Exception as ex: err_res = ErrorResponse(request, error_message=MessageUtils.encode(str(ex))) self._dxl_client.send_response(err_res)
def on_request(self, request): """ Invoked when a request message is received. :param request: The request message """ # Handle request logger.info("Request received on topic: '%s' with payload: '%s'", request.destination_topic, MessageUtils.decode_payload(request)) try: # API URL api_url = self._app.URL_VOID_API_URL_FORMAT.format(self._app.api_key) command = request.destination_topic[self._app.SERVICE_TYPE_LENGTH + 1:] params = {} if request.payload: params = MessageUtils.json_payload_to_dict(request) if self._required_params: self._validate(params) if command == self._app.CMD_HOST_INFO: host = params[self.PARAM_HOST] api_url = "{0}{1}/{2}".format(api_url, self.PARAM_HOST, host) elif command == self._app.CMD_HOST_RESCAN: host = params[self.PARAM_HOST] api_url = "{0}{1}/{2}/rescan".format(api_url, self.PARAM_HOST, host) elif command == self._app.CMD_HOST_SCAN: host = params[self.PARAM_HOST] api_url = "{0}{1}/{2}/scan".format(api_url, self.PARAM_HOST, host) elif command == self._app.CMD_STATS_REMAINED: api_url = "{0}{1}".format(api_url, command) # Invoke URLVoid API url_void_api_response = requests.get(api_url) # Check HTTP response code url_void_api_response.raise_for_status() # Create response res = Response(request) # Set payload MessageUtils.encode_payload(res, url_void_api_response.text) # Send response self._app.client.send_response(res) except Exception as ex: logger.exception("Error handling request") err_res = ErrorResponse(request, error_message=MessageUtils.encode(str(ex))) self._app.client.send_response(err_res)
def on_event(self, event): """ Invoked when an event is received. :param event: The event """ # Handle event event_payload = MessageUtils.json_payload_to_dict(event) if event.destination_topic in self._callbacks: self._callbacks[event.destination_topic](event_payload) else: raise NotImplementedError( MessageUtils.encode("Unknown topic: " + event.destination_topic))
def on_request(self, request): """ Invoked when a request message is received. :param request: The request message """ # Handle request logger.info("Request received on topic: '%s' with payload: '%s'", request.destination_topic, MessageUtils.decode_payload(request)) try: # Retrieve the parameters from the request params = MessageUtils.json_payload_to_dict(request) if self._PARAM_HOST not in params: raise Exception( "Required parameter not specified: '{0}'".format( self._PARAM_HOST)) target = params[self._PARAM_HOST] ipaddr = socket.gethostbyname(target) logger.debug("IP to be located: %s", ipaddr) # Lookup the IP/host results = self._app.database.lookup_ip(ipaddr) logger.debug(results) # Create response res = Response(request) # Set payload MessageUtils.dict_to_json_payload(res, results) # Send response self._app.client.send_response(res) except Exception as ex: logger.exception("Error handling request") err_res = ErrorResponse(request, error_code=0, error_message=MessageUtils.encode(str(ex))) self._app.client.send_response(err_res)
def on_request(self, request): """ Invoked when a request message is received. :param request: The request message """ # Handle request logger.info( "Request received on topic: '{0}' with payload: '{1}'".format( request.destination_topic, MessageUtils.decode_payload(request))) try: # Create response res = Response(request) # Read DXL request payload into dictionary params = MessageUtils.decode_payload(request) headers = {'Content-Type': 'application/json'} #print(params) # Invoke API call http_res = requests.post( "https://api.activetrust.net:8000/api/services/intel/lookup/jobs?wait=true", headers=headers, data=params, auth=(self._app.api_key, '')) content = unicode(http_res.content, "utf-8") # Add web service response to DXL response payload MessageUtils.encode_payload(res, content, enc='utf-8') # Send response self._app.client.send_response(res) except Exception as ex: logger.exception("Error handling request") err_res = ErrorResponse(request, error_code=0, error_message=MessageUtils.encode(str(ex))) self._app.client.send_response(err_res)
def on_request(self, request): """ Callback invoked when a request is received. :param dxlclient.message.Request request: The request """ logger.info("Request received on topic '%s'", request.destination_topic) logger.debug("Payload for topic %s: %s", request.destination_topic, request.payload) try: res = Response(request) request_dict = MessageUtils.json_payload_to_dict(request) \ if request.payload else {} response_data = self._api_method(**request_dict) MessageUtils.dict_to_json_payload(res, response_data) except TransportError as ex: error_str = str(ex) logger.exception("TransportError handling request: %s", error_str) res = ErrorResponse(request, error_message=MessageUtils.encode(error_str)) error_dict = { "module": ex.__module__, "class": ex.__class__.__name__ } if isinstance(ex.info, dict): error_info = ex.info else: # If the error info is not already a dict, make a dict with # just the original class name of the error info object and # an associated error message. This is done to ensure that the # error response can be serialized into JSON for the DXL error_info = { "class": ex.info.__class__.__name__, "error": ex.info.__str__() } error_dict["data"] = { "status_code": ex.status_code, "error": ex.error, "info": error_info } MessageUtils.dict_to_json_payload(res, error_dict) except (ImproperlyConfigured, ElasticsearchException) as ex: error_str = str(ex) logger.exception("Elasticsearch exception handling request: %s", error_str) res = ErrorResponse(request, error_message=MessageUtils.encode(error_str)) error_dict = { "module": ex.__module__, "class": ex.__class__.__name__ } MessageUtils.dict_to_json_payload(res, error_dict) except Exception as ex: error_str = str(ex) logger.exception("Error handling request: %s", error_str) if not error_str: error_str = ex.__class__.__name__ res = ErrorResponse(request, error_message=MessageUtils.encode(error_str)) self._app.client.send_response(res)
def on_request(self, request): #pylint """ Invoked when a request message is received. :param request: The request message """ # Handle request logger.info("Request received on topic: '%s' with payload: '%s'", request.destination_topic, MessageUtils.decode_payload(request)) try: command = request.destination_topic[self._app.SERVICE_TYPE_LENGTH + 1:] stats_remained = False if self._app.REQ_TOPIC_SUFFIX_STATS in command: stats_remained = True command = command[:len(command) - len(self._app.REQ_TOPIC_SUFFIX_STATS) - 1] # API URL api_url = self._app.API_VOID_URL_FORMAT.format( command, self._app.KEY_PARAM_FORMAT.format(self._app.api_key)) if stats_remained: api_url = "{0}&{1}".format(api_url, self.PARAM_STATS) else: params = {} if request.payload: params = MessageUtils.json_payload_to_dict(request) if self._required_params: self._validate(params) if command == self._app.CMD_IP_REPUTATION: ip_param = params[self.PARAM_IP] api_url = "{0}&{1}={2}".format(api_url, self.PARAM_IP, ip_param) elif command == self._app.CMD_DOMAIN_REPUTATION: host = params[self.PARAM_HOST] api_url = "{0}&{1}={2}".format(api_url, self.PARAM_HOST, host) elif command == self._app.CMD_DNS_LOOKUP: action = params[self.PARAM_ACTION] host = params[self.PARAM_HOST] api_url = "{0}&{1}={2}&{3}={4}".format( api_url, self.PARAM_ACTION, action, self.PARAM_HOST, host) elif command in self._app.CMD_LIST: host = params[self.PARAM_HOST] api_url = "{0}&{1}={2}".format(api_url, self.PARAM_HOST, host) # Invoke URLVoid API api_void_response = requests.get(api_url) # Check HTTP response code api_void_response.raise_for_status() # Create response res = Response(request) # Set payload MessageUtils.encode_payload(res, api_void_response.text) # Send response self._app.client.send_response(res) except Exception as ex: logger.exception("Error handling request") err_res = ErrorResponse(request, error_message=MessageUtils.encode(str(ex))) self._app.client.send_response(err_res)
def on_request(self, request): """ Invoked when a request message is received. :param request: The request message """ # Handle request logger.info( "Request received on topic: '{0}' with payload: '{1}'".format( request.destination_topic, MessageUtils.decode_payload(request))) try: # API URL api_url = self._app.VTAPI_URL_FORMAT.format( request.destination_topic[self._app.SERVICE_TYPE_LENGTH:]) # Parameters params = MessageUtils.json_payload_to_dict(request) params["apikey"] = self._app.api_key # Validate parameters self._validate(params) # Invoke VirusTotal API if self._is_get: vtapi_response = requests.get(api_url, params=params, headers=self._headers) else: vtapi_response = requests.post(api_url, params=params, headers=self._headers) # Check HTTP response code status_code = vtapi_response.status_code if status_code != 200: vtapi_response.raise_for_status() http_message = self._get_http_error_message(status_code) if http_message: raise Exception("VirusTotal error, {0} ({1})".format( http_message, str(status_code))) else: raise Exception( "VirusTotal error, HTTP response code: {0}".format( status_code)) # Create response res = Response(request) # Set payload MessageUtils.dict_to_json_payload(res, vtapi_response.json()) # Send response self._app.client.send_response(res) except Exception as ex: logger.exception("Error handling request") err_message = str(ex) err_message = err_message.replace(self._app.api_key, "--api-key--") err_res = ErrorResponse( request, error_message=MessageUtils.encode(err_message)) self._app.client.send_response(err_res)
def on_request(self, request): """ Invoked when a request message is received. :param request: The request message """ request_payload = MessageUtils.json_payload_to_dict(request) logger.info( "Request received on topic: '{0}' with payload: '{1}'".format( request.destination_topic, request_payload)) try: res = Response(request) payload = {"code": 200, "body": {}} if request_payload["target"] == "/v1/simple": if request_payload["method"] != 'POST': payload = self._make_error_response( 405, "Unsupported method") elif "body" not in request_payload or \ "projections" not in request_payload["body"]: payload = self._make_error_response( 400, "Missing body or projections parameter") else: projections_as_str = self._get_projection_as_string( request_payload["body"]["projections"]) if projections_as_str in self.PROJECTION_TO_ID: search_id = str(uuid.uuid4()).replace("-", "")[:24] self.SEARCHES[search_id] = { "statusChecksUntilFinished": \ self._status_checks_until_finished, "projectionId": self.PROJECTION_TO_ID[ projections_as_str] } payload["body"]["id"] = search_id else: payload = self._make_error_response( 501, "Unsupported projection") else: search_id_match = re.match(r".*/v1/(\w+)/.*", request_payload["target"]) if search_id_match and search_id_match.group(1) in self.SEARCHES: search_entry = self.SEARCHES[search_id_match.group(1)] request_items = self.RESULTS[search_entry["projectionId"]] if request_payload["target"].endswith("/status"): if request_payload["method"] != 'GET': payload = self._make_error_response( 405, "Unsupported method") elif search_entry["statusChecksUntilFinished"]: search_entry["statusChecksUntilFinished"] -= 1 payload["body"]["status"] = "RUNNING" else: payload["body"]["results"] = len(request_items) payload["body"]["errors"] = 0 payload["body"]["hosts"] = 1 payload["body"]["subscribedHosts"] = 1 payload["body"]["status"] = "FINISHED" elif request_payload["target"].endswith("/results"): if request_payload["method"] != 'GET': payload = self._make_error_response( 405, "Unsupported method") elif "parameters" in request_payload and \ "$offset" in request_payload["parameters"] and \ "$limit" in request_payload["parameters"]: offset = request_payload["parameters"]["$offset"] limit = request_payload["parameters"]["$limit"] payload["body"]["items"] = request_items[ offset: offset + limit ] else: payload["body"]["items"] = request_items else: payload = self._make_error_response( 404, "Id not found") MessageUtils.dict_to_json_payload(res, payload) self._app.client.send_response(res) except Exception as ex: logger.exception("Error handling request") err_res = ErrorResponse(request, error_code=0, error_message=MessageUtils.encode(str(ex))) self._app.client.send_response(err_res)
def on_request(self, request): """ Invoked when a request message is received. :param request: The request message """ # Handle request logger.info( "Request received on topic: '{0}' with payload: '{1}'".format( request.destination_topic, MessageUtils.decode_payload(request))) try: # Create response res = Response(request) # Read DXL request payload into dictionary params = MessageUtils.json_payload_to_dict(request) # Invoke API call if 'data' in params.keys() and params['data'] != "": dossier_data = params['data'] if 'type' in params.keys(): dossier_type = params['type'] else: dossier_type = get_ioc_type(dossier_data, 'dossier') if 'sources' in params.keys(): payload = { "target": { "one": { "type": dossier_type, "target": dossier_data, "sources": params['sources'] } } } else: payload = { "target": { "one": { "type": dossier_type, "target": dossier_data } } } headers = {'Content-Type': 'application/json'} http_res = requests.post( "https://api.activetrust.net:8000/api/services/intel/lookup/jobs?wait=true", json=payload, auth=(self._app.api_key, '')) content = unicode(http_res.content, "utf-8") else: content = "{'status':'error','errorMessage':'The data field is required'}" #Error data is required # Add web service response to DXL response payload MessageUtils.encode_payload(res, content, enc='utf-8') # Send response self._app.client.send_response(res) except Exception as ex: logger.exception("Error handling request") err_res = ErrorResponse(request, error_code=0, error_message=MessageUtils.encode(str(ex))) self._app.client.send_response(err_res)