def _do_artifact_mapping(query_definition, event_message, metadata, response, res_client, context_token, additional_map_data=None): """ Map query results to new artifact and add to Resilient """ incident = event_message.get("incident", {}) incident_id = incident.get("id") if not response: return existing_artifacts = [ _artifact_key(artifact) for artifact in _get_artifacts(incident_id, res_client) ] LOG.debug(u"Existing Artifacts:\n%s", u"\n".join(existing_artifacts)) if query_definition.result_container: # Create an artifact for each query result row for row in response: for artifact_template in query_definition.artifact_mapping: mapdata = {"result": row} # Add in any query result metadata mapdata.update(metadata) if additional_map_data: mapdata.update(additional_map_data) artifact = template_functions.render_json( artifact_template, mapdata) if artifact.get("value") and _unique_artifact( artifact, existing_artifacts): _add_artifact(res_client, incident_id, artifact, context_token) existing_artifacts.append(_artifact_key(artifact)) else: # Create a single artifact from the query result for artifact_template in query_definition.artifact_mapping: artifact = template_functions.render_json(artifact_template, response) if artifact.get("value") and _unique_artifact( artifact, existing_artifacts): _add_artifact(res_client, incident_id, artifact, context_token) existing_artifacts.append(_artifact_key(artifact))
def _do_task_mapping(query_definition, event_message, metadata, response, res_client, context_token, additional_map_data=None): """ Map query results to new task and add to Resilient """ incident = event_message.get("incident", {}) incident_id = incident.get("id") if not response: return def _add_task(client, incident_id, task): """ Create resilient task """ try: LOG.info(u"Adding task to incident %s: %s", incident_id, task) client.post("/incidents/%s/tasks" % incident_id, task, co3_context_token=context_token) except SimpleHTTPException as error: LOG.error("Failed to post new task. Status %s", str(error.response.status_code)) LOG.exception("Failed to post task") if query_definition.result_container: # Create a task for each query result row for row in response: for task_template in query_definition.task_mapping: mapdata = {"result": row} # Add in any query result metadata mapdata.update(metadata) if additional_map_data: mapdata.update(additional_map_data) task = template_functions.render_json(task_template, mapdata) _add_task(res_client, incident_id, task) else: # Create a single task from the query result for task_template in query_definition.task_mapping: task = template_functions.render_json(task_template, response) _add_task(res_client, incident_id, task)
def _do_note_mapping(query_definition, event_message, metadata, response, res_client, context_token, additional_map_data=None): """ Map query results to new note and add to Resilient """ incident = event_message.get("incident", {}) incident_id = incident.get("id") if not response: return def _add_note(client, incident_id, note): """ Create resilient note """ try: LOG.info(u"Adding note to incident %s: %s", incident_id, note) client.post("/incidents/%s/comments" % incident_id, note, co3_context_token=context_token) except SimpleHTTPException as error: LOG.error("Failed to post new note. Status %s", str(error.response.status_code)) LOG.exception("Failed to post note") if query_definition.result_container: # Create a note for each query result row for row in response: mapdata = {"result": row} # Add in any query result metadata mapdata.update(metadata) if additional_map_data: mapdata.update(additional_map_data) for note_template in query_definition.note_mapping: note = template_functions.render_json(note_template, mapdata) _add_note(res_client, incident_id, note) else: # Create a single note from the query result for note_template in query_definition.note_mapping: note = template_functions.render_json(note_template, response) _add_note(res_client, incident_id, note)
def _close_incident(self, incident, ticket): """ Close a Resilient incident by rendering a jinja2 template :param ticket: Secureworks CTP ticket (json object) :return: Resilient incident """ try: # Close a Resilient incident from this ticket # using a JSON (JINJA2) template file template_file_path = self.options.get('template_file_close') if template_file_path and not os.path.exists(template_file_path): LOG.warning(u"Template file '%s' not found.", template_file_path) template_file_path = None if not template_file_path: # Use the template file installed by this package template_file_path = resource_filename( Requirement("fn-secureworks-ctp"), "fn_secureworks_ctp/data/scwx_ctp_template_close.jinja") if not os.path.exists(template_file_path): raise Exception( u"Template file for close'{0}' not found".format( template_file_path)) LOG.info( u"Secureworks CTP jinja template file for closing incident: %s", template_file_path) with open(template_file_path, "r") as definition: close_template = definition.read() incident_payload = render_json(close_template, ticket) # Set the scwx_ctp_status incident field to the ticket status (Closed or Resolved) so that the # automatic rule to close the Securework ticket is not triggered as the ticket is already closed in SCWX. incident_payload['properties']['scwx_ctp_status'] = ticket.get( "status") # Render the template. incident_id = incident.get('id') result = self._update_incident(incident_id, incident_payload) LOG.debug(incident_payload) ticket_id = ticket.get('ticketId') if result and result.get('success'): message = u"Closed incident {0} for Secureworks CTP ticket {1}".format( incident_id, ticket_id) LOG.info(message) else: message = u"Unable to update incident {0} for closing. Secureworks CTP ticket {1}".format( incident_id, ticket_id) LOG.error(message) return result except Exception as err: raise IntegrationError(err)
def _process_approval_request(self, event): # Process one approval request log = self.log request = event.request request_id = request["id"] # special "test the process by escalating a single request" mode test_single_request = self.options.get("test_single_request") if test_single_request: if str(request_id) not in str(test_single_request).split(","): log.info(u"Skipping request %s, test", request_id) return # Find the Resilient incident corresponding to this CbProtect approval request (if available) resilient_incident = self._find_resilient_incident_for_req(request_id) if resilient_incident: log.info(u"Skipping request %s, already escalated", request_id) return log.info(u"Processing request %s", request_id) try: # Create a new Resilient incident from this approval request # using a JSON (JINJA2) template file template_file_path = self.options.get("template_file") if template_file_path and not os.path.exists(template_file_path): log.warn(u"Template file '%s' not found.", template_file_path) template_file_path = None if not template_file_path: # Use the template file installed by this package template_file_path = resource_filename( Requirement("fn-cb-protection"), "fn_cb_protection/data/template.jinja") if not os.path.exists(template_file_path): raise Exception(u"Template file '{}' not found".format( template_file_path)) log.info(u"Template file: %s", template_file_path) with open(template_file_path, "r") as definition: escalate_template = definition.read() # Render the template. Be sure to set the CbProtect ID in the result! new_resilient_inc = render_json(escalate_template, request) new_resilient_inc["properties"][REQUEST_ID_FIELDNAME] = request_id log.debug(new_resilient_inc) inc = self.rest_client().post("/incidents", new_resilient_inc) rs_inc_id = inc["id"] message = u"Created incident {} for CbProtect {}".format( rs_inc_id, request_id) log.info(message) except Exception as exc: log.exception(exc) raise
def _update_custom_fields(self, incident, ticket): """ Update a Resilient incident by rendering a jinja2 template :param ticket: Secureworks CTP ticket (json object) :return: Resilient incident """ try: # Update Resilient custom incident fields from this ticket # using a JSON (JINJA2) template file template_file_path = self.options.get('template_file_update') if template_file_path and not os.path.exists(template_file_path): LOG.warning(u"Template file '%s' not found.", template_file_path) template_file_path = None if not template_file_path: # Use the template file installed by this package template_file_path = resource_filename( Requirement("fn-secureworks-ctp"), "fn_secureworks_ctp/data/scwx_ctp_template_update.jinja") if not os.path.exists(template_file_path): raise Exception( u"Template file for updating incident'{0}' not found". format(template_file_path)) LOG.info( u"Secureworks CTP jinja template file for updating incident: %s", template_file_path) with open(template_file_path, "r") as definition: update_template = definition.read() incident_payload = render_json(update_template, ticket) # Render the template. incident_id = incident.get('id') result = self._update_incident(incident_id, incident_payload) LOG.debug(incident_payload) ticket_id = ticket.get('ticketId') if result and result.get('success'): message = u"Updated incident {0} for Secureworks CTP ticket {1}".format( incident_id, ticket_id) LOG.info(message) else: message = u"Unable to update incident {0} for Secureworks CTP ticket {1}".format( incident_id, ticket_id) LOG.error(message) return result except Exception as err: raise IntegrationError(err)
def make_payload_from_template(self, template_override, default_template, payload): """convert a payload into a newformat based on a specified template Args: template_override ([str]): [/path/to/template.jinja] default_template ([str]): [/path/to/template.jinja] payload ([dict]): [data to convert] Returns: [dict]: [converted payload] """ template_data = self.get_template(template_override, default_template) # Render the template. rendered_payload = render_json(template_data, payload) LOG.debug(rendered_payload) return rendered_payload
def _create_incident(self, ticket): """ Create a new Resilient incident by rendering a jinja2 template :param ticket: Secureworks CTP ticket (json object) :return: Resilient incident """ ticket_id = ticket.get('ticketId') try: # Create a new Resilient incident from this ticket # using a JSON (JINJA2) template file template_file_path = self.options.get('template_file_escalate') if template_file_path and not os.path.exists(template_file_path): LOG.warning(u"Template file '%s' not found.", template_file_path) template_file_path = None if not template_file_path: # Use the template file installed by this package template_file_path = resource_filename( Requirement("fn-secureworks-ctp"), "fn_secureworks_ctp/data/scwx_ctp_template_escalate.jinja") if not os.path.exists(template_file_path): raise Exception(u"Template file '{0}' not found".format( template_file_path)) LOG.info(u"Secureworks CTP Template file: %s", template_file_path) with open(template_file_path, "r") as definition: escalate_template = definition.read() # Render the template. new_incident_payload = render_json(escalate_template, ticket) LOG.debug(new_incident_payload) # Post incident to Resilient incident = self.rest_client().post("/incidents", new_incident_payload) incident_id = incident.get('id') message = u"Created incident {0} for Secureworks CTP ticket {1}".format( incident_id, ticket_id) LOG.info(message) return incident except Exception as err: raise IntegrationError(err)
def codegen_from_template(client, export_file, template_file_path, package, message_destination_names, function_names, workflow_names, action_names, field_names, datatable_names, task_names, script_names, output_dir, output_file): """Based on a template-file, produce the generated file or package. To codegen a single file, the template will be a JSON dict with just one entry, such as {"file_to_generate.py": "path/to/template.jinja2"} To codegen a whole directory, the template dict can have multiple values, including nested subdirectories. Each source ("path/to/template.jinja2") will be rendered using jinja2, then written to the target ("file_to_generate.py"). :param client: the REST client :param export_file: file containing customization exports (default is to use the server's latest) :param template_file_path: location of templates :param package: name of the package to be generated :param message_destination_names: list of message desctinations; generate all the functions that use them :param function_names: list of named functions to be generated :param workflow_names: list of workflows whose customization def should be included in the package :param action_names: list of actions (rules) whose customization def should be included in the package :param field_names: list of incident fields whose customization def should be included in the package :param datatable_names: list of data tables whose customization def should be included in the package :param task_names: list of automatic tasks whose customization def should be included in the package :param script_names: list of scripts whose customization def should be included in the package :param output_dir: output location :param output_file: output file name """ functions = {} function_params = {} message_destinations = {} incident_fields = {} action_fields = {} datatables = {} datatable_fields = {} phases = {} automatic_tasks = {} scripts = {} workflows = {} actions = {} if export_file: with io.open(export_file, 'r', encoding="utf-8") as export: export_data = json.loads(export.read()) LOG.info( u"Codegen is based on the organization export from '{}'.".format( export_file)) else: # Get the most recent org export that includes actions and tasks export_uri = "/configurations/exports/history" export_list = client.get(export_uri)["histories"] last_date = 0 last_id = 0 for export in export_list: if export["options"]["actions"] and export["options"][ "phases_and_tasks"]: if export["date"] > last_date: last_date = export["date"] last_id = export["id"] if last_date == 0: LOG.error( u"ERROR: No suitable export is available. " u"Create an export for code generation. (Administrator Settings -> Organization -> Export)." ) return dt = datetime.datetime.utcfromtimestamp(last_date / 1000.0) LOG.info( u"Codegen is based on the organization export from {}.".format(dt)) export_uri = "/configurations/exports/{}".format(last_id) export_data = client.get(export_uri) all_destinations = dict( (dest["programmatic_name"], dest) for dest in export_data.get("message_destinations", [])) all_destinations_2 = dict( (dest["name"], dest) for dest in export_data.get("message_destinations", [])) if function_names or message_destination_names: # Check that 'functions' are available (v30 onward) function_defs = export_data.get("functions") if not function_defs: LOG.error(u"ERROR: Functions are not available in this export.") return function_names = function_names or [] available_names = [ function_def["name"] for function_def in function_defs ] if message_destination_names: # Build a list of all the functions that use the specified message destination(s) for function_def in function_defs: if function_def[ "destination_handle"] in message_destination_names: function_names.append(function_def["name"]) # Check that each named function is available for function_name in function_names or []: if function_name not in available_names: LOG.error(u"ERROR: Function '%s' not found in this export.", function_name) list_functions(function_defs) return # Check that the named message destination is available for message_destination_name in message_destination_names or []: if message_destination_name not in all_destinations: LOG.error( u"ERROR: Message destination '%s' not found in this export.", message_destination_name) list_message_destinations( export_data.get("message_destinations")) return if workflow_names: # Check that 'workflows' are available (v28 onward) workflow_defs = export_data.get("workflows") if not workflow_defs: LOG.error(u"ERROR: Workflows are not available in this export.") return else: workflow_names = [] if action_names: # Check that 'actions' are available action_defs = export_data.get("actions") if not action_defs: LOG.error(u"ERROR: Rules are not available in this export.") return # Check that each named action is available actions = { action_def["name"]: clean(copy.deepcopy(action_def), ACTION_ATTRIBUTES) for action_def in action_defs if action_def["name"] in action_names } all_action_fields = dict((field["uuid"], field) for field in export_data.get("fields") if field["type_id"] == ACTION_TYPE_ID) for action_name in action_names: if action_name not in actions: LOG.error(u"ERROR: Rule '%s' not found in this export.", action_name) list_actions(action_defs) return action_def = actions[action_name] # Get the activity-fields for this action (if any) action_field_uuids = [ item.get("content") for item in action_def["view_items"] if "content" in item ] fields = [] for field_uuid in action_field_uuids: field = copy.deepcopy(all_action_fields.get(field_uuid)) clean(field, ACTION_FIELD_ATTRIBUTES) for template in field.get("templates", []): clean(template, TEMPLATE_ATTRIBUTES) for value in field.get("values", []): clean(value, VALUE_ATTRIBUTES) fields.append(field) action_fields[field["name"]] = field # Get the workflow(s) for this rule (if any) wf_names = action_def["workflows"] for wf_name in wf_names: if wf_name not in workflow_names: workflow_names.append(wf_name) # Get the message destination(s) for this rule (if any) dest_names = action_def["message_destinations"] for dest_name in dest_names: if dest_name not in message_destinations: dest = copy.deepcopy(all_destinations_2[dest_name]) clean(dest, MESSAGE_DESTINATION_ATTRIBUTES) message_destinations[dest_name] = dest all_functions = dict((function["name"], function) for function in export_data.get("functions")) all_function_fields = dict((field["uuid"], field) for field in export_data.get("fields") if field["type_id"] == FUNCTION_TYPE_ID) for function_name in (function_names or []): # Get the function definition function_def = copy.deepcopy(all_functions.get(function_name)) # Remove the attributes we don't want to serialize clean(function_def, FUNCTION_ATTRIBUTES) for view_item in function_def.get("view_items", []): clean(view_item, VIEW_ITEM_ATTRIBUTES) functions[function_name] = function_def # Get the parameters (input fields) for this function param_names = [ item.get("content") for item in function_def["view_items"] if "content" in item ] params = [] for param_name in param_names: param = copy.deepcopy(all_function_fields[param_name]) clean(param, FUNCTION_FIELD_ATTRIBUTES) for template in param.get("templates", []): clean(template, TEMPLATE_ATTRIBUTES) for value in param.get("values", []): clean(value, VALUE_ATTRIBUTES) params.append(param) function_params[param["uuid"]] = param # Get the message destination for this function dest_name = function_def["destination_handle"] if dest_name not in message_destinations: dest = copy.deepcopy(all_destinations[dest_name]) clean(dest, MESSAGE_DESTINATION_ATTRIBUTES) message_destinations[dest_name] = dest if workflow_names: all_workflows = dict((workflow["programmatic_name"], workflow) for workflow in export_data.get("workflows")) for workflow_name in workflow_names: # Get the workflow definition workflow_def = all_workflows.get(workflow_name) if workflow_def: # Remove the attributes we don't want to serialize workflow = clean(copy.deepcopy(workflow_def), WORKFLOW_ATTRIBUTES) clean(workflow["content"], WORKFLOW_CONTENT_ATTRIBUTES) workflows[workflow_name] = workflow else: LOG.error(u"ERROR: Workflow '%s' not found in this export.", workflow_name) list_workflows(export_data.get("workflows")) return if field_names: # Get definitions for custom incident fields all_fields = dict((field["name"], field) for field in export_data.get("fields") if field["type_id"] == INCIDENT_TYPE_ID and field.get("prefix") == "properties") for field_name in field_names: fielddef = all_fields.get(field_name) if fielddef: field = clean(copy.deepcopy(fielddef), INCIDENT_FIELD_ATTRIBUTES) for template in field.get("templates", []): clean(template, TEMPLATE_ATTRIBUTES) for value in field.get("values", []): clean(value, VALUE_ATTRIBUTES) incident_fields[field["uuid"]] = field else: LOG.error( u"ERROR: Custom incident field '%s' not found in this export.", field_name) list_incident_fields(export_data.get("fields")) return if datatable_names: # Get datatable definitions all_datatables = dict((table["type_name"], table) for table in export_data.get("types") if table["type_id"] == DATATABLE_TYPE_ID) for datatable_name in datatable_names: datatable = all_datatables.get(datatable_name) if datatable: for (fieldname, fielddef) in datatable["fields"].items(): field = clean(copy.deepcopy(fielddef), DATATABLE_FIELD_ATTRIBUTES) for template in field.get("templates", []): clean(template, TEMPLATE_ATTRIBUTES) for value in field.get("values", []): clean(value, VALUE_ATTRIBUTES) datatable_fields[field["uuid"]] = field datatables[datatable_name] = datatable else: LOG.error(u"ERROR: Datatable '%s' not found in this export.", datatable_name) list_datatables(export_data.get("types", [])) return # Automtic tasks determine the list of phases phase_names = set() if task_names: # Get task definitions all_tasks = dict((task["programmatic_name"], task) for task in export_data.get("automatic_tasks")) for task_name in task_names: task = all_tasks.get(task_name) if task: automatic_tasks[task_name] = clean(copy.deepcopy(task), AUTOMATIC_TASK_ATTRIBUTES) phase_names.add(task["phase_id"]) else: LOG.error(u"ERROR: Task '%s' not found in this export.", task_name) list_automatic_tasks(export_data.get("automatic_tasks", [])) return if phase_names: # Get phase definitions all_phases = dict( (phase["name"], phase) for phase in export_data.get("phases")) for phase_name in phase_names: # Assume phase-name is found. It was derived from the automatic task. phase = all_phases[phase_name] phases[phase_name] = clean(copy.deepcopy(phase), PHASE_ATTRIBUTES) if script_names: # Get script definitions all_scripts = dict( (script["name"], script) for script in export_data.get("scripts")) for script_name in script_names: script = all_scripts.get(script_name) if script: scripts[script_name] = clean(copy.deepcopy(script), SCRIPT_ATTRIBUTES) else: LOG.error(u"ERROR: Script '%s' not found in this export.", script_name) list_scripts(export_data.get("scripts", [])) return # Minify the export_data fields_list = [] if len(incident_fields) == 0: # import requires at least one, use placeholder fields_list.extend(["incident/inc_training"]) else: fields_list.extend([ "incident/{}".format(fld["name"]) for fld in incident_fields.values() ]) fields_list.extend([ "actioninvocation/{}".format(fld["name"]) for fld in action_fields.values() ]) fields_list.extend([ "__function/{}".format(fld["name"]) for fld in function_params.values() ]) keep_keys = [ "export_date", "export_format_version", "id", "server_version" ] minify_keys = { "actions": { "name": actions.keys() }, "automatic_tasks": { "programmatic_name": automatic_tasks.keys() }, "fields": { "export_key": fields_list }, "functions": { "name": functions.keys() }, "message_destinations": { "programmatic_name": message_destinations.keys() }, "phases": { "name": phases.keys() }, "scripts": { "name": scripts.keys() }, "types": { "type_name": datatables.keys() }, "workflows": { "programmatic_name": workflows.keys() }, } for key in export_data.keys(): if key in keep_keys: pass elif key in minify_keys.keys(): name = list(minify_keys[key].keys())[0] # The property we match on values = minify_keys[key][ name] # These are the names of the things to keep for data in list(export_data[key]): if not data.get(name): LOG.warning("No %s in %s", name, key) if not data.get(name) in values: export_data[key].remove(data) elif isinstance(export_data[key], list): export_data[key] = [] elif isinstance(export_data[key], dict): export_data[key] = {} else: export_data[key] = None # Incident types are special, add one for this specific package # (because not enabled, this doesn't actually get loaded into the destination) t0 = int(time.time() * 1000) export_data["incident_types"] = [{ "update_date": t0, "create_date": t0, "uuid": str(UUID_CODEGEN), "description": "Customization Packages (internal)", "export_key": "Customization Packages (internal)", "name": "Customization Packages (internal)", "enabled": False, "system": False, "parent_id": None, "hidden": False, "id": 0 }] # Prepare the dictionary of substitution values for jinja2 # (includes all the configuration elements related to the functions) data = { "package": package, "function_names": function_names, "output_dir": output_dir, "output_file": output_file, "functions": functions, "function_params": function_params, "message_destinations": message_destinations, "incident_fields": incident_fields, "action_fields": action_fields, "datatables": datatables, "datatable_fields": datatable_fields, "phases": phases, "automatic_tasks": automatic_tasks, "scripts": scripts, "workflows": workflows, "actions": actions, "export_data": export_data } LOG.debug(u"Configuration data:\n%s", json.dumps(data, indent=2)) # Read the files/package template and render it # to produce the file-mapping dictionary from template-files to generated-files with io.open(template_file_path, 'r', encoding="utf-8") as template_file: file_mapping_template = template_file.read() file_mapping = template_functions.render_json(file_mapping_template, data) LOG.debug(u"Codegen template:\n%s", json.dumps(file_mapping, indent=2)) # Write all the files defined in the mapping definition src_dir = os.path.dirname(template_file_path) render_file_mapping(file_mapping, data, src_dir, output_dir)
def rest_call(options, query_definition, event_message): """ Make a REST call and return result """ # options timeout = int(options.get("query_timeout", 60)) verify = options.get("verify", "") if verify[:1].lower() in ("0", "f", "n"): verify = False else: verify = True # The REST URL to call is the rendered query expression rest_url = query_definition.query # The REST method is can be set in 'vars' http_method = query_definition.vars.get("http-method", "GET") LOG.debug("HTTP method: %s", http_method) # HTTP headers can be set in 'vars' http_headers = query_definition.vars.get("http-headers", {}) LOG.debug("HTTP headers: %s", http_headers) # HTTP post body can be set in 'vars' http_body = query_definition.vars.get("http-body") if isinstance(http_body, string_types): http_body = json.loads(http_body) LOG.debug("HTTP body: %s", http_body) session = requests.Session() error = None response = None try: response = session.request(http_method, rest_url, headers=http_headers, json=http_body, verify=verify, timeout=timeout) if response.status_code not in [200, 201]: raise SimpleHTTPException(response) response = response.json() except Exception as exc: if not query_definition.onerror: raise LOG.error(exc) error = u"{}".format(exc) if error: mapdata = copy.deepcopy(event_message) mapdata.update(query_definition.vars) mapdata.update({"query": query_definition.query}) mapdata.update({"error": error}) error_template = json.dumps({"events": [query_definition.onerror]}, indent=2) error_rendered = template_functions.render_json(error_template, mapdata) response = error_rendered if not response: LOG.warn("No data returned from query") if query_definition.default: mapdata = copy.deepcopy(event_message) mapdata.update(query_definition.vars) mapdata.update({"query": query_definition.query}) default_template = json.dumps({"events": [query_definition.default]}, indent=2) default_rendered = template_functions.render_json(default_template, mapdata) response = default_rendered LOG.debug("Response: %s", json.dumps(response)) return {"result": response}
def _do_datatable_mapping(query_definition, dtinfo, event_message, metadata, response, datatable_locks, res_client, context_token, additional_map_data=None): """ Map query results to Resilient data table rows """ incident = event_message.get("incident", {}) incident_id = incident.get("id") if query_definition.result_container: rows = response else: rows = [ response, ] # Contains: "name", "key" (optional), "cells" (optional) dtname = dtinfo.get("name") dtkey = dtinfo.get("keys", []) dtrow_id = dtinfo.get("row_id", None) dtcells = dtinfo.get("cells", None) limit = dtinfo.get("limit", 0) # Get access to the data table if not datatable_locks[dtname].acquire(timeout=600): LOG.error("Couldn't acquire lock on table %s. No update done.", dtname) return try: datatable = DataTable(res_client, table_name=dtname) dtrows = [] row_to_update = None if dtrow_id: # We are updating a single existing row row_to_update = datatable.find_row(incident['id'], dtrow_id) if not row_to_update: LOG.error("Row [%s] not found. No update done.", dtrow_id) return elif dtkey: # Read all the rows dtrows = datatable.rows(incident_id) # Map for rendering starts with the event (incident, etc) mapdata = copy.deepcopy(event_message) if additional_map_data: mapdata.update(additional_map_data) # Add in any rendered vars mapdata.update(query_definition.vars) # Add in any query result metadata mapdata.update(metadata) LOG.debug("Key columns: %s", dtkey) cells_template = json.dumps({"cells": dtcells}, indent=2) LOG.debug("Cells template: %s", cells_template) num_created = 0 for result_row in rows: # If a key is specified, it's for upsert: # - Each row in the result should correspond to one row in the data table # - Render key with **the event_message and the result row** # (because the key could be e.g. artifact.value, or task.id, or row.somevalue) # - It looks like {"cell":"value"} when rendered # - We expect a single row matching the key, or none # (If multiple rows match the key, just pick the randomly-first one and carry on) # - Update it based on the response row, or insert mapdata["result"] = result_row # Render the result row to cells using the template provided in the query definition. cells_rendered = template_functions.render_json( cells_template, mapdata) datatable.update_cell_value_types(cells_rendered) dtrow = None if dtkey: LOG.debug("Find matching row to update!") key_dict = { key: cells_rendered["cells"].get(key, {}).get("value", None) for key in dtkey } matching_rows = datatable.match(dtrows, key_dict, limit=1) if matching_rows: dtrow = matching_rows[0] elif row_to_update: dtrow = row_to_update if dtrow is None: # Insert a new row in the data table LOG.debug("Adding Row: %s", json.dumps(cells_rendered, indent=2)) new_row = datatable.add_row(incident_id, cells_rendered) if new_row: dtrows.append(new_row) else: # Update the row in the data table LOG.debug("Updating Row: %s", json.dumps(cells_rendered, indent=2)) datatable.update(incident_id, dtrow, cells_rendered, co3_context_token=context_token) num_created = num_created + 1 if num_created == limit: LOG.info("Limiting Datatable row creation to first %d results", limit) break finally: datatable_locks[dtname].release()
def run_search(options, query_definition, event_message): """ Run Ariel search and return result """ # Read the options and construct a QRadar client qradar_url = options.get("qradar_url", "") qradar_token = options.get("qradar_service_token", "") timeout = int(options.get("query_timeout", 600)) polling_interval = int(options.get("polling_interval", 5)) if not all((qradar_url, qradar_token, timeout, polling_interval)): LOG.error("Configuration file missing required values!") raise Exception("Missing Configuration Values") verify = options.get("qradar_verify", "") if verify[:1].lower() in ("0", "f", "n"): verify = False else: verify = True qradar_client = QRadarClient(qradar_url, qradar_token, verify=verify) error = None response = None try: params = {'query_expression': query_definition.query} url = "ariel/searches" response = qradar_client.post(url, params=params) LOG.debug(response) search_id = response.get('search_id', '') if not search_id: error = "Query Failed: " + response.get("message", "No Error Message Found") else: LOG.info("Queued Search %s", search_id) _wait_for_query_to_complete(search_id, qradar_client, timeout, polling_interval) # Query Execution Finished, Get Results response = _get_query_results(search_id, qradar_client, query_definition.range) except Exception as exc: if not query_definition.onerror: raise LOG.error(exc) error = u"{}".format(exc) if error: mapdata = copy.deepcopy(event_message) mapdata.update(query_definition.vars) mapdata.update({"query": query_definition.query}) mapdata.update({"error": error}) error_template = json.dumps({"events": [query_definition.onerror]}, indent=2) error_rendered = template_functions.render_json( error_template, mapdata) response = error_rendered if not response or len(response["events"]) == 0: LOG.warn("No data returned from query") if query_definition.default: mapdata = copy.deepcopy(event_message) mapdata.update(query_definition.vars) mapdata.update({"query": query_definition.query}) default_template = json.dumps( {"events": [query_definition.default]}, indent=2) default_rendered = template_functions.render_json( default_template, mapdata) response = default_rendered return response
def run_search(options, query_definition, event_message): """ Run LDAP search and return result """ # Read the LDAP configuration options ldap_server = options["server"] ldap_port = int(options["port"] or LDAP_DEFAULT_PORT) ldap_user = options["user"] ldap_password = options["password"] ldap_ssl = options["ssl"] == "True" # anything else is false ldap_auth = LDAP_AUTH_TYPES[options["auth"] or "ANONYMOUS"] results = None # CLIENT Active Directory client_ad_server = Server(ldap_server, ldap_port, get_info=ldap3.ALL, use_ssl=options["ssl"] == "True", connect_timeout=3) client_ad_creds = (ldap_user, ldap_password, ldap_auth) if query_definition.params is None: raise Exception("LDAP query requires 'search_base' parameter") search_base = query_definition.params.get("search_base") if search_base is None: raise Exception("LDAP query requires 'search_base' parameter") # Connect to the LDAP server LOG.debug("LDAP connect") with Connection(client_ad_server, user=client_ad_creds[0], password=client_ad_creds[1], authentication=client_ad_creds[2], auto_bind=True) as conn: LOG.debug("LDAP search {0} / {1}".format(search_base, query_definition.query)) conn.search(search_base, query_definition.query, attributes=ldap3.ALL_ATTRIBUTES) entries = conn.entries if entries is None: LOG.info("LDAP query returned None") results = {"entries": None} if query_definition.default: mapdata = copy.deepcopy(event_message) mapdata.update(query_definition.vars) mapdata.update({"query": query_definition.query}) default_template = json.dumps( {"entries": [query_definition.default]}, indent=2) default_rendered = template_functions.render_json( default_template, mapdata) results = default_rendered else: # List of entries. entries = json.loads(conn.response_to_json())["entries"] LOG.info("Result contains %s entries", len(entries)) # Each entry has 'dn' and dict of 'attributes'. Move attributes to the top level for easier processing. for entry in entries: entry.update(entry.pop("attributes", None)) results = {"entries": entries} LOG.debug(json.dumps(results, indent=2)) return results
def render_json(self, template, mapdata): # Render a JINJA template, using our filters etc return template_functions.render_json(template, mapdata)