class Automators(Main): def __init__(self, cfg, use_case_config): self.logger = logging.getLogger(__name__) self.logger.info('Initiating QRadar Automators') self.cfg = cfg self.use_case_config = use_case_config self.TheHiveConnector = TheHiveConnector(cfg) self.TheHiveAutomators = TheHiveAutomators(cfg, use_case_config) self.QRadarConnector = QRadarConnector(cfg) def search(self, action_config, webhook): # Only continue if the right webhook is triggered self.logger.debug("action_config:{}".format(action_config)) if webhook.isImportedAlert(): pass else: return False # Define variables and actions based on certain webhook types self.case_id = webhook.data['object']['case'] self.logger.debug(self.case_id) self.enriched = False for query_name, query_config in action_config.items(): try: self.logger.debug('Found the following query: {}'.format( query_config['query'])) self.query_variables = {} self.query_variables['input'] = {} # Render query try: # Prepare the template self.template = Template(query_config['query']) # Find variables in the template self.template_env = Environment() self.template_parsed = self.template_env.parse( query_config['query']) # Grab all the variales from the template and try to find them in the description self.template_vars = meta.find_undeclared_variables( self.template_parsed) self.logger.debug( "Found the following variables in query: {}".format( self.template_vars)) for template_var in self.template_vars: # Skip dynamically generated Stop_time variable if template_var == "Stop_Time": continue self.logger.debug( "Looking up variable required for template: {}". format(template_var)) # Replace the underscore from the variable name to a white space as this is used in the description table self.template_var_with_ws = template_var.replace( "_", " ") self.case_data = self.TheHiveConnector.getCase( self.case_id) self.logger.debug('output for get_case: {}'.format( self.case_data)) self.query_variables['input'][ template_var] = self.TheHiveAutomators.fetchValueFromMDTable( self.case_data['description'], self.template_var_with_ws) if 'Start_Time' not in self.query_variables['input']: self.logger.warning( "Could not find Start Time value required to build the search" ) # Parse times required for the query (with or without offset) if template_var == "Start_Time": self.logger.debug( "Found Start Time: %s" % self.query_variables['input']['Start_Time']) if 'start_time_offset' in query_config: self.query_variables['input'][ 'Start_Time'] = self.parseTimeOffset( self.query_variables['input'] ['Start_Time'], self.cfg.get( 'Automation', 'event_start_time_format'), query_config['start_time_offset'], self.cfg.get('QRadar', 'time_format')) else: self.query_variables['input'][ 'Start_Time'] = self.query_variables[ 'input']['Start_Time'] if 'stop_time_offset' in query_config: self.query_variables['input'][ 'Stop_Time'] = self.parseTimeOffset( self.query_variables['input'] ['Start_Time'], self.cfg.get( 'Automation', 'event_start_time_format'), query_config['stop_time_offset'], self.cfg.get('QRadar', 'time_format')) else: self.query_variables['input'][ 'Stop_Time'] = datetime.now().strftime( self.cfg.get( 'Automation', 'event_start_time_format')) self.rendered_query = self.template.render( self.query_variables['input']) self.logger.debug("Rendered the following query: %s" % self.rendered_query) except Exception as e: self.logger.warning( "Could not render query due to missing variables", exc_info=True) continue # Perform search queries try: self.rendered_query_result = self.QRadarConnector.aqlSearch( self.rendered_query) # Check results self.logger.debug( 'The search result returned the following information: \n %s' % self.rendered_query_result) except Exception as e: self.logger.warning("Could not perform query", exc_info=True) continue try: if query_config['create_thehive_task']: self.logger.debug("create task is enabled") # Task name self.uc_task_title = query_config['thehive_task_title'] self.uc_task_description = "The following information is found. Investigate the results and act accordingly:\n\n\n\n" # create a table header self.table_header = "|" self.rows = "|" if len(self.rendered_query_result['events']) != 0: for key in self.rendered_query_result['events'][ 0].keys(): self.table_header = self.table_header + " %s |" % key self.rows = self.rows + "---|" self.table_header = self.table_header + "\n" + self.rows + "\n" self.uc_task_description = self.uc_task_description + self.table_header # Create the data table for the results for event in self.rendered_query_result['events']: self.table_data_row = "|" for field_key, field_value in event.items(): # Escape pipe signs if field_value: field_value = field_value.replace( "|", "|") # Use to create some additional spacing self.table_data_row = self.table_data_row + " %s |" % field_value self.table_data_row = self.table_data_row + "\n" self.uc_task_description = self.uc_task_description + self.table_data_row else: self.uc_task_description = self.uc_task_description + "No results \n" # Add the case task self.uc_task = self.TheHiveAutomators.craftUcTask( self.uc_task_title, self.uc_task_description) self.TheHiveConnector.createTask( self.case_id, self.uc_task) except Exception as e: self.logger.debug(e) pass try: if query_config['create_ioc']: self.logger.debug("create IOC is enabled") self.comment = "offense enrichment" #static tags list self.tags = ['synapse'] #want to add SECID of the rule as well in the tag rule_secid = [ x for x in webhook.data['object']['tags'] if x.startswith('SEC') ] self.tags.extend(rule_secid) self.uc_ioc_type = query_config['ioc_type'] if len(self.rendered_query_result['events']) != 0: for event in self.rendered_query_result['events']: for field_key, field_value in event.items(): self.TheHiveConnector.addObservable( self.case_id, self.uc_ioc_type, list(field_value.split(",")), self.tags, self.comment) except Exception as e: self.logger.debug(e) pass except Exception as e: self.logger.debug( 'Could not process the following query: {}\n{}'.format( query_config, e)) continue # Return True when succesful return True def enrichAlert(self, action_config, webhook): # Only continue if the right webhook is triggered if webhook.isNewAlert(): pass else: return False # Define variables and actions based on certain webhook types # Alerts self.alert_id = webhook.data['object']['id'] self.alert_description = webhook.data['object']['description'] self.query_variables = {} self.query_variables['input'] = {} self.enriched = False # Prepare search queries for searches for query_name, query_config in action_config.items(): try: self.logger.info('Found the following query: %s' % (query_name)) self.query_variables[query_name] = {} # Render query try: # Prepare the template self.template = Template(query_config['query']) # Find variables in the template self.template_env = Environment() self.template_parsed = self.template_env.parse( query_config['query']) # Grab all the variales from the template and try to find them in the description self.template_vars = meta.find_undeclared_variables( self.template_parsed) self.logger.debug( "Found the following variables in query: {}".format( self.template_vars)) for template_var in self.template_vars: # Skip dynamically generated Stop_time variable if template_var == "Stop_Time": continue self.logger.debug( "Looking up variable required for template: {}". format(template_var)) # Replace the underscore from the variable name to a white space as this is used in the description table self.template_var_with_ws = template_var.replace( "_", " ") self.alert_data = self.TheHiveConnector.getAlert( self.alert_id) self.logger.debug('output for get_alert: {}'.format( self.alert_data)) self.query_variables['input'][ template_var] = self.TheHiveAutomators.fetchValueFromMDTable( self.alert_data['description'], self.template_var_with_ws) # Parse times required for the query (with or without offset) if template_var == "Start_Time": self.logger.debug( "Found Start Time: %s" % self.query_variables['input']['Start_Time']) if 'start_time_offset' in query_config: self.query_variables['input'][ 'Start_Time'] = self.parseTimeOffset( self.query_variables['input'] ['Start_Time'], self.cfg.get( 'Automation', 'event_start_time_format'), query_config['start_time_offset'], self.cfg.get('QRadar', 'time_format')) else: self.query_variables['input'][ 'Start_Time'] = self.query_variables[ 'input']['Start_Time'] if 'stop_time_offset' in query_config: self.query_variables['input'][ 'Stop_Time'] = self.parseTimeOffset( self.query_variables['input'] ['Start_Time'], self.cfg.get( 'Automation', 'event_start_time_format'), query_config['stop_time_offset'], self.cfg.get('QRadar', 'time_format')) else: self.query_variables['input'][ 'Stop_Time'] = datetime.now().strftime( self.cfg.get( 'Automation', 'event_start_time_format')) if not self.query_variables['input']['Start_Time']: self.logger.warning("Could not find Start Time value ") raise GetOutOfLoop self.query_variables[query_name][ 'query'] = self.template.render( self.query_variables['input']) self.logger.debug( "Rendered the following query: %s" % self.query_variables[query_name]['query']) except Exception as e: self.logger.warning( "Could not render query due to missing variables", exc_info=True) raise GetOutOfLoop # Perform search queries try: self.query_variables[query_name][ 'result'] = self.QRadarConnector.aqlSearch( self.query_variables[query_name]['query']) except Exception as e: self.logger.warning("Could not perform query", exc_info=True) raise GetOutOfLoop # Check results self.logger.debug( 'The search result returned the following information: \n %s' % self.query_variables[query_name]['result']) # making enrichment results presentable clean_enrichment_results = self.TheHiveAutomators.make_it_presentable( self.query_variables[query_name]['result']['events'][0] ['enrichment_result']) # Add results to description success = self.enrichAlertDescription( self.alert_data['description'], query_name, self.query_variables[query_name]['result']['events'][0] ['enrichment_result']) if not success: self.logger.warning( "Could not add results from the query to the description. Error: {}" .format(e)) raise GetOutOfLoop except GetOutOfLoop: pass return True
class Automators(Main): def __init__(self, cfg, use_case_config): self.logger = logging.getLogger(__name__) self.logger.info('Initiating QRadar Automators') self.cfg = cfg self.use_case_config = use_case_config self.TheHiveConnector = TheHiveConnector(cfg) self.TheHiveAutomators = TheHiveAutomators(cfg, use_case_config) self.QRadarConnector = QRadarConnector(cfg) def checkSiem(self, action_config, webhook): #Only continue if the right webhook is triggered if webhook.isImportedAlert() or webhook.isNewAlert() or webhook.isQRadarAlertUpdateFollowTrue(): pass else: return False #Define variables and actions based on certain webhook types #Alerts if webhook.isNewAlert() or webhook.isQRadarAlertUpdateFollowTrue(): self.alert_id = webhook.data['object']['id'] self.alert_description = webhook.data['object']['description'] self.supported_query_type = 'enrichment_queries' #Cases elif webhook.isImportedAlert(): self.case_id = webhook.data['object']['case'] self.supported_query_type = 'search_queries' self.query_variables = {} self.query_variables['input'] = {} self.enriched = False #Prepare search queries for searches for query_name, query_config in action_config[self.supported_query_type].items(): try: self.logger.info('Found the following query: %s' % (query_name)) self.query_variables[query_name] = {} #Render query try: #Prepare the template self.template = Template(query_config['query']) #Find variables in the template self.template_env = Environment() self.template_parsed = self.template_env.parse(query_config['query']) #Grab all the variales from the template and try to find them in the description self.template_vars = meta.find_undeclared_variables(self.template_parsed) self.logger.debug("Found the following variables in query: {}".format(self.template_vars)) for template_var in self.template_vars: #Skip dynamically generated Stop_time variable if template_var == "Stop_Time": continue self.logger.debug("Looking up variable required for template: {}".format(template_var)) #Replace the underscore from the variable name to a white space as this is used in the description table self.template_var_with_ws = template_var.replace("_", " ") self.query_variables['input'][template_var] = self.TheHiveAutomators.fetchValueFromDescription(webhook,self.template_var_with_ws) #Parse times required for the query (with or without offset) if template_var == "Start_Time": self.logger.debug("Found Start Time: %s" % self.query_variables['input']['Start_Time']) if 'start_time_offset' in query_config: self.query_variables['input']['Start_Time'] = self.parseTimeOffset(self.query_variables['input']['Start_Time'], self.cfg.get('Automation', 'event_start_time_format'), query_config['start_time_offset'], self.cfg.get('QRadar', 'time_format')) else: self.query_variables['input']['Start_Time'] = self.query_variables['input']['Start_Time'] if 'stop_time_offset' in query_config: self.query_variables['input']['Stop_Time'] = self.parseTimeOffset(self.query_variables['input']['Start_Time'], self.cfg.get('Automation', 'event_start_time_format'), query_config['stop_time_offset'], self.cfg.get('QRadar', 'time_format')) else: self.query_variables['input']['Stop_Time'] = datetime.now().strftime(self.cfg.get('Automation', 'event_start_time_format')) if not self.query_variables['input']['Start_Time']: self.logger.warning("Could not find Start Time value ") raise GetOutOfLoop self.query_variables[query_name]['query'] = self.template.render(self.query_variables['input']) self.logger.debug("Rendered the following query: %s" % self.query_variables[query_name]['query']) except Exception as e: self.logger.warning("Could not render query due to missing variables", exc_info=True) raise GetOutOfLoop #Perform search queries try: self.query_variables[query_name]['result'] = self.QRadarConnector.aqlSearch(self.query_variables[query_name]['query']) except Exception as e: self.logger.warning("Could not perform query", exc_info=True) raise GetOutOfLoop #Check results self.logger.debug('The search result returned the following information: \n %s' % self.query_variables[query_name]['result']) if self.supported_query_type == "search_queries": #Task name self.uc_task_title = query_config['task_title'] self.uc_task_description = "The following information is found. Investigate the results and act accordingly:\n\n\n\n" #create a table header self.table_header = "|" self.rows = "|" if len(self.query_variables[query_name]['result']['events']) != 0: for key in self.query_variables[query_name]['result']['events'][0].keys(): self.table_header = self.table_header + " %s |" % key self.rows = self.rows + "---|" self.table_header = self.table_header + "\n" + self.rows + "\n" self.uc_task_description = self.uc_task_description + self.table_header #Create the data table for the results for event in self.query_variables[query_name]['result']['events']: self.table_data_row = "|" for field_key, field_value in event.items(): # Escape pipe signs if field_value: field_value = field_value.replace("|", "|") # Use to create some additional spacing self.table_data_row = self.table_data_row + " %s |" % field_value self.table_data_row = self.table_data_row + "\n" self.uc_task_description = self.uc_task_description + self.table_data_row else: self.uc_task_description = self.uc_task_description + "No results \n" #Add the case task self.uc_task = self.TheHiveAutomators.craftUcTask(self.uc_task_title, self.uc_task_description) self.TheHiveConnector.createTask(self.case_id, self.uc_task) if self.supported_query_type == "enrichment_queries": #Add results to description try: if self.TheHiveAutomators.fetchValueFromDescription(webhook,query_name) != self.query_variables[query_name]['result']['events'][0]['enrichment_result']: self.regex_end_of_table = ' \|\\n\\n\\n' self.end_of_table = ' |\n\n\n' self.replacement_description = '|\n | **%s** | %s %s' % (query_name, self.query_variables[query_name]['result']['events'][0]['enrichment_result'], self.end_of_table) self.th_alert_description = self.TheHiveConnector.getAlert(self.alert_id)['description'] self.alert_description = re.sub(self.regex_end_of_table, self.replacement_description, self.th_alert_description) self.enriched = True #Update Alert with the new description field self.updated_alert = Alert self.updated_alert.description = self.alert_description self.TheHiveConnector.updateAlert(self.alert_id, self.updated_alert, ["description"]) except Exception as e: self.logger.warning("Could not add results from the query to the description. Error: {}".format(e)) raise GetOutOfLoop except GetOutOfLoop: pass return True
class Automator(): def __init__(self, webhook, cfg, automation_config, modules): """ Class constructor :return: use case report :rtype: API call """ self.logger = logging.getLogger(__name__) self.logger.info('Initiating Siem Integration') self.cfg = cfg self.app_dir = os.path.dirname(os.path.abspath(__file__)) + "/.." self.automation_config = automation_config self.TheHiveConnector = TheHiveConnector(cfg) self.webhook = webhook self.modules = modules if cfg.getboolean('Automation', 'enable_customer_list', fallback=False): self.logger.info('Loading Customer configuration') # Load optional customer config self.customer_cfg = ConfigParser( converters={ 'list': lambda x: [i.strip() for i in x.split(';')] }) self.confPath = self.app_dir + '/conf/customers.conf' try: self.logger.debug('Loading configuration from %s' % self.confPath) self.customer_cfg.read(self.confPath) self.customers = self.customer_cfg.sections() self.logger.debug('Loaded configuration for %s' % self.customers) except Exception as e: self.logger.error('%s', __name__, exc_info=True) def check_automation(self): self.logger.info( 'Start parsing use cases for the SIEM based alerts/cases') self.ucTaskId = False self.report_action = 'None' if 'tags' in self.webhook.data['object']: self.tags = self.webhook.data['object']['tags'] # Add tagging to webhooks that are missing tags elif 'artifactId' in self.webhook.data[ 'object'] and self.webhook.isCaseArtifactJob: self.logger.debug( 'Found artifact id {} for webhook {}. Retrieving tags from there' .format(self.webhook.data['object']['artifactId'], self.webhook.id)) self.tags = self.TheHiveConnector.getCaseObservable( self.webhook.data['object']['artifactId'])['tags'] else: self.tags = [] self.logger.warning("no tags found for webhook {}".format( self.webhook.id)) self.automation_regexes = self.cfg.get('Automation', 'automation_regexes', fallback=None) if not self.automation_regexes: self.logger.error( "Could not find any regexes to find tags for automation") return self.report_action self.automation_ids = self.automation_config['automation_ids'] # loop through tags to see if there is a use case present for tag in self.tags: for automation_regex in self.automation_regexes: # The tag should match this regex otherwise it is no use case try: tag = re.search(automation_regex, tag).group(0) except Exception: self.logger.debug("Tag: %s is not matching the uc regex" % tag) continue # check if use case that is provided, matches the case if tag in self.automation_ids: self.found_a_id = tag # Try to retrieve the defined actions self.use_case_actions = self.automation_ids[ self.found_a_id]['automation'] # perform actions defined for the use case for action, action_config in self.use_case_actions.items(): # Give automator information regarding the webhook as some actions are limited to the state of the alert/case self.logger.info( 'Found the following action for %s: %s, with task %s' % (self.found_a_id, action, action_config['task'])) # Add support for multiple tasks, loop them 1 by 1 if 'tasks' in action_config: for task in action_config['tasks']: action_config['task'] = task # Run actions through the automator if self.Automate(action_config, self.webhook, self.modules): continue else: self.logger.info( 'Did not find any supported actions with details: task:{} tag:{} action:{}' .format(action_config['task'], self.found_a_id, action_config['task'])) #Run actions through the automator else: if self.Automate(action_config, self.webhook, self.modules): continue else: self.logger.info( 'Fallback: Did not find any supported actions with details: task:{} tag:{} action:{}' .format(action_config['task'], self.found_a_id, action_config['task'])) self.logger.info("Report action snapshot for the tag: {}".format( self.report_action)) return self.report_action def Automate(self, task_config, webhook, modules): # Split the task name on the dot to have a module and a function variable in a list try: task = task_config['task'].split(".") # Should probably also do some matching for words to mitigate some security concerns? module_name = task[0] function_name = task[1] except Exception: self.logger.error( "{} does not seem to be a valid automator task name".format( task), exc_info=True) return try: # Load the Automators class from the module to initialise it automators = modules['automators'][module_name].Automators( self.cfg, self.automation_config) except KeyError as e: self.logger.warning( "Automator module not found: {}".format(module_name), exc_info=True) return False try: automator = getattr(automators, '{}'.format(function_name)) # Run the function for the task and return the results results = automator(task_config, webhook) # Return the results or True if the task was succesful without returning information if results: return results else: return False except KeyError as e: self.logger.warning("Automator task not found for {}: {}".format( module_name, function_name), exc_info=True) return False def enrichAlertDescription(self, alert_id, description, enrichment_key, enrichment_value): th_alert_description = self.TheHiveConnector.getAlert( alert_id)['description'] # Split again, then parse the enrichment table part. After that the two splitted parts can be put together again original_description, enrichment_table = retrieveSplittedDescription( description) # Primarily check if the split action worked. If the variables are the same, then the check if the key is found and does not contain the value already if enrichment_table and self.fetchValueFromMDTable( enrichment_table, enrichment_key) != enrichment_value: regex_end_of_table = ' \|\\n\\n\\n' end_of_table = ' |\n\n\n' replacement_description = '|\n | **%s** | %s %s' % ( enrichment_key, enrichment_value, end_of_table) alert_description = re.sub(regex_end_of_table, replacement_description, th_alert_description) enriched = True # Concat enrichment table with rest of description # Update Alert with the new description field updated_alert = Alert updated_alert.description = alert_description self.TheHiveConnector.updateAlert(alert_id, updated_alert, ["description"])