def process_event(event, sip_campaign_names): """ Process the event. """ logging.info('Starting to process event: {}'.format(event['name'])) start_time = time.time() # Store the SIP campaign names with a lowercase wiki tag version. campaign_dict = dict() for campaign in sip_campaign_names: campaign_dict[campaign.replace(' ', '').lower()] = campaign # Create the event object. try: e = Event(event, sip) except: logging.exception('Error creating the Event object: {}'.format(event['name'])) return # Build the event.json file. try: e.setup(alert_uuids=event['alerts']) except: logging.exception('Error setting up the Event object: {}'.format(event['name'])) return # Connect to the wiki page. wiki = ConfluenceEventPage(e.name_wiki, sip) # Add the wiki page URL to the event JSON. e.json['wiki_url'] = wiki.get_page_url() # If the event has changed or we are forcing a refresh, we need to update the wiki page. if e.changed or wiki.is_page_refresh_checked(): # 99% of the same things need to be updated if the event has changed or if someone # forced a refresh using the wiki page. However, there are a couple differences between # the two, so if the event has changed somehow, we want to make sure that takes # precedence over the wiki refresh button. if e.changed: wiki_refresh = False logging.info('Event has changed. Updating wiki: {}'.format(e.json['name'])) else: wiki_refresh = True logging.info('Forcing event refresh. Updating wiki: {}'.format(e.json['name'])) """ FIGURE OUT THE EVENT SOURCE """ wiki_labels = wiki.get_labels() for tag in config['core']['event_sources']: if tag in wiki_labels: event_source = config['core']['event_sources'][tag] break """ ADD ANY WHITELISTED INDICATORS FROM THE SUMMARY TABLE TO SIP """ # Read the Indicator Summary table to see if there are any checked (whitelisted) indicators. good_indicators, whitelisted_indicators = wiki.read_indicator_summary_table() if whitelisted_indicators: logging.info('Detected newly whitelisted indicators: {}'.format(e.json['name'])) # If there were any Hash indicators checked as whitelisted, we need to check if there are any related # Hash indicators that were NOT checked. If there were, we want to make sure to treat them as whitelisted. hash_cache = [] for i in whitelisted_indicators: if i['type'].startswith('Hash - '): # Loop over the indicators in the event JSON to find the matching indicator. for json_indicator in e.json['indicators']: if i['type'] == json_indicator['type'] and i['value'] == json_indicator['value']: # Loop over the relationships (the corresponding hashes) and see if any of them # are in the good indicators list (as in they were not checked as whitelisted on the wiki). relationships = json_indicator['relationships'] for rel in relationships: # Only continue if we haven't already verified this hash. if not rel in hash_cache: hash_cache.append(rel) for good_indicator in good_indicators: if good_indicator['type'].startswith('Hash - ') and good_indicator['value'] == rel: # Add the good hash indicator to the whitelisted indicator list. logging.debug('Whitelisting "{}" indicator "{}" by association to: {}'.format(good_indicator['type'], rel, i['value'])) whitelisted_indicators.append(good_indicator) if sip: # Add the whitelisted indicators to the SIP whitelist. for i in whitelisted_indicators: # If this is a "URI - Path" or "URI - URL" indicator, check its relationships to see if its # corresponding "URI - Domain Name" or "Address - ipv4-addr" indicator was also checked. If it was, # we want to ignore the path and URL indicators since the domain/IP serves as a least common denominator. # This prevents the SIP whitelist from ballooning in size and slowing things down over time. skip = False if i['type'] == 'URI - Path' or i['type'] == 'URI - URL': # Loop over the indicators in the event JSON to find the matching indicator. for json_indicator in e.json['indicators']: if i['type'] == json_indicator['type'] and i['value'] == json_indicator['value']: # Loop over the whitelisted indicators and see any of them are a whitelisted (checked) # domain name or IP address. If the domain/IP appears in the relationships (for the # "URI - Path" indicators) or in the value (for "URI - URL" indicators), we can ignore it. relationships = json_indicator['relationships'] for x in whitelisted_indicators: if x['type'] == 'URI - Domain Name' or x['type'] == 'Address - ipv4-addr': if any(x['value'] in rel for rel in relationships) or x['value'] in i['value']: logging.debug('Ignoring redundant "{}" indicator "{}" for SIP whitelist.'.format(i['type'], i['value'])) skip = True if not skip: logging.warning('Adding "{}" indicator "{}" to SIP whitelist.'.format(i['type'], i['value'])) try: data = {'references': [{'source': event_source, 'reference': wiki.get_page_url()}], 'status': 'Deprecated', 'confidence': 'low', 'impact': 'low', 'tags': ['whitelist:e2w'], 'type': i['type'], 'username': '******', 'value': i['value']} result = sip.post('indicators', data) except ConflictError: pass except: logging.exception('Error adding "{}" indicator "{}" to SIP whitelist'.format(i['type'], i['value'])) """ READ MANUAL INDICATORS FROM WIKI PAGE AND ADD NEW ONES TO SIP """ if sip: # Read whatever manual indicators are listed on the wiki page so they can be added to SIP and the event. manual_indicators = wiki.read_manual_indicators() for i in manual_indicators: # Add a "manual_indicator" tag to the indicators so that we can exclude them from the # monthly indicator pruning process. i['tags'].append('manual_indicator') # Try to add the indicator to SIP. A RequestError will be raised it it already exists. try: # Assemble the correct tags to add to this indicator. ignore_these_tags = config['wiki']['ignore_these_tags'] add_these_tags = [tag for tag in e.json['tags'] if not tag in ignore_these_tags] i['tags'] += add_these_tags # Perform the API call to add the indicator. data = {'references': [{'source': event_source, 'reference': wiki.get_page_url()}], 'confidence': 'low', 'impact': 'low', 'tags': i['tags'], 'type': i['type'], 'username': '******', 'value': i['value']} try: result = sip.post('indicators', data) logging.warning('Added "{}" manual indicator "{}" to SIP: {}'.format(i['type'], i['value'], result['id'])) except ConflictError: pass except: logging.exception('Error addding "{}" manual indicator "{}" to SIP'.format(i['type'], i['value'])) except ConflictError: # Since the indicator already exists, try to update it to make sure that it # has all of the latest wiki page tags. Start by getting the existing indicator. result = sip.get('/indicators?type={}&exact_value={}'.format(i['type'], urllib.parse.quote(i['value']))) if result: try: id_ = result[0]['id'] data = {'tags': i['tags']} sip.put('/indicators/{}'.format(id_), data) except ConflictError: pass except: logging.exception('Error updating tags on manual indicator: {}'.format(id_)) except: logging.exception('Error adding "{}" manual indicator "{}" to SIP'.format(i['type'], i['value'])) # Check if there are any manual indicators in the event JSON that do not appear in this current # reading of the Manual Indicators section. This implies that someone removed something from the # table on the wiki page and refreshed the page. Presumably this means they did not actually want # that indicator, so the best we can do for now it to change its status to Informational. # TODO: Possible improvement to this would be to search for FA Queue ACE alerts for this indicator # and FP them, which would also set the indicator's status to Informational. old_manual_indicators = [i for i in e.json['indicators'] if 'manual_indicator' in i['tags']] for old_indicator in old_manual_indicators: if not [i for i in manual_indicators if i['type'] == old_indicator['type'] and i['value'] == old_indicator['value']]: try: # Find the indicator's SIP ID and disable it. result = sip.get('indicators?type={}&exact_value={}'.format(old_indicator['type'], urllib.parse.quote(old_indicator['value']))) if result: id_ = result[0]['id'] data = {'status': 'Informational'} result = sip.put('indicators/{}'.format(id_), data) logging.error('Disabled deleted "{}" manual indicator "{}" in SIP: {}'.format(old_indicator['type'], old_indicator['value'], id_)) except: logging.exception('Error disabling deleted "{}" manual indicator "{}" in SIP'.format(old_indicator['type'], old_indicator['value'])) """ RE-SETUP THE EVENT """ # Parse the event. try: e.setup(manual_indicators=manual_indicators, force=True) except: logging.exception('Error refreshing Event object: {}'.format(e.json['name'])) return # Get the remediation status for the e-mails in the event. try: for email in e.json['emails']: email['remediated'] = False """ key = '' if email['original_recipient']: key = '{}:{}'.format(email['message_id'], email['original_recipient']) elif len(email['to_addresses']) == 1: key = '{}:{}'.format(email['message_id'], email['to_addresses'][0]) # Continue if we were able to create the MySQL "key" value for this e-mail. if key: # Search the ACE database for the remediation status. c = ace_db.cursor() query = 'SELECT * FROM remediation WHERE `key`="{}"'.format(key) c.execute(query) # Fetch all of the rows. rows = c.fetchall() for row in rows: result = row[6] # A successful result string in the database looks like: # (200) [{"address":"*****@*****.**","code":200,"message":"success"}] if '"code":200' in result and '"message":"success"' in result: email['remediated'] = True """ except: logging.exception('Error getting remediation status for e-mail.') """ ADD SIP STATUS OF EACH INDICATOR TO THE EVENT JSON """ if sip: # Used as a cache so we don't query SIP for the same indicator. queried_indicators = {} # Query SIP to get the status of the indicators. logging.debug('Querying SIP for indicator statuses.') for i in e.json['indicators']: # Skip this indicator if it is whitelisted. if i['status'] == 'Whitelisted' or i['whitelisted']: continue type_value = '{}{}'.format(i['type'], i['value']) # Continue if we haven't already processed this type/value pair indicator. if not type_value in queried_indicators: # Get the indicator status from SIP. Ignore any indicators that were already set to Informational. if not i['status'] == 'Informational': result = None try: result = sip.get('indicators?type={}&exact_value={}'.format(i['type'], urllib.parse.quote(i['value']))) except RequestError as E: if 'uri too large' in str(E).lower(): logging.warning("414 Request-URI Too Large for indicator value: {}".format(urllib.parse.quote(i['value']))) if result: id_ = result[0]['id'] result = sip.get('indicators/{}'.format(id_)) i['status'] = result['status'] # Add the indicator to the queried cache. queried_indicators[type_value] = i['status'] # We've already queried SIP for this type/value, so just set the status. else: i['status'] = queried_indicators[type_value] """ RUN ALL OF THE CLEAN INDICATOR MODULES """ e.clean_indicators() """ RUN ALL OF THE EVENT DETECTION MODULES """ # Save a copy of the old event tags to compare against to see if any were manually removed. old_tags = e.json['tags'][:] e.event_detections() """ GATHER UP ALL OF THE EVENT TAGS """ # Add the wiki tags to the event tags. This ensures that tags that we add to the wiki page # get added to the indicators in the Indicator Summary table. wiki_tags = wiki.get_labels() e.json['tags'] += wiki_tags e.json['tags'] = list(set(e.json['tags'])) # Check if the event tags have a campaign name in them. if 'campaign' in e.json['tags']: # See if any of the event tags are a valid campaign name. for tag in e.json['tags']: if tag in campaign_dict: # Set the campaign name in the event JSON. e.json['campaign'] = {'sip': campaign_dict[tag], 'wiki': tag} # Replace any campaign tag with the "apt:" version. try: e.json['tags'].append('apt:{}'.format(e.json['campaign']['wiki'])) e.json['tags'].remove(e.json['campaign']['wiki']) except: pass # Now check if any of the wiki tags were manually removed. This can happen if we have an FP # event detection, so we want to make sure we don't keep applying those FP tags to the page. # NOTE: We only do this check if the event has NOT changed and the wiki refresh button was checked. if wiki_refresh: for tag in e.json['tags'][:]: if not tag in wiki_tags: try: e.json['tags'].remove(tag) logging.info('Tag manually removed from wiki page: {}'.format(tag)) except: pass """ NOTIFY SLACK OF AN INCIDENT """ if config['slack']['enabled']: if 'incidents' in e.json['tags'] and not e.json['slack_notify']: e.json['slack_notify'] = str(datetime.datetime.now()) data = {'text': '<!channel> :rotating_light: Possible incident detected: {}'.format(e.json['wiki_url'])} try: slack_webhook_url = config['slack']['slack_webhook_url'] proxy = config['network']['proxy'] if slack_webhook_url: if proxy: requests.post(config['slack']['slack_webhook_url'], json=data, proxies={'http': proxy, 'https': proxy}) else: requests.post(config['slack']['slack_webhook_url'], json=data) except: e.json['slack_notify'] = '' logging.exception('Unable to notify Slack of incident') """ UPDATE THE WIKI PAGE """ # Refresh the wiki page using the updated JSON. try: wiki.refresh_event_page(e.json) except: logging.exception('Error refreshing wiki page: {}'.format(e.json['name'])) # Since we updated the wiki page, add the version to the event JSON. This is used # so that the intel processing button can not process a wiki page that has a newer # version without first refreshing the page. e.json['wiki_version'] = wiki.get_page_version() # Read the indicator summary table. good_indicators, whitelisted_indicators = wiki.read_indicator_summary_table() # Write out the event JSON. e.write_json() # If the intel processing checkbox is checked... if wiki.is_event_ready_for_sip_processing(e.json['wiki_version']): logging.info('Processing the event intel: {}'.format(e.json['name'])) # Figure out the event source. wiki_labels = wiki.get_labels() for tag in config['core']['event_sources']: if tag in wiki_labels: source = config['core']['event_sources'][tag] break # Add each good indicator into SIP. if sip: good_indicators, whitelisted_indicators = wiki.read_indicator_summary_table() for i in good_indicators: tags = sorted(list(set(i['tags'] + e.json['tags']))) ignore_these_tags = config['wiki']['ignore_these_tags'] for label in ignore_these_tags: try: tags.remove(label) except: pass try: data = {'references': [{'source': source, 'reference': wiki.get_page_url()}], 'status': i['status'], 'confidence': 'low', 'impact': 'low', 'tags': tags, 'type': i['type'], 'username': '******', 'value': i['value']} result = sip.post('indicators', data) except ConflictError: pass except: logging.exception('Error when adding "{}" indicator "{}" into SIP.'.format(i['type'], i['value'])) # Close the event in ACE. try: ace_api.update_event_status(e.json['ace_event']['id'], 'CLOSED') logging.warning('Closed event in ACE: {}'.format(e.json['name'])) # Update the wiki to reflect that the event was processed. wiki.update_event_processed() except: logging.exception('Error when closing the event in ACE: {}'.format(e.json['name'])) logging.info('Finished event "{0:s}" in {1:.5f} seconds.'.format(event['name'], time.time() - start_time))