def execute_hunt(self, hunt): # are we ready to run another one of these types of hunts? # NOTE this will BLOCK until a semaphore is ready OR this manager is shutting down start_time = local_time() hunt.semaphore = self.acquire_concurrency_lock() if self.manager_control_event.is_set(): if hunt.semaphore is not None: hunt.semaphore.release() return # keep track of how long it's taking to acquire the resource if hunt.semaphore is not None: self.record_semaphore_acquire_time(local_time() - start_time) # start the execution of the hunt on a new thread hunt_execution_thread = threading.Thread( target=self.execute_threaded_hunt, args=(hunt, ), name=f"Hunt Execution {hunt}") hunt_execution_thread.start() # wait for the signal that the hunt has started # this will block for a short time to ensure we don't wrap back around before the # execution lock is acquired hunt.startup_barrier.wait()
def execute_with_lock(self, *args, **kwargs): # we use this lock to determine if a hunt is running, and, to wait for execution to complete. logging.debug(f"waiting for execution lock on {self}") self.execution_lock.acquire() # remember the last time we executed self.last_executed_time = local_time() # notify the manager that this is now executing # this releases the manager thread to continue processing hunts logging.debug(f"clearing barrier for {self}") self.startup_barrier.wait() submission_list = None try: logging.info(f"executing {self}") start_time = local_time() return self.execute(*args, **kwargs) self.record_execution_time(local_time() - start_time) except Exception as e: logging.error(f"{self} failed: {e}") report_exception() self.record_hunt_exception(e) finally: self.startup_barrier.reset() self.execution_lock.release()
def get_alerts(self): # if we don't have a last_api_call, then we default to 48 hours ago if self.last_api_call is None: self.last_api_call = local_time() - datetime.timedelta(hours=48) logging.debug(f"last_api_call is empty so defaulting to 48 hours ago: {self.last_api_call}") now = local_time() duration = self.get_duration() start_time = format_iso8601(self.last_api_call) try: for alert in self.fe_client.get_alerts(self.last_api_call, duration): yield alert except requests.exceptions.HTTPError as e: if e.response.status_code in [ 502, 503 ]: logging.warning(f"fireeye returned {e.response.status_code} (unavailable)") return raise e # the next time we make this call, we start at last_api_call + duration_in_hours next_api_call = self.last_api_call + datetime.timedelta(hours=duration) if next_api_call > now: # if our duration puts us past right now, then just use right now self.last_api_call = now else: self.last_api_call = next_api_call logging.debug(f"next fireeye api call will start at {self.last_api_call}")
def acquire_concurrency_lock(self): """Acquires a concurrency lock for this type of hunt if specified in the configuration for the hunt. Returns a NetworkSemaphoreClient object if the concurrency_type is CONCURRENCY_TYPE_NETWORK_SEMAPHORE or a reference to the threading.Semaphore object if concurrency_type is CONCURRENCY_TYPE_LOCAL_SEMAPHORE. Immediately returns None if non concurrency limits are in place for this type of hunt.""" if self.concurrency_type is None: return None result = None start_time = local_time() if self.concurrency_type == CONCURRENCY_TYPE_NETWORK_SEMAPHORE: logging.debug(f"acquiring network concurrency semaphore {self.concurrency_semaphore} " f"for hunt type {self.hunt_type}") result = NetworkSemaphoreClient(cancel_request_callback=self.manager_control_event.is_set) # make sure we cancel outstanding request # when shutting down result.acquire(self.concurrency_semaphore) else: logging.debug(f"acquiring local concurrency semaphore for hunt type {self.hunt_type}") while not self.manager_control_event.is_set(): if self.concurrency_semaphore.acquire(blocking=True, timeout=0.1): result = self.concurrency_semaphore break if result is not None: total_seconds = (local_time() - start_time).total_seconds() logging.debug(f"acquired concurrency semaphore for hunt type {self.hunt_type} in {total_seconds} seconds") return result
def start_time(self): """Returns the starting time of this query based on the last time we searched.""" # if this hunt is configured for full coverage, then the starting time for the search # will be equal to the ending time of the last executed search if self.full_coverage: # have we not executed this search yet? if self.last_end_time is None: return local_time() - self.time_range else: return self.last_end_time else: # if we're not doing full coverage then we don't worry about the last end time return local_time() - self.time_range
def test_tuning_rules_observable_match(self): # sample observable layout # { # "time": "2020-02-14T20:45:00.620518+0000", # "type": "ipv4", # "value": "1.2.3.4" # }, with open(os.path.join(self.tuning_rule_dir, 'test.yar'), 'w') as fp: fp.write(""" rule test_observable { meta: targets = "observable" strings: $ = /"type": "ipv4"/ $ = /"value": "1.2.3.4"/ condition: all of them } """) submission_filter = self.create_submission_filter() submission = _custom_submission() submission.observables = [ { 'type': F_IPV4, 'value': '1.2.3.4', 'time': local_time(), }, ] matches = submission_filter.get_tuning_matches(submission) submission_filter.log_tuning_matches(submission, matches) self.assertTrue(len(matches), 1) self.assertTrue(matches[0]['rule'] == 'test_observable')
def next_execution_time(self): """Returns the next time this hunt should execute.""" # if it hasn't executed at all yet, then execute it now if self.last_executed_time is None: return local_time() return self.last_executed_time + self.frequency
def ready(self): """Returns True if the hunt is ready to execute, False otherwise.""" # if it's already running then it's not ready to run again if self.running: return False # if we haven't executed it yet then it's ready to go if self.last_executed_time is None: return True # if the difference between now and the last_end_time is >= the time_range # then we are playing catchup and we need to run again if self.last_end_time is not None and local_time() - self.last_end_time >= self.time_range: return True # otherwise we're not ready until it's past the next execution time return local_time() >= self.next_execution_time
def get_duration(self): """Returns the duration to use based on the last time we made the api call.""" result = None for hours in VALID_DURATIONS: result = hours if self.last_api_call + datetime.timedelta(hours=hours) >= local_time(): break return hours
def ready(self): """Returns True if the hunt is ready to execute, False otherwise.""" # if it's already running then it's not ready to run again if self.running: return False # if we haven't executed it yet then it's ready to go if self.last_executed_time is None: return True # otherwise we're not ready until it's past the next execution time return local_time() >= self.next_execution_time
def execute(self): # the next one to run should be the first in our list for hunt in self.hunts: if hunt.ready: self.execute_hunt(hunt) continue else: # this one isn't ready so wait for this hunt to be ready wait_time = (hunt.next_execution_time - local_time()).total_seconds() logging.info(f"next hunt is {hunt} @ {hunt.next_execution_time} ({wait_time} seconds)") self.wait_control_event.wait(wait_time) self.wait_control_event.clear() # if a hunt ends while we're waiting, wait_control_event will break out before wait_time seconds # at this point, it's possible there's another hunt ready to execute before this one we're waiting on # so no matter what, we break out so that we re-enter with a re-ordered list of hunts return
def end_time(self): """Returns the ending time of this query based on the start time and the hunt configuration.""" # if this hunt is configured for full coverage, then the ending time for the search # will be equal to the ending time of the last executed search plus the total range of the search now = local_time() if self.full_coverage: # have we not executed this search yet? if self.last_end_time is None: return now else: # if the difference in time between the end of the range and now is larger than # the time_range, then we switch to using the max_time_range, if it is configured if self.max_time_range is not None: extended_end_time = self.last_end_time + self.max_time_range if now - (self.last_end_time + self.time_range) >= self.time_range: return now if extended_end_time > now else extended_end_time return now if (self.last_end_time + self.time_range) > now else self.last_end_time + self.time_range else: # if we're not doing full coverage then we don't worry about the last end time return now
def process_query_results(self, query_results): if query_results is None: return submissions = [] # of Submission objects def _create_submission(): return Submission(description=self.description, # TODO support other analysis modes! analysis_mode=ANALYSIS_MODE_CORRELATION, tool=f'hunter-{self.type}', #tool_instance=saq.CONFIG['qradar']['url'], # XXX <-- tool_instance=self.tool_instance, type=self.type, tags=self.tags, details=[], observables=[], event_time=None, files=[]) event_grouping = {} # key = self.group_by field value, value = Submission # this is used when grouping is specified but some events don't have that field missing_group = None # map results to observables for event in query_results: observable_time = None event_time = self.extract_event_timestamp(event) or local_time() # pull the observables out of this event observables = [] for field_name, observable_type in self.observable_mapping.items(): if field_name in event and event[field_name] is not None: observable = { 'type': observable_type, 'value': event[field_name] } if field_name in self.directives: observable['directives'] = self.directives[field_name] if field_name in self.temporal_fields: observable['time'] = event_time if observable not in observables: observables.append(observable) # if we are NOT grouping then each row is an alert by itself if self.group_by is None or self.group_by not in event: submission = _create_submission() submission.event_time = event_time submission.observables = observables submission.details.append(event) submissions.append(submission) # if we are grouping but the field we're grouping by is missing elif self.group_by not in event: if missing_group is None: missing_group = _create_submission() submissions.append(missing_group) missing_group.observables.extend([_ for _ in observables if _ not in missing_group.observables]) missing_group.details.append(event) # see below about grouped events and event_time if missing_group.event_time is None: missing_group.event_time = event_time elif event_time < missing_group.event_time: missing_group.event_time = event_time # if we are grouping then we start pulling all the data into groups else: if event[self.group_by] not in event_grouping: event_grouping[event[self.group_by]] = _create_submission() event_grouping[event[self.group_by]].description += f': {event[self.group_by]}' submissions.append(event_grouping[event[self.group_by]]) event_grouping[event[self.group_by]].observables.extend([_ for _ in observables if _ not in event_grouping[event[self.group_by]].observables]) event_grouping[event[self.group_by]].details.append(event) # for grouped events, the overall event time is the earliest event time in the group # this won't really matter if the observables are temporal if event_grouping[event[self.group_by]].event_time is None: event_grouping[event[self.group_by]].event_time = event_time elif event_time < event_grouping[event[self.group_by]].event_time: event_grouping[event[self.group_by]].event_time = event_time # update the descriptions of grouped alerts with the event counts if self.group_by is not None: for submission in submissions: submission.description += f' ({len(submission.details)} events)' return submissions
fe_username = args.username fe_password = args.password if args.config: config = configparser.ConfigParser() config.read(args.config) fe_host = config['fireeye']['host'] fe_username = config['fireeye']['user_name'] fe_password = config['fireeye']['password'] with FireEyeAPIClient(fe_host, fe_username, fe_password) as api_client: # are we making a query? if args.query_alerts: start_time = args.start_time if args.start_time is None: start_time = local_time() - datetime.timedelta(hours=48) duration = args.duration if duration is None: duration = 48 for alert in api_client.get_alerts(start_time, duration): print(json.dumps(alert)) sys.exit(0) # are we downloading alert data? if args.alert_id: alert_json = None if args.alert_json is not None: target_path = _get_path(args.alert_json, args.output_dir)
def test_tuning_rules_submission_all_fields_match(self): # sample observable layout # [ # { # "time": "2020-02-14T20:45:00.620518+0000", # "type": "ipv4", # "value": "1.2.3.4" # }, # { # "time": "2020-02-14T20:45:00.620565+0000", # "type": "ipv4", # "value": "1.2.3.5" # } # ] # same as above but testing multiple rule matches with open(os.path.join(self.tuning_rule_dir, 'test.yar'), 'w') as fp: fp.write(""" rule test_description { meta: targets = "submission" strings: $ = "description = test_description" condition: all of them } rule test_analysis_mode { meta: targets = "submission" strings: $ = "analysis_mode = analysis" condition: all of them } rule test_tool { meta: targets = "submission" strings: $ = "tool = unittest_tool" condition: all of them } rule test_tool_instance { meta: targets = "submission" strings: $ = "tool_instance = unittest_tool_instance" condition: all of them } rule test_type { meta: targets = "submission" strings: $ = "type = unittest_type" condition: all of them } rule test_event_time { meta: targets = "submission" strings: $ = /\\nevent_time =/ condition: all of them } rule test_tags { meta: targets = "submission" strings: $ = /\\ntags = .*tag_1.*\\n/ condition: all of them } rule test_observable { meta: targets = "observable" strings: $ = /"type": "ipv4"/ $ = /"value": "1.2.3.5"/ condition: all of them } """) submission_filter = self.create_submission_filter() submission = _custom_submission() submission.tags = ['tag_1', 'tag_2'] submission.observables = [ { 'type': F_IPV4, 'value': '1.2.3.4', 'time': local_time(), }, { 'type': F_IPV4, 'value': '1.2.3.5', 'time': local_time(), }, ] matches = submission_filter.get_tuning_matches(submission) submission_filter.log_tuning_matches(submission, matches) # looks like there's a bug in the library that is returning multiple match results for the same match #self.assertTrue(len(matches) == 7) rule_names = [_['rule'] for _ in matches] for rule_name in [ 'test_description', 'test_analysis_mode', 'test_tool', 'test_tool_instance', 'test_type', 'test_event_time', 'test_tags', 'test_observable', ]: self.assertTrue(rule_name in rule_names)
def _execute(self, *args, **kwargs): if not self.password: logging.error(f"no password given for {self.section}. authentication will not be attempted.") return if not self.delete_emails: if not os.path.exists(self.tracking_db_path): with sqlite3.connect(self.tracking_db_path) as db: c = db.cursor() c.execute(""" CREATE TABLE IF NOT EXISTS ews_tracking ( exchange_id TEXT NOT NULL, message_id TEXT NOT NULL, insert_date INT NOT NULL )""") c.execute(""" CREATE INDEX IF NOT EXISTS idx_exchange_id ON ews_tracking(exchange_id)""") c.execute(""" CREATE INDEX IF NOT EXISTS idx_insert_date ON ews_tracking(insert_date)""") db.commit() # get the next emails from this account credentials = Credentials(self.username, self.password) config = Configuration(server=self.server, credentials=credentials, auth_type=NTLM) # TODO auth_type should be configurable _account_class = kwargs.get('account_class') or Account # Account class connects to exchange. account = _account_class(self.target_mailbox, config=config, autodiscover=False, access_type=DELEGATE) # TODO autodiscover, access_type should be configurable for folder in self.folders: path_parts = [_.strip() for _ in folder.split('/')] root = path_parts.pop(0) _account = kwargs.get('account_object') or account try: target_folder = getattr(_account, root) except AttributeError: public_folders_root = _account.public_folders_root target_folder = public_folders_root / root #print(target_folder.tree()) for path_part in path_parts: target_folder = target_folder / path_part target_folder.refresh() logging.info(f"checking for emails in {self.target_mailbox} target {folder}") total_count = 0 already_processed_count = 0 error_count = 0 for message in target_folder.all().order_by('-datetime_received'): if isinstance(message, ResponseMessageError): logging.warning(f"error when iterating mailbox {self.target_mailbox} folder {folder}: {message} ({type(message)})") continue # XXX not sure why this is happening? if message.id is None: continue total_count += 1 try: # if we're not deleting emails then we need to make sure we keep track of which ones we've already processed if not self.delete_emails: with sqlite3.connect(self.tracking_db_path) as db: c = db.cursor() c.execute("SELECT message_id FROM ews_tracking WHERE exchange_id = ?", (message.id,)) result = c.fetchone() if result is not None: #logging.debug("already processed exchange message {} message id {} from {}@{}".format( #message.id, message.message_id, self.target_mailbox, self.server)) already_processed_count += 1 continue # otherwise process the email message (subclasses deal with the site logic) self.email_received(message) except Exception as e: logging.error(f"unable to process email: {e}") report_exception() error_count += 1 if self.delete_emails: try: logging.debug(f"deleting message {message.id}") message.delete() except Exception as e: logging.error(f"unable to delete message: {e}") else: # if we're not deleting the emails then we track which ones we've already processed with sqlite3.connect(self.tracking_db_path) as db: c = db.cursor() c.execute(""" INSERT INTO ews_tracking ( exchange_id, message_id, insert_date ) VALUES ( ?, ?, ? )""", (message.id, message.message_id, local_time().timestamp())) # TODO delete anything older than X days db.commit() logging.info(f"finished checking for emails in {self.target_mailbox} target {folder}" f" total {total_count} already_processed {already_processed_count} error {error_count}")