def execute_hunt(self, hunt):
# are we ready to run another one of these types of hunts?
# NOTE this will BLOCK until a semaphore is ready OR this manager is shutting down
start_time = local_time()
hunt.semaphore = self.acquire_concurrency_lock()
if self.manager_control_event.is_set():
if hunt.semaphore is not None:
hunt.semaphore.release()
return
# keep track of how long it's taking to acquire the resource
if hunt.semaphore is not None:
self.record_semaphore_acquire_time(local_time() - start_time)
# start the execution of the hunt on a new thread
hunt_execution_thread = threading.Thread(
target=self.execute_threaded_hunt,
args=(hunt, ),
name=f"Hunt Execution {hunt}")
hunt_execution_thread.start()
# wait for the signal that the hunt has started
# this will block for a short time to ensure we don't wrap back around before the
# execution lock is acquired
hunt.startup_barrier.wait()
def execute_with_lock(self, *args, **kwargs):
# we use this lock to determine if a hunt is running, and, to wait for execution to complete.
logging.debug(f"waiting for execution lock on {self}")
self.execution_lock.acquire()
# remember the last time we executed
self.last_executed_time = local_time()
# notify the manager that this is now executing
# this releases the manager thread to continue processing hunts
logging.debug(f"clearing barrier for {self}")
self.startup_barrier.wait()
submission_list = None
try:
logging.info(f"executing {self}")
start_time = local_time()
return self.execute(*args, **kwargs)
self.record_execution_time(local_time() - start_time)
except Exception as e:
logging.error(f"{self} failed: {e}")
report_exception()
self.record_hunt_exception(e)
finally:
self.startup_barrier.reset()
self.execution_lock.release()
def get_alerts(self):
# if we don't have a last_api_call, then we default to 48 hours ago
if self.last_api_call is None:
self.last_api_call = local_time() - datetime.timedelta(hours=48)
logging.debug(f"last_api_call is empty so defaulting to 48 hours ago: {self.last_api_call}")
now = local_time()
duration = self.get_duration()
start_time = format_iso8601(self.last_api_call)
try:
for alert in self.fe_client.get_alerts(self.last_api_call, duration):
yield alert
except requests.exceptions.HTTPError as e:
if e.response.status_code in [ 502, 503 ]:
logging.warning(f"fireeye returned {e.response.status_code} (unavailable)")
return
raise e
# the next time we make this call, we start at last_api_call + duration_in_hours
next_api_call = self.last_api_call + datetime.timedelta(hours=duration)
if next_api_call > now: # if our duration puts us past right now, then just use right now
self.last_api_call = now
else:
self.last_api_call = next_api_call
logging.debug(f"next fireeye api call will start at {self.last_api_call}")
def acquire_concurrency_lock(self):
"""Acquires a concurrency lock for this type of hunt if specified in the configuration for the hunt.
Returns a NetworkSemaphoreClient object if the concurrency_type is CONCURRENCY_TYPE_NETWORK_SEMAPHORE
or a reference to the threading.Semaphore object if concurrency_type is CONCURRENCY_TYPE_LOCAL_SEMAPHORE.
Immediately returns None if non concurrency limits are in place for this type of hunt."""
if self.concurrency_type is None:
return None
result = None
start_time = local_time()
if self.concurrency_type == CONCURRENCY_TYPE_NETWORK_SEMAPHORE:
logging.debug(f"acquiring network concurrency semaphore {self.concurrency_semaphore} "
f"for hunt type {self.hunt_type}")
result = NetworkSemaphoreClient(cancel_request_callback=self.manager_control_event.is_set)
# make sure we cancel outstanding request
# when shutting down
result.acquire(self.concurrency_semaphore)
else:
logging.debug(f"acquiring local concurrency semaphore for hunt type {self.hunt_type}")
while not self.manager_control_event.is_set():
if self.concurrency_semaphore.acquire(blocking=True, timeout=0.1):
result = self.concurrency_semaphore
break
if result is not None:
total_seconds = (local_time() - start_time).total_seconds()
logging.debug(f"acquired concurrency semaphore for hunt type {self.hunt_type} in {total_seconds} seconds")
return result
def start_time(self):
"""Returns the starting time of this query based on the last time we searched."""
# if this hunt is configured for full coverage, then the starting time for the search
# will be equal to the ending time of the last executed search
if self.full_coverage:
# have we not executed this search yet?
if self.last_end_time is None:
return local_time() - self.time_range
else:
return self.last_end_time
else:
# if we're not doing full coverage then we don't worry about the last end time
return local_time() - self.time_range
def test_tuning_rules_observable_match(self):
# sample observable layout
# {
# "time": "2020-02-14T20:45:00.620518+0000",
# "type": "ipv4",
# "value": "1.2.3.4"
# },
with open(os.path.join(self.tuning_rule_dir, 'test.yar'), 'w') as fp:
fp.write("""
rule test_observable {
meta:
targets = "observable"
strings:
$ = /"type": "ipv4"/
$ = /"value": "1.2.3.4"/
condition:
all of them
}
""")
submission_filter = self.create_submission_filter()
submission = _custom_submission()
submission.observables = [
{
'type': F_IPV4,
'value': '1.2.3.4',
'time': local_time(),
},
]
matches = submission_filter.get_tuning_matches(submission)
submission_filter.log_tuning_matches(submission, matches)
self.assertTrue(len(matches), 1)
self.assertTrue(matches[0]['rule'] == 'test_observable')
def next_execution_time(self):
"""Returns the next time this hunt should execute."""
# if it hasn't executed at all yet, then execute it now
if self.last_executed_time is None:
return local_time()
return self.last_executed_time + self.frequency
def ready(self):
"""Returns True if the hunt is ready to execute, False otherwise."""
# if it's already running then it's not ready to run again
if self.running:
return False
# if we haven't executed it yet then it's ready to go
if self.last_executed_time is None:
return True
# if the difference between now and the last_end_time is >= the time_range
# then we are playing catchup and we need to run again
if self.last_end_time is not None and local_time() - self.last_end_time >= self.time_range:
return True
# otherwise we're not ready until it's past the next execution time
return local_time() >= self.next_execution_time
def get_duration(self):
"""Returns the duration to use based on the last time we made the api call."""
result = None
for hours in VALID_DURATIONS:
result = hours
if self.last_api_call + datetime.timedelta(hours=hours) >= local_time():
break
return hours
def ready(self):
"""Returns True if the hunt is ready to execute, False otherwise."""
# if it's already running then it's not ready to run again
if self.running:
return False
# if we haven't executed it yet then it's ready to go
if self.last_executed_time is None:
return True
# otherwise we're not ready until it's past the next execution time
return local_time() >= self.next_execution_time
def execute(self):
# the next one to run should be the first in our list
for hunt in self.hunts:
if hunt.ready:
self.execute_hunt(hunt)
continue
else:
# this one isn't ready so wait for this hunt to be ready
wait_time = (hunt.next_execution_time - local_time()).total_seconds()
logging.info(f"next hunt is {hunt} @ {hunt.next_execution_time} ({wait_time} seconds)")
self.wait_control_event.wait(wait_time)
self.wait_control_event.clear()
# if a hunt ends while we're waiting, wait_control_event will break out before wait_time seconds
# at this point, it's possible there's another hunt ready to execute before this one we're waiting on
# so no matter what, we break out so that we re-enter with a re-ordered list of hunts
return
def end_time(self):
"""Returns the ending time of this query based on the start time and the hunt configuration."""
# if this hunt is configured for full coverage, then the ending time for the search
# will be equal to the ending time of the last executed search plus the total range of the search
now = local_time()
if self.full_coverage:
# have we not executed this search yet?
if self.last_end_time is None:
return now
else:
# if the difference in time between the end of the range and now is larger than
# the time_range, then we switch to using the max_time_range, if it is configured
if self.max_time_range is not None:
extended_end_time = self.last_end_time + self.max_time_range
if now - (self.last_end_time + self.time_range) >= self.time_range:
return now if extended_end_time > now else extended_end_time
return now if (self.last_end_time + self.time_range) > now else self.last_end_time + self.time_range
else:
# if we're not doing full coverage then we don't worry about the last end time
return now
def process_query_results(self, query_results):
if query_results is None:
return
submissions = [] # of Submission objects
def _create_submission():
return Submission(description=self.description,
# TODO support other analysis modes!
analysis_mode=ANALYSIS_MODE_CORRELATION,
tool=f'hunter-{self.type}',
#tool_instance=saq.CONFIG['qradar']['url'], # XXX <--
tool_instance=self.tool_instance,
type=self.type,
tags=self.tags,
details=[],
observables=[],
event_time=None,
files=[])
event_grouping = {} # key = self.group_by field value, value = Submission
# this is used when grouping is specified but some events don't have that field
missing_group = None
# map results to observables
for event in query_results:
observable_time = None
event_time = self.extract_event_timestamp(event) or local_time()
# pull the observables out of this event
observables = []
for field_name, observable_type in self.observable_mapping.items():
if field_name in event and event[field_name] is not None:
observable = { 'type': observable_type,
'value': event[field_name] }
if field_name in self.directives:
observable['directives'] = self.directives[field_name]
if field_name in self.temporal_fields:
observable['time'] = event_time
if observable not in observables:
observables.append(observable)
# if we are NOT grouping then each row is an alert by itself
if self.group_by is None or self.group_by not in event:
submission = _create_submission()
submission.event_time = event_time
submission.observables = observables
submission.details.append(event)
submissions.append(submission)
# if we are grouping but the field we're grouping by is missing
elif self.group_by not in event:
if missing_group is None:
missing_group = _create_submission()
submissions.append(missing_group)
missing_group.observables.extend([_ for _ in observables if _ not in missing_group.observables])
missing_group.details.append(event)
# see below about grouped events and event_time
if missing_group.event_time is None:
missing_group.event_time = event_time
elif event_time < missing_group.event_time:
missing_group.event_time = event_time
# if we are grouping then we start pulling all the data into groups
else:
if event[self.group_by] not in event_grouping:
event_grouping[event[self.group_by]] = _create_submission()
event_grouping[event[self.group_by]].description += f': {event[self.group_by]}'
submissions.append(event_grouping[event[self.group_by]])
event_grouping[event[self.group_by]].observables.extend([_ for _ in observables if _ not in
event_grouping[event[self.group_by]].observables])
event_grouping[event[self.group_by]].details.append(event)
# for grouped events, the overall event time is the earliest event time in the group
# this won't really matter if the observables are temporal
if event_grouping[event[self.group_by]].event_time is None:
event_grouping[event[self.group_by]].event_time = event_time
elif event_time < event_grouping[event[self.group_by]].event_time:
event_grouping[event[self.group_by]].event_time = event_time
# update the descriptions of grouped alerts with the event counts
if self.group_by is not None:
for submission in submissions:
submission.description += f' ({len(submission.details)} events)'
return submissions
fe_username = args.username
fe_password = args.password
if args.config:
config = configparser.ConfigParser()
config.read(args.config)
fe_host = config['fireeye']['host']
fe_username = config['fireeye']['user_name']
fe_password = config['fireeye']['password']
with FireEyeAPIClient(fe_host, fe_username, fe_password) as api_client:
# are we making a query?
if args.query_alerts:
start_time = args.start_time
if args.start_time is None:
start_time = local_time() - datetime.timedelta(hours=48)
duration = args.duration
if duration is None:
duration = 48
for alert in api_client.get_alerts(start_time, duration):
print(json.dumps(alert))
sys.exit(0)
# are we downloading alert data?
if args.alert_id:
alert_json = None
if args.alert_json is not None:
target_path = _get_path(args.alert_json, args.output_dir)
def test_tuning_rules_submission_all_fields_match(self):
# sample observable layout
# [
# {
# "time": "2020-02-14T20:45:00.620518+0000",
# "type": "ipv4",
# "value": "1.2.3.4"
# },
# {
# "time": "2020-02-14T20:45:00.620565+0000",
# "type": "ipv4",
# "value": "1.2.3.5"
# }
# ]
# same as above but testing multiple rule matches
with open(os.path.join(self.tuning_rule_dir, 'test.yar'), 'w') as fp:
fp.write("""
rule test_description {
meta:
targets = "submission"
strings:
$ = "description = test_description"
condition:
all of them
}
rule test_analysis_mode {
meta:
targets = "submission"
strings:
$ = "analysis_mode = analysis"
condition:
all of them
}
rule test_tool {
meta:
targets = "submission"
strings:
$ = "tool = unittest_tool"
condition:
all of them
}
rule test_tool_instance {
meta:
targets = "submission"
strings:
$ = "tool_instance = unittest_tool_instance"
condition:
all of them
}
rule test_type {
meta:
targets = "submission"
strings:
$ = "type = unittest_type"
condition:
all of them
}
rule test_event_time {
meta:
targets = "submission"
strings:
$ = /\\nevent_time =/
condition:
all of them
}
rule test_tags {
meta:
targets = "submission"
strings:
$ = /\\ntags = .*tag_1.*\\n/
condition:
all of them
}
rule test_observable {
meta:
targets = "observable"
strings:
$ = /"type": "ipv4"/
$ = /"value": "1.2.3.5"/
condition:
all of them
}
""")
submission_filter = self.create_submission_filter()
submission = _custom_submission()
submission.tags = ['tag_1', 'tag_2']
submission.observables = [
{
'type': F_IPV4,
'value': '1.2.3.4',
'time': local_time(),
},
{
'type': F_IPV4,
'value': '1.2.3.5',
'time': local_time(),
},
]
matches = submission_filter.get_tuning_matches(submission)
submission_filter.log_tuning_matches(submission, matches)
# looks like there's a bug in the library that is returning multiple match results for the same match
#self.assertTrue(len(matches) == 7)
rule_names = [_['rule'] for _ in matches]
for rule_name in [
'test_description',
'test_analysis_mode',
'test_tool',
'test_tool_instance',
'test_type',
'test_event_time',
'test_tags',
'test_observable',
]:
self.assertTrue(rule_name in rule_names)
def _execute(self, *args, **kwargs):
if not self.password:
logging.error(f"no password given for {self.section}. authentication will not be attempted.")
return
if not self.delete_emails:
if not os.path.exists(self.tracking_db_path):
with sqlite3.connect(self.tracking_db_path) as db:
c = db.cursor()
c.execute("""
CREATE TABLE IF NOT EXISTS ews_tracking (
exchange_id TEXT NOT NULL,
message_id TEXT NOT NULL,
insert_date INT NOT NULL )""")
c.execute("""
CREATE INDEX IF NOT EXISTS idx_exchange_id ON ews_tracking(exchange_id)""")
c.execute("""
CREATE INDEX IF NOT EXISTS idx_insert_date ON ews_tracking(insert_date)""")
db.commit()
# get the next emails from this account
credentials = Credentials(self.username, self.password)
config = Configuration(server=self.server, credentials=credentials, auth_type=NTLM) # TODO auth_type should be configurable
_account_class = kwargs.get('account_class') or Account # Account class connects to exchange.
account = _account_class(self.target_mailbox, config=config, autodiscover=False, access_type=DELEGATE) # TODO autodiscover, access_type should be configurable
for folder in self.folders:
path_parts = [_.strip() for _ in folder.split('/')]
root = path_parts.pop(0)
_account = kwargs.get('account_object') or account
try:
target_folder = getattr(_account, root)
except AttributeError:
public_folders_root = _account.public_folders_root
target_folder = public_folders_root / root
#print(target_folder.tree())
for path_part in path_parts:
target_folder = target_folder / path_part
target_folder.refresh()
logging.info(f"checking for emails in {self.target_mailbox} target {folder}")
total_count = 0
already_processed_count = 0
error_count = 0
for message in target_folder.all().order_by('-datetime_received'):
if isinstance(message, ResponseMessageError):
logging.warning(f"error when iterating mailbox {self.target_mailbox} folder {folder}: {message} ({type(message)})")
continue
# XXX not sure why this is happening?
if message.id is None:
continue
total_count += 1
try:
# if we're not deleting emails then we need to make sure we keep track of which ones we've already processed
if not self.delete_emails:
with sqlite3.connect(self.tracking_db_path) as db:
c = db.cursor()
c.execute("SELECT message_id FROM ews_tracking WHERE exchange_id = ?", (message.id,))
result = c.fetchone()
if result is not None:
#logging.debug("already processed exchange message {} message id {} from {}@{}".format(
#message.id, message.message_id, self.target_mailbox, self.server))
already_processed_count += 1
continue
# otherwise process the email message (subclasses deal with the site logic)
self.email_received(message)
except Exception as e:
logging.error(f"unable to process email: {e}")
report_exception()
error_count += 1
if self.delete_emails:
try:
logging.debug(f"deleting message {message.id}")
message.delete()
except Exception as e:
logging.error(f"unable to delete message: {e}")
else:
# if we're not deleting the emails then we track which ones we've already processed
with sqlite3.connect(self.tracking_db_path) as db:
c = db.cursor()
c.execute("""
INSERT INTO ews_tracking (
exchange_id,
message_id,
insert_date ) VALUES ( ?, ?, ? )""",
(message.id, message.message_id, local_time().timestamp()))
# TODO delete anything older than X days
db.commit()
logging.info(f"finished checking for emails in {self.target_mailbox} target {folder}"
f" total {total_count} already_processed {already_processed_count} error {error_count}")
