def test_load_hunts(self):
hunter = HuntManager(**manager_kwargs())
hunter.load_hunts_from_config()
self.assertEquals(len(hunter.hunts), 2)
self.assertTrue(isinstance(hunter.hunts[0], TestHunt))
self.assertTrue(isinstance(hunter.hunts[1], TestHunt))
for hunt in hunter.hunts:
hunt.last_executed_time = datetime.datetime.now()
self.assertTrue(hunter.hunts[1].enabled)
self.assertEquals(hunter.hunts[1].name, 'unit_test_1')
self.assertEquals(hunter.hunts[1].description,
'Unit Test Description 1')
self.assertEquals(hunter.hunts[1].type, 'test')
self.assertTrue(
isinstance(hunter.hunts[1].frequency, datetime.timedelta))
self.assertEquals(hunter.hunts[1].tags, ['tag1', 'tag2'])
self.assertTrue(hunter.hunts[0].enabled)
self.assertEquals(hunter.hunts[0].name, 'unit_test_2')
self.assertEquals(hunter.hunts[0].description,
'Unit Test Description 2')
self.assertEquals(hunter.hunts[0].type, 'test')
self.assertTrue(
isinstance(hunter.hunts[0].frequency, datetime.timedelta))
self.assertEquals(hunter.hunts[0].tags, ['tag1', 'tag2'])
def test_load_hunt_ini(self):
manager = HuntManager(**manager_kwargs())
manager.load_hunts_from_config()
self.assertEquals(len(manager.hunts), 1)
hunt = manager.hunts[0]
self.assertTrue(hunt.enabled)
self.assertEquals(hunt.name, 'query_test_1')
self.assertEquals(hunt.description, 'Query Test Description 1')
self.assertEquals(hunt.type, 'test_query')
self.assertEquals(hunt.frequency, create_timedelta('00:01:00'))
self.assertEquals(hunt.tags, ['tag1', 'tag2'])
self.assertEquals(hunt.time_range, create_timedelta('00:01:00'))
self.assertEquals(hunt.max_time_range, create_timedelta('01:00:00'))
self.assertEquals(hunt.offset, create_timedelta('00:05:00'))
self.assertTrue(hunt.full_coverage)
self.assertEquals(hunt.group_by, 'field1')
self.assertEquals(hunt.query, 'Test query.')
self.assertTrue(hunt.use_index_time)
self.assertEquals(hunt.observable_mapping, {
'src_ip': 'ipv4',
'dst_ip': 'ipv4'
})
self.assertEquals(hunt.temporal_fields, {
'src_ip': True,
'dst_ip': True
})
def test_full_coverage(self):
manager = HuntManager(**manager_kwargs())
hunt = default_hunt(time_range=create_timedelta('01:00:00'),
frequency=create_timedelta('01:00:00'))
manager.add_hunt(hunt)
# first test that the start time and end time are correct for normal operation
# for first-time hunt execution
self.assertTrue(hunt.ready)
# now put the last time we executed to 5 minutes ago
# ready should return False
hunt.last_executed_time = local_time() - datetime.timedelta(minutes=5)
self.assertFalse(hunt.ready)
# now put the last time we executed to 65 minutes ago
# ready should return True
hunt.last_executed_time = local_time() - datetime.timedelta(minutes=65)
self.assertTrue(hunt.ready)
# set the last time we executed to 3 hours ago
hunt.last_executed_time = local_time() - datetime.timedelta(hours=3)
# and the last end date to 2 hours ago
hunt.last_end_time = local_time() - datetime.timedelta(hours=2)
# so now we have 2 hours to cover under full coverage
# ready should return True, start should be 3 hours ago and end should be 2 hours ago
self.assertTrue(hunt.ready)
self.assertEquals(hunt.start_time, hunt.last_end_time)
self.assertEquals(hunt.end_time, hunt.last_end_time + hunt.time_range)
#logging.info(f"MARKER: start {hunt.start_time} end {hunt.end_time} comp {hunt.end_time - hunt.start_time} >= {hunt.time_range}")
# now let's pretend that we just executed that
# at this point, the last_end_time becomes the end_time
hunt.last_end_time = hunt.end_time
# and the last_executed_time becomes now
hunt.last_executed_time = local_time()
# at this point the hunt should still be ready because we're not caught up yet
self.assertTrue(hunt.ready)
# now give the hunt the ability to cover 2 hours instead of 1 to get caught up
hunt.max_time_range = create_timedelta('02:00:00')
# set the last time we executed to 3 hours ago
hunt.last_executed_time = local_time() - datetime.timedelta(hours=3)
# and the last end date to 2 hours ago
hunt.last_end_time = local_time() - datetime.timedelta(hours=2)
# now the difference between the stop and stop should be 2 hours instead of one
#logging.info(f"MARKER: start {hunt.start_time} end {hunt.end_time} comp {hunt.end_time - hunt.start_time} >= {hunt.time_range}")
self.assertTrue(hunt.end_time - hunt.start_time >= hunt.max_time_range)
# set the last time we executed to 3 hours ago
hunt.last_executed_time = local_time() - datetime.timedelta(hours=3)
# and the last end date to 2 hours ago
hunt.last_end_time = local_time() - datetime.timedelta(hours=2)
# so now we have 2 hours to cover but let's turn off full coverage
hunt.full_coverage = False
# it should be ready to run
self.assertTrue(hunt.ready)
def test_load_hunt_with_includes(self):
ips_txt = 'hunts/test/splunk/ips.txt'
with open(ips_txt, 'w') as fp:
fp.write('1.1.1.1\n')
manager = HuntManager(**manager_kwargs())
manager.load_hunts_from_config(hunt_filter=lambda hunt: hunt.name == 'query_test_includes')
hunt = manager.get_hunt_by_name('query_test_includes')
self.assertIsNotNone(hunt)
# same as above except that ip address comes from a different file
self.assertEquals(hunt.query, 'index=proxy {time_spec} src_ip=1.1.1.1\n')
# and then change it and it should have a different value
with open(ips_txt, 'a') as fp:
fp.write('1.1.1.2\n')
self.assertEquals(hunt.query, 'index=proxy {time_spec} src_ip=1.1.1.1\n1.1.1.2\n')
os.remove(ips_txt)
def test_offset(self):
manager = HuntManager(**manager_kwargs())
hunt = default_hunt(time_range=create_timedelta('01:00:00'),
frequency=create_timedelta('01:00:00'),
offset=create_timedelta('00:30:00'))
manager.add_hunt(hunt)
# set the last time we executed to 3 hours ago
hunt.last_executed_time = local_time() - datetime.timedelta(hours=3)
# and the last end date to 2 hours ago
target_start_time = hunt.last_end_time = local_time(
) - datetime.timedelta(hours=2)
self.assertTrue(hunt.ready)
hunt.execute()
# the times passed to hunt.execute_query should be 30 minutes offset
self.assertEquals(target_start_time - hunt.offset,
hunt.exec_start_time)
self.assertEquals(hunt.last_end_time - hunt.offset, hunt.exec_end_time)
def test_hunt_persistence(self):
hunter = HuntManager(**manager_kwargs())
hunter.add_hunt(default_hunt())
hunter.hunts[0].last_executed_time = datetime.datetime(
2019, 12, 10, 8, 21, 13)
with open_hunt_db(hunter.hunts[0].type) as db:
c = db.cursor()
c.execute(
"""SELECT last_executed_time FROM hunt WHERE hunt_name = ?""",
(hunter.hunts[0].name, ))
row = c.fetchone()
self.assertIsNotNone(row)
last_executed_time = row[0]
self.assertTrue(isinstance(last_executed_time, datetime.datetime))
self.assertEquals(last_executed_time.year, 2019)
self.assertEquals(last_executed_time.month, 12)
self.assertEquals(last_executed_time.day, 10)
self.assertEquals(last_executed_time.hour, 8)
self.assertEquals(last_executed_time.minute, 21)
self.assertEquals(last_executed_time.second, 13)
def test_load_hunt_ini(self):
manager = HuntManager(**manager_kwargs())
manager.load_hunts_from_config(hunt_filter=lambda hunt: hunt.name == 'query_test_1')
self.assertEquals(len(manager.hunts), 1)
hunt = manager.get_hunt_by_name('query_test_1')
self.assertIsNotNone(hunt)
self.assertTrue(hunt.enabled)
self.assertEquals(hunt.name, 'query_test_1')
self.assertEquals(hunt.description, 'Query Test Description 1')
self.assertEquals(hunt.type, 'splunk')
self.assertEquals(hunt.frequency, create_timedelta('00:01:00'))
self.assertEquals(hunt.tags, ['tag1', 'tag2'])
self.assertEquals(hunt.time_range, create_timedelta('00:01:00'))
self.assertEquals(hunt.max_time_range, create_timedelta('01:00:00'))
self.assertEquals(hunt.offset, create_timedelta('00:05:00'))
self.assertTrue(hunt.full_coverage)
self.assertEquals(hunt.group_by, 'field1')
self.assertEquals(hunt.query, 'index=proxy {time_spec} src_ip=1.1.1.1\n')
self.assertTrue(hunt.use_index_time)
self.assertEquals(hunt.observable_mapping, { 'src_ip': 'ipv4', 'dst_ip': 'ipv4' })
self.assertEquals(hunt.temporal_fields, { 'src_ip': True, 'dst_ip': True })
def test_hunt_order(self):
hunter = HuntManager(**manager_kwargs())
# test initial hunt order
# these are added in the wrong order but the should be sorted when we access them
hunter.add_hunt(
default_hunt(name='test_hunt_3',
frequency=create_timedelta('00:30')))
hunter.add_hunt(
default_hunt(name='test_hunt_2',
frequency=create_timedelta('00:20')))
hunter.add_hunt(
default_hunt(name='test_hunt_1',
frequency=create_timedelta('00:10')))
# assume we've executed all of these hunts
for hunt in hunter.hunts:
hunt.last_executed_time = datetime.datetime.now()
# now they should be in this order
self.assertEquals(hunter.hunts[0].name, 'test_hunt_1')
self.assertEquals(hunter.hunts[1].name, 'test_hunt_2')
self.assertEquals(hunter.hunts[2].name, 'test_hunt_3')
def test_splunk_query(self):
manager = HuntManager(**manager_kwargs())
manager.load_hunts_from_config(hunt_filter=lambda hunt: hunt.name == 'test_query')
self.assertEquals(len(manager.hunts), 1)
hunt = manager.get_hunt_by_name('test_query')
self.assertIsNotNone(hunt)
with open('test_data/hunts/splunk/test_output.json', 'r') as fp:
query_results = json.load(fp)
result = hunt.execute(unit_test_query_results=query_results)
self.assertTrue(isinstance(result, list))
self.assertEquals(len(result), 4)
for submission in result:
with self.subTest(description=submission.description):
self.assertEquals(submission.analysis_mode, ANALYSIS_MODE_CORRELATION)
self.assertTrue(isinstance(submission.details, list))
self.assertTrue(all([isinstance(_, dict) for _ in submission.details]))
self.assertEquals(submission.files, [])
self.assertEquals(submission.tags, ['tag1', 'tag2'])
self.assertEquals(submission.tool, 'hunter-splunk')
self.assertEquals(submission.tool_instance, saq.CONFIG['splunk']['uri'])
self.assertEquals(submission.type, 'splunk')
if submission.description == 'Test Splunk Query: 29380 (3 events)':
self.assertEquals(submission.event_time, datetime.datetime(2019, 12, 23, 16, 5, 36))
self.assertEquals(submission.observables, [ {'type': 'file_name', 'value': '__init__.py'} ])
elif submission.description == 'Test Splunk Query: 29385 (2 events)':
self.assertEquals(submission.event_time, datetime.datetime(2019, 12, 23, 16, 5, 37))
self.assertEquals(submission.observables, [ {'type': 'file_name', 'value': '__init__.py'} ])
elif submission.description == 'Test Splunk Query: 29375 (2 events)':
self.assertEquals(submission.event_time, datetime.datetime(2019, 12, 23, 16, 5, 36))
self.assertEquals(submission.observables, [ {'type': 'file_name', 'value': '__init__.py'} ])
elif submission.description == 'Test Splunk Query: 31185 (93 events)':
self.assertEquals(submission.event_time, datetime.datetime(2019, 12, 23, 16, 5, 22))
self.assertEquals(submission.observables, [ {'type': 'file_name', 'value': '__init__.py'} ])
else:
self.fail("invalid description")
def test_remove_hunt(self):
hunter = HuntManager(**manager_kwargs())
hunt = hunter.add_hunt(default_hunt())
removed = hunter.remove_hunt(hunt)
self.assertEquals(hunt.name, removed.name)
self.assertEquals(len(hunter.hunts), 0)
def test_add_duplicate_hunt(self):
# should not be allowed to add a hunt that already exists
hunter = HuntManager(**manager_kwargs())
hunter.add_hunt(default_hunt())
with self.assertRaises(KeyError):
hunter.add_hunt(default_hunt())
def test_add_hunt(self):
hunter = HuntManager(**manager_kwargs())
hunter.add_hunt(default_hunt())
self.assertEquals(len(hunter.hunts), 1)