def __init__(self, cfg, sentryClient=None):
Destiny.__init__(self,
cfg,
lgrName=__name__,
sentryClient=sentryClient)
# create revalation instance
self.alertNode = Revelation(cfg, sentryClient=sentryClient)
self.initTrafficNodeAnalysisCalculations()
def setUp(self):
# generate config
cfg = configparser.ConfigParser()
cfg.read('build/tests/unit_tests_GOOD.cfg')
self.destiny = Destiny(cfg=cfg, lgrName=__name__)
# generate test log entries
logData = {'field1': 'value', 'field2': 1}
baselineLogData = { 'field1': 'value', 'field2': 1, 'threat': 0 }
intelLogData = { 'field1': 'value', 'field2': 1, 'threat': 1 }
self.destiny.logDbHandle.hmset('log:non_existent_parser:1234', logData)
self.destiny.logDbHandle.hmset('baseline:log:non_existent_parser:1234', baselineLogData)
self.destiny.logDbHandle.hmset('intel:non_existent_intel:1234', intelLogData)
def test_initializeLogData_noLogData(self):
logSet = self.destiny.fetchLogData('baseline')
uniqueFields = Destiny.getUniqueLogDataFields(logSet)
try:
encTrainingData, targetData = self.destiny.initializeLogData(logData={}, uniqueFields=uniqueFields,
dataUsage='training', targetFieldName='threat')
except ValueError:
self.assertTrue(True)
def test_initializeLogData_sendTrainingDataWithoutTFN(self):
logSet = self.destiny.fetchLogData('baseline')
uniqueFields = Destiny.getUniqueLogDataFields(logSet)
try:
encTrainingData, targetData = self.destiny.initializeLogData(logData=logSet, uniqueFields=uniqueFields,
dataUsage='training')
except RuntimeError:
self.assertTrue(True)
def test_initializeLogData(self):
logSet = self.destiny.fetchLogData('baseline')
uniqueFields = Destiny.getUniqueLogDataFields(logSet)
encTrainingData, targetData = self.destiny.initializeLogData(logData=logSet, uniqueFields=uniqueFields,
dataUsage='training', targetFieldName='threat')
logSet = self.destiny.fetchLogData('raw')
encTestingData = self.destiny.initializeLogData(logData=logSet, uniqueFields=uniqueFields, dataUsage='testing')
self.assertIsNotNone(encTrainingData)
self.assertGreater(len(targetData), 0)
self.assertIsNotNone(encTestingData)
def test_getUniqueLogDataFields(self):
logSet = self.destiny.fetchLogData()
uniqueFields = Destiny.getUniqueLogDataFields(logSet)
self.assertGreater(len(uniqueFields), 0)
class DestinyTestCase(unittest.TestCase):
def setUp(self):
# generate config
cfg = configparser.ConfigParser()
cfg.read('build/tests/unit_tests_GOOD.cfg')
self.destiny = Destiny(cfg=cfg, lgrName=__name__)
# generate test log entries
logData = {'field1': 'value', 'field2': 1}
baselineLogData = { 'field1': 'value', 'field2': 1, 'threat': 0 }
intelLogData = { 'field1': 'value', 'field2': 1, 'threat': 1 }
self.destiny.logDbHandle.hmset('log:non_existent_parser:1234', logData)
self.destiny.logDbHandle.hmset('baseline:log:non_existent_parser:1234', baselineLogData)
self.destiny.logDbHandle.hmset('intel:non_existent_intel:1234', intelLogData)
def test_fetchLogData_default(self):
logSet = self.destiny.fetchLogData()
self.assertGreater(len(logSet), 0)
def test_fetchLogData_raw(self):
logSet = self.destiny.fetchLogData(logType='raw')
self.assertGreater(len(logSet), 0)
def test_fetchLogData_baseline(self):
logSet = self.destiny.fetchLogData(logType='baseline')
self.assertGreater(len(logSet), 0)
def test_fetchLogData_intel(self):
logSet = self.destiny.fetchLogData(logType='intel')
self.assertGreater(len(logSet), 0)
def test_fetchLogData_invalidLogType(self):
try:
logSet = self.destiny.fetchLogData(logType='invalid')
except ValueError:
self.assertTrue(True)
def test_getUniqueLogDataFields(self):
logSet = self.destiny.fetchLogData()
uniqueFields = Destiny.getUniqueLogDataFields(logSet)
self.assertGreater(len(uniqueFields), 0)
def test_initializeLogData(self):
logSet = self.destiny.fetchLogData('baseline')
uniqueFields = Destiny.getUniqueLogDataFields(logSet)
encTrainingData, targetData = self.destiny.initializeLogData(logData=logSet, uniqueFields=uniqueFields,
dataUsage='training', targetFieldName='threat')
logSet = self.destiny.fetchLogData('raw')
encTestingData = self.destiny.initializeLogData(logData=logSet, uniqueFields=uniqueFields, dataUsage='testing')
self.assertIsNotNone(encTrainingData)
self.assertGreater(len(targetData), 0)
self.assertIsNotNone(encTestingData)
def test_initializeLogData_noLogData(self):
logSet = self.destiny.fetchLogData('baseline')
uniqueFields = Destiny.getUniqueLogDataFields(logSet)
try:
encTrainingData, targetData = self.destiny.initializeLogData(logData={}, uniqueFields=uniqueFields,
dataUsage='training', targetFieldName='threat')
except ValueError:
self.assertTrue(True)
def test_initializeLogData_noUniqueFields(self):
logSet = self.destiny.fetchLogData('baseline')
try:
encTrainingData, targetData = self.destiny.initializeLogData(logData=logSet, uniqueFields=[],
dataUsage='training', targetFieldName='threat')
except ValueError:
self.assertTrue(True)
def test_initializeLogData_invalidDataUsageOption(self):
logSet = self.destiny.fetchLogData('baseline')
uniqueFields = Destiny.getUniqueLogDataFields(logSet)
try:
encTrainingData, targetData = self.destiny.initializeLogData(logData=logSet, uniqueFields=uniqueFields,
dataUsage='invalidOption',
targetFieldName='threat')
except ValueError:
self.assertTrue(True)
def test_initializeLogData_invalidTargetFieldName(self):
logSet = self.destiny.fetchLogData('baseline')
uniqueFields = Destiny.getUniqueLogDataFields(logSet)
try:
encTrainingData, targetData = self.destiny.initializeLogData(logData=logSet, uniqueFields=uniqueFields,
dataUsage='training', targetFieldName='invalidTFN')
except ValueError:
self.assertTrue(True)
def test_initializeLogData_sendTrainingDataWithoutTFN(self):
logSet = self.destiny.fetchLogData('baseline')
uniqueFields = Destiny.getUniqueLogDataFields(logSet)
try:
encTrainingData, targetData = self.destiny.initializeLogData(logData=logSet, uniqueFields=uniqueFields,
dataUsage='training')
except RuntimeError:
self.assertTrue(True)
def tearDown(self):
# remove test log entries
self.destiny.logDbHandle.delete('log:non_existent_parser:1234')
self.destiny.logDbHandle.delete('baseline:log:non_existent_parser:1234')
self.destiny.logDbHandle.delete('intel:non_existent_intel:1234')
def startNetworkThreatEngine(self):
"""
Run network threat analysis engine and all needed components
:return: void
"""
# check if this is a test run
testRun = self.getCfgValue(section='cli', name='test_run', defaultVal=False, dataType=bool)
newTrainerPID = 0
if not testRun:
# fork process before beginning analysis
self.lgr.debug('forking off engine to child process')
newTrainerPID = fork()
if newTrainerPID == 0 or testRun:
# in child process, bounce inquisition DB handle (see issue #66)
try:
self.bounceInquisitionDbConnection()
except OperationalError as e:
self.lgr.critical('could not create database connection :: [ ' + str(e) + ' ]')
if self.sentryClient:
self.sentryClient.captureException()
exit(1)
# create model
self.lgr.debug('initializing classifier')
self.initClassifier()
# train and predict model after every $sleepTime seconds
sleepTime = self.getCfgValue('learning', 'networkThreatDetectionSleepTime', defaultVal=30, dataType=int)
while True:
# fetch all needed data
self.gatherAllData()
# model created and data is fetched - let's try to train it
if not self.intelLogStore:
self.lgr.info('no threat intel data available; not able to start training')
elif not self.baselineLogStore:
self.lgr.info('no baseline log data available; not able to start training for threat detection model')
else:
# initialize intel and baseline data
self.lgr.debug('initializing intel data')
targetFieldName = 'threat'
# combine intel and baseline data
rawTrainingDataset = {}
rawTrainingDataset.update(self.baselineLogStore)
rawTrainingDataset.update(self.intelLogStore)
# get list of unique fields in both training and testing log sets
self.lgr.debug('calculating unique field list for training and testing data initialization')
uniqueFieldsForTraining = Destiny.getUniqueLogDataFields(rawTrainingDataset)
uniqueFieldsForTesting = Destiny.getUniqueLogDataFields(self.logStore)
uniqueFields = list(set(uniqueFieldsForTraining + uniqueFieldsForTesting))
trainingData, trainingTargets = self.initializeLogData(rawTrainingDataset, uniqueFields, 'training',
targetFieldName)
# run model evaluation and print results
self.lgr.info('training threat detection model')
if self.networkThreatClassifier.fit(trainingData, trainingTargets):
self.lgr.info('training complete; starting network threat analysis against current log data')
# initialize raw logs
if self.logStore:
self.lgr.debug('initializing log data')
testingData = self.initializeLogData(self.logStore, uniqueFields, 'testing')
if testingData == None:
self.lgr.info('no data after initialization for threat detection - sleeping...')
else:
self.lgr.info('making predictions for testing data')
predictionResults = self.networkThreatClassifier.predict(testingData)
self.lgr.info('threat detection results :: { ' + str(predictionResults) + ' }')
self.lgr.debug('processing threat detection results')
self.processTestingResults(results=predictionResults)
else:
self.lgr.info('no raw log available for threat detection - sleeping...')
else:
self.lgr.warn('could not train network threat model; threat detection not performed')
# check if running a test run
if testRun:
self.lgr.debug('test run, exiting sage loop')
break
# sleep for determined time
self.lgr.debug('network threat engine is sleeping for [ ' + str(sleepTime)
+ ' ] seconds before restarting routines')
sleep(sleepTime)
def __init__(self, cfg, sentryClient=None):
Destiny.__init__(self, cfg, lgrName=__name__, sentryClient=sentryClient)
# create revalation instance
self.alertNode = Revelation(cfg, sentryClient=sentryClient)