def __init__(self, cfg, sentryClient=None):
Destiny.__init__(self,
cfg,
lgrName=__name__,
sentryClient=sentryClient)
# create revalation instance
self.alertNode = Revelation(cfg, sentryClient=sentryClient)
self.initTrafficNodeAnalysisCalculations()
def __init__(self, cfg, sentryClient=None):
Destiny.__init__(self, cfg, lgrName=__name__, sentryClient=sentryClient)
# create revalation instance
self.alertNode = Revelation(cfg, sentryClient=sentryClient)
class Sage(Destiny):
"""
Sage engine for detecting network threats from known threat data
"""
intelLogStore = {}
baselineLogStore = {}
logStore = {}
networkThreatClassifier = None
alertNode = None
def __init__(self, cfg, sentryClient=None):
Destiny.__init__(self, cfg, lgrName=__name__, sentryClient=sentryClient)
# create revalation instance
self.alertNode = Revelation(cfg, sentryClient=sentryClient)
def initClassifier(self):
"""
Initialize classifier model for network threat detection
:return: None
"""
self.networkThreatClassifier = svm.SVC()
def gatherAllData(self):
"""
Fetches raw logs, baseline logs, and intel all at once
:return: void
"""
# fetch training data
self.lgr.info('fetching intel, baseline, and log data (in that order) for threat detection model')
self.lgr.debug('fetching training intel data')
self.intelLogStore = self.fetchLogData('intel')
self.lgr.debug('fetching training baseline data')
self.baselineLogStore = self.fetchLogData('baseline')
# refresh log data for testing
# we need to do this now in order to get the full list of unique fields
self.lgr.debug('fetching testing (raw log) data')
self.logStore = self.fetchLogData('raw')
def processTestingResults(self, results):
"""
Check results and generate alerts if needed
:param results: results from running testing data through network threat engine model
:return: void
"""
for idx, result in enumerate(results):
if result:
# get log at matching index
logData = list(self.logStore)[idx]
# model detected this as a threat, raise alert
self.alertNode.addAlert(timestamp=int(time()), alertType=3, alertDetails='Network threat detected!',
logData=logData)
def startNetworkThreatEngine(self):
"""
Run network threat analysis engine and all needed components
:return: void
"""
# check if this is a test run
testRun = self.getCfgValue(section='cli', name='test_run', defaultVal=False, dataType=bool)
newTrainerPID = 0
if not testRun:
# fork process before beginning analysis
self.lgr.debug('forking off engine to child process')
newTrainerPID = fork()
if newTrainerPID == 0 or testRun:
# in child process, bounce inquisition DB handle (see issue #66)
try:
self.bounceInquisitionDbConnection()
except OperationalError as e:
self.lgr.critical('could not create database connection :: [ ' + str(e) + ' ]')
if self.sentryClient:
self.sentryClient.captureException()
exit(1)
# create model
self.lgr.debug('initializing classifier')
self.initClassifier()
# train and predict model after every $sleepTime seconds
sleepTime = self.getCfgValue('learning', 'networkThreatDetectionSleepTime', defaultVal=30, dataType=int)
while True:
# fetch all needed data
self.gatherAllData()
# model created and data is fetched - let's try to train it
if not self.intelLogStore:
self.lgr.info('no threat intel data available; not able to start training')
elif not self.baselineLogStore:
self.lgr.info('no baseline log data available; not able to start training for threat detection model')
else:
# initialize intel and baseline data
self.lgr.debug('initializing intel data')
targetFieldName = 'threat'
# combine intel and baseline data
rawTrainingDataset = {}
rawTrainingDataset.update(self.baselineLogStore)
rawTrainingDataset.update(self.intelLogStore)
# get list of unique fields in both training and testing log sets
self.lgr.debug('calculating unique field list for training and testing data initialization')
uniqueFieldsForTraining = Destiny.getUniqueLogDataFields(rawTrainingDataset)
uniqueFieldsForTesting = Destiny.getUniqueLogDataFields(self.logStore)
uniqueFields = list(set(uniqueFieldsForTraining + uniqueFieldsForTesting))
trainingData, trainingTargets = self.initializeLogData(rawTrainingDataset, uniqueFields, 'training',
targetFieldName)
# run model evaluation and print results
self.lgr.info('training threat detection model')
if self.networkThreatClassifier.fit(trainingData, trainingTargets):
self.lgr.info('training complete; starting network threat analysis against current log data')
# initialize raw logs
if self.logStore:
self.lgr.debug('initializing log data')
testingData = self.initializeLogData(self.logStore, uniqueFields, 'testing')
if testingData == None:
self.lgr.info('no data after initialization for threat detection - sleeping...')
else:
self.lgr.info('making predictions for testing data')
predictionResults = self.networkThreatClassifier.predict(testingData)
self.lgr.info('threat detection results :: { ' + str(predictionResults) + ' }')
self.lgr.debug('processing threat detection results')
self.processTestingResults(results=predictionResults)
else:
self.lgr.info('no raw log available for threat detection - sleeping...')
else:
self.lgr.warn('could not train network threat model; threat detection not performed')
# check if running a test run
if testRun:
self.lgr.debug('test run, exiting sage loop')
break
# sleep for determined time
self.lgr.debug('network threat engine is sleeping for [ ' + str(sleepTime)
+ ' ] seconds before restarting routines')
sleep(sleepTime)
class Erudite(Destiny):
"""
An elegant and low-friction log anomaly detection engine
"""
hostStore = []
logStore = {}
nodeTypes = ('src', 'dst')
fieldTypeKeyToIDCache = {}
nodeCounts = {}
nodeOPSResults = {}
prevNodeOPSResults = {}
alertNode = None
runStartTime = 0
runEndTime = 0
def __init__(self, cfg, sentryClient=None):
Destiny.__init__(self,
cfg,
lgrName=__name__,
sentryClient=sentryClient)
# create revalation instance
self.alertNode = Revelation(cfg, sentryClient=sentryClient)
self.initTrafficNodeAnalysisCalculations()
def fetchKnownHostData(self):
"""
Fetches known host records from inquisition DB
:return: dict
"""
hosts = ()
# define sql
sql = """
SELECT
host_val
FROM
KnownHosts
"""
# execute query
with self.inquisitionDbHandle.cursor() as dbCursor:
dbCursor.execute(sql)
# fetch results
hosts = dbCursor.fetchall()
dbCursor.close()
return hosts
def getFieldNameFromType(self, fieldType='log_host'):
"""
Get name of field to use based on type ID
:param fieldType: type of field to fetch the name of
:return: str
"""
hostFieldName = ''
# define sql
sql = """
SELECT
F.field_name
FROM
Fields F
JOIN
FieldTypes FT
ON
(F.field_type=FT.type_id)
WHERE
FT.type_name = %s
LIMIT 1
"""
# execute query
with self.inquisitionDbHandle.cursor() as dbCursor:
dbCursor.execute(sql, fieldType)
# fetch results
dbResults = dbCursor.fetchone()
if dbResults:
try:
hostFieldName = dbResults['field_name']
except KeyError as e:
self.lgr.warn(
'results returned, but field name value not found (see Issue #65 on GitHub) :: [ MSG: '
+ str(e) + ' ]')
dbCursor.close()
return hostFieldName
def getFieldTypeIDFromTypeKey(self, typeKey='hst'):
"""
Get type ID of field type with given type key
:param typeKey: type key to find field type ID of
:return: int
"""
fieldTypeID = None
# define sql
sql = """
SELECT
type_id
FROM
FieldTypes
WHERE
type_key = %s
LIMIT 1
"""
# execute query
with self.inquisitionDbHandle.cursor() as dbCursor:
dbCursor.execute(sql, typeKey)
# fetch results
dbResults = dbCursor.fetchone()
if dbResults:
fieldTypeID = dbResults['type_id']
dbCursor.close()
return fieldTypeID
def identifyUnknownHosts(self, hostFieldName):
"""
Searches raw logs for hosts not in host store
:param hostFieldName: string to use as key/field name to identify host val
:return: list (unknown host data), list (host list)
"""
hostCache = []
unknownHosts = []
if not hostFieldName:
raise ValueError('no host field name provided')
# traverse through raw logs
if self.logStore:
# raw logs present, start traversal
self.lgr.debug('searching log store for any unknown hosts')
for logIdx in self.logStore:
try:
logHost = self.logStore[logIdx][hostFieldName]
# check if log host is already in host store OR if we've already added it to the unknown hosts list
if logHost not in self.hostStore and logHost not in hostCache:
# unknown host, add to unknown host list and host list cache
hostData = {'host': logHost, 'logIndex': logIdx}
unknownHosts.append(hostData)
hostCache.append(logHost)
except KeyError:
# log doesn't have anything set for host field, skip it
continue
return unknownHosts
def addUnknownHostData(self, hostData):
"""
Add host value(s) of unknown hosts to Inquisition DB
:param hostData: host-identification values (hostname/IP/other identifier)
:return: void
"""
if hostData:
# new unknown hosts sent, add them to db
sql = """
INSERT INTO
KnownHosts
(host_val)
VALUES
(%s)
"""
with self.inquisitionDbHandle.cursor() as dbCursor:
for hostObj in hostData:
try:
host = hostObj['host']
dbCursor.execute(sql, host)
self.inquisitionDbHandle.commit()
if self.getCfgValue(section='logging',
name='verbose',
defaultVal=False,
dataType=bool):
self.lgr.debug('synced new unknown host [ ' +
host + ' ] to inquisition DB')
except err as e:
self.inquisitionDbHandle.rollback()
self.lgr.critical(
'database error when inserting known host into Inquisition database while '
+ 'in baseline mode :: [ ' + str(e) + ' ]')
dbCursor.close()
self.lgr.debug(
'added [ ' + str(len(hostData)) +
' ] new hosts to known hosts table in Inquisition database'
)
def performUnknownHostAnalysis(self):
"""
Perform analysis that runs checks for unknown hosts that are generating logs
:return: void
"""
# clear current master host list
self.hostStore = []
# fetch fresh host list from inquisition DB
self.lgr.info('fetching list of already known hosts')
knownHostStore = self.fetchKnownHostData()
# traverse through list of host records to add raw host_val entries to master host list
for hostEntry in knownHostStore:
try:
self.hostStore.append(hostEntry['host_val'])
except KeyError as e:
self.lgr.warn(
'host entry found with no host value set (see Issue #47 on GitHub) :: [ MSG: '
+ str(e) + ' ]')
# read through log entries for new hosts
# get host field
hostField = self.getFieldNameFromType(fieldType='log_source')
if not hostField:
self.lgr.warn(
'no host field specified, not able to perform host-anomaly detection'
)
else:
self.lgr.debug('using [ ' + hostField +
' ] as host-identifying field name')
self.lgr.info('beginning unknown host identification')
unknownHosts = self.identifyUnknownHosts(hostField)
self.lgr.info('host identification complete: [ ' +
str(len(unknownHosts)) +
' ] unknown hosts identified')
# check if any unknown hosts were found
if 0 < len(unknownHosts):
# generate alerts if not in baseline mode
if not self.getCfgValue(section='learning',
name='enableBaselineMode',
defaultVal=False,
dataType=bool):
# not in baseline mode, generate alert(s)
for hostData in unknownHosts:
self.alertNode.addAlert(
timestamp=int(time()),
alertType=1,
host=hostData['host'],
alertDetails='Host anomaly detected!',
logData=self.logStore[hostData['logIndex']])
# add unknown hosts from host list to db
self.addUnknownHostData(unknownHosts)
self.lgr.debug(
'unknown hosts now identified as known and added to database'
)
def initTrafficNodeAnalysisCalculations(self):
"""
Resets all class variables for traffic analysis calculations
:return: void
"""
for nodeType in self.nodeTypes:
# initialize dict with node type as key for multiple node-related class vars
self.nodeCounts[nodeType] = {}
self.nodeOPSResults[nodeType] = {}
self.prevNodeOPSResults[nodeType] = {}
def calculateNodeOccurrenceCounts(self, nodeFieldName, nodeFieldType):
"""
Calculates the total number of occurrences we see each individual node of given type and name in the raw log store
:param nodeFieldName: key name to use for identifying field val in log record dicts
:param nodeFieldType: field value type to search for ('src' or 'dst')
:return: void
"""
nodeFieldType = nodeFieldType.lower()
if not nodeFieldName:
raise ValueError(
'node field name not specified for OCC calculations')
if nodeFieldType not in self.nodeTypes:
raise ValueError(
'invalid node field type specified for traffic node OCC calculation'
)
for logIdx in self.logStore:
# see if field name is present
try:
nodeFieldVal = self.logStore[logIdx][nodeFieldName]
# update log counts for val
if nodeFieldVal in self.nodeCounts[nodeFieldType]:
# previous entries already found, increase count
self.nodeCounts[nodeFieldType][nodeFieldVal]['count'] += 1
else:
# first occurrence found, initialize data set for node
self.nodeCounts[nodeFieldType][nodeFieldVal] = {'count': 1}
# add example log for node (to be used later for possible alerts)
# we always want the latest log, so this will always get set if a value for the nodeFiledName is set
self.nodeCounts[nodeFieldType][nodeFieldVal]['logIdx'] = logIdx
except KeyError:
# log doesn't have anything set for field name, continue on
pass
def calculateOPSForNode(self, occurrenceCount):
"""
Calculates occurrences per second for node with given OCC by comparing OCC with elapsed time
:param occurrenceCount: number of occurrences we saw node value in given timeframe
:return: float
"""
if occurrenceCount <= 0:
# no occurrences seen, we can just return 0 w/o doing anything else
return 0
timeValString = '[ START: { ' + str(
self.runStartTime) + ' } // END: { ' + str(
self.runEndTime) + ' } ]'
if self.runStartTime < 1 or self.runEndTime < 1:
raise ValueError('invalid start or end time set :: ' +
timeValString)
# calculate elapsed time in seconds and validate it
elapsedTime = self.runEndTime - self.runStartTime
if elapsedTime <= 0:
raise ValueError(
'end time is the same or before start time; could not calculate occurrences per second for'
+ ' node :: ' + timeValString)
# calculate occ/sec
occurrencesPerSec = float(occurrenceCount / elapsedTime)
return occurrencesPerSec
def calculateOPSResultsForNodeSet(self, nodeSetType):
"""
Calculate OPS for all nodes in given set type
:param nodeSetType: which set of nodes to perform calculations on (src|dst)
:return: void
"""
# validate node set type
nodeSetType = nodeSetType.lower()
if nodeSetType not in self.nodeTypes:
raise ValueError('invalid node set type provided')
for node in self.nodeCounts[nodeSetType]:
occPerSec = self.calculateOPSForNode(
self.nodeCounts[nodeSetType][node]['count'])
# add OPS to result set
self.nodeOPSResults[nodeSetType][node] = {
'logIdx': self.nodeCounts[nodeSetType][node]['logIdx'],
'ops': occPerSec
}
if self.getCfgValue(section='logging',
name='verbose',
defaultVal=False,
dataType=bool):
self.lgr.debug('calculated occurrences of node [ ' +
str(node) + ' // TYPE: ' + nodeSetType +
' ] to be [ ' + str(occPerSec) + ' / sec ]')
def fetchNodeOPSRecordInDB(self):
"""
Fetch all node OPS results that were previously collected from the inquisition DB
:return: void
"""
# define sql
sql = """
SELECT
type_key,
node_val,
occ_per_sec
FROM
TrafficNodeStats TNS
JOIN
FieldTypes FT
ON
(TNS.field_type_id=FT.type_id)
"""
# execute query
with self.inquisitionDbHandle.cursor() as dbCursor:
dbCursor.execute(sql)
# fetch results
dbResults = dbCursor.fetchall()
if dbResults:
self.lgr.debug('fetched [ ' + str(len(dbResults)) +
' ] previously-calculated OPS results from db')
for result in dbResults:
if result['type_key']:
self.prevNodeOPSResults[result['type_key']][
result['node_val']] = {
'ops': result['occ_per_sec']
}
if self.getCfgValue(section='logging',
name='verbose',
defaultVal=False,
dataType=bool):
self.lgr.debug(
'fetched past OPS result from db :: [ NODE: ' +
result['node_val'] + ' // TYPE: ' +
result['type_key'] + ' // OPS: ' +
str(result['occ_per_sec']))
dbCursor.close()
def determineOPSStdDevSignificance(self, node, nodeType,
prevNodeTrafficResult,
currentNodeTrafficResult):
"""
Calculates the standard deviation of prev and current node traffic results and determine if it's significant
:param node: node value that we're making calculations on
:param nodeType: type of node we're performing calc on
:param prevNodeTrafficResult: the previous OPS result calculated from the last run
:param currentNodeTrafficResult: the OPS results for node calculated from current run
:return: bool
"""
if prevNodeTrafficResult < 0 or currentNodeTrafficResult < 0:
raise ValueError(
'invalid previous or current node traffic results provided')
# check if std dev was set
try:
maxStdDev = self.getCfgValue('alerting',
'maxTrafficNodeStdDev',
defaultVal=1.0,
dataType=float)
except KeyError:
# config not set, use default
maxStdDev = 1.0
# calculate std deviation
standardDeviation = stdev(
[prevNodeTrafficResult, currentNodeTrafficResult])
if self.getCfgValue(section='logging',
name='verbose',
defaultVal=False,
dataType=bool):
self.lgr.debug('calculated std dev for [ NODE: ' + str(node) +
' // TYPE ID: ' + str(nodeType) + ' ] as [ ' +
str(standardDeviation) +
' ] w/ max std dev set to [ ' + str(maxStdDev) +
' ]')
if maxStdDev < standardDeviation:
return standardDeviation
return 0
def analyzeOPSResultsForAnomalies(self):
"""
Performs analysis on OPS results by comparing previous OPS results with the results from the latest run
:return: void
"""
# traverse current batch of nodes to compare with previous batch
for nodeType in self.nodeOPSResults:
for node in self.nodeOPSResults[nodeType]:
# check if node was found in prev results and is the same type (nodes can have diff src and dst results)
if node in self.prevNodeOPSResults[nodeType]:
# node was previously found, compare values by checking std deviation
prevOPSResult = self.prevNodeOPSResults[nodeType][node][
'ops']
currentOPSResult = self.nodeOPSResults[nodeType][node][
'ops']
stdDev = self.determineOPSStdDevSignificance(
node=node,
nodeType=nodeType,
prevNodeTrafficResult=prevOPSResult,
currentNodeTrafficResult=currentOPSResult)
if 0 < stdDev:
# raise anomaly alert
logData = self.logStore[self.nodeOPSResults[nodeType]
[node]['logIdx']]
# set default src and dst node, then set correct one to node val depending on node type
srcNode = '0.0.0.0'
dstNode = '0.0.0.0'
if nodeType == 'src':
srcNode = node
else:
dstNode = node
# set alert details
alertDetails = 'Detected significant change in OPS for traffic node :: [ NODE: ' + node \
+ ' // TYPE: ' + nodeType + ' // { ' + str(prevOPSResult) + ' -> ' \
+ str(currentOPSResult) + ' } // STD DEV: { ' + str(stdDev) + ' } ]'
# nodes and alert metadata set, add alert if required
if not self.getCfgValue(section='learning',
name='enableBaselineMode',
defaultVal=False,
dataType=bool):
self.alertNode.addAlert(timestamp=int(time()),
alertType=2,
srcNode=srcNode,
dstNode=dstNode,
alertDetails=alertDetails,
logData=logData)
def updateNodeOPSRecordInDB(self, nodeVal, nodeType):
"""
Update record with given node value and field type with OPS val
:param nodeVal: node value of record to update, e.g. IP or hostname
:param nodeType: type of field this node is
:return: bool
"""
verboseLogging = self.getCfgValue(section='logging',
name='verbose',
defaultVal=False,
dataType=bool)
status = False
if not nodeVal or not nodeType:
raise ValueError('missing node data :: [ NODE: ' + nodeVal +
' // FT: ' + nodeType + ' ]')
# get OPS results
try:
ops = self.nodeOPSResults[nodeType][nodeVal]['ops']
except KeyError:
# node OPS data is missing, nothing to insert into db
return status
# set sql
sql = """
INSERT INTO
TrafficNodeStats
(
node_val,
field_type_id,
occ_per_sec
)
VALUES
(
%s,
%s,
%s
)
ON DUPLICATE KEY UPDATE
occ_per_sec=VALUES(occ_per_sec)
"""
# check if we've already matched the key to a field type ID
if nodeType in self.fieldTypeKeyToIDCache:
fieldTypeID = self.fieldTypeKeyToIDCache[nodeType]['id']
else:
# convert field ID from given node type
fieldTypeID = self.getFieldTypeIDFromTypeKey(nodeType)
# add to mapping cache
if verboseLogging:
self.lgr.debug(
'adding field type ID to field type key to ID cache :: [ KEY: '
+ nodeType + ' // TYPE ID: ' + str(fieldTypeID) + ' ]')
self.fieldTypeKeyToIDCache[nodeType] = {'id': fieldTypeID}
if fieldTypeID:
# try inserting/updating in db
with self.inquisitionDbHandle.cursor() as dbCursor:
nodeString = '[ ' + str(nodeVal) + ' // TYPE ID: ' + str(
fieldTypeID) + ' ]'
try:
dbCursor.execute(sql, (nodeVal, fieldTypeID, ops))
self.inquisitionDbHandle.commit()
if verboseLogging:
self.lgr.debug(
'successfully synced node ' + nodeString +
' to traffic node tracking table in Inquisition database'
)
status = True
except err as e:
self.lgr.critical('database error when syncing data for ' +
nodeString + ' :: [ ' + str(e) + ' ]')
self.inquisitionDbHandle.rollback()
finally:
dbCursor.close()
else:
self.lgr.debug(
'field type ID not found for field type key :: [ KEY: ' +
nodeType + ' ]')
return status
def syncOPSResultsToDB(self):
"""
Traverse through set of OPS results for each node and update associated entry in db with value
:return:
"""
# check if OPS results are present
if self.nodeOPSResults:
# traverse results and update each in db
self.lgr.debug(
'beginning update of traffic node OPS results in DB')
for nodeType in self.nodeOPSResults:
for node in self.nodeOPSResults[nodeType]:
# send to db
if not self.updateNodeOPSRecordInDB(nodeVal=node,
nodeType=nodeType):
self.lgr.critical(
'unable to sync node OPS data to inquisition DB :: [ NODE: '
+ str(node) + ' // TYPE ID: ' + str(nodeType) +
' // OPS: ' +
str(self.nodeOPSResults[nodeType][node]['ops']) +
' ]')
numNodeOPSResults = 0
for nodeType in self.nodeTypes:
numNodeOPSResults += len(self.nodeOPSResults[nodeType])
self.lgr.info('synced traffic node stats for [ ' +
str(numNodeOPSResults) + ' ] nodes')
def performTrafficNodeAnalysis(self):
"""
Perform analysis on network traffic related nodes, e.g. source and destination IPs/hostnames
:return: void
"""
self.lgr.info('starting traffic node analysis')
# get field names associated with each field type
try:
srcNodeFieldName = self.getFieldNameFromType(
fieldType='traffic_source')
dstNodeFieldName = self.getFieldNameFromType(
fieldType='traffic_destination')
self.lgr.debug(
'using [ ' + str(srcNodeFieldName) +
' ] field as source traffic node field name and [ ' +
str(dstNodeFieldName) +
' ] field as destination traffic node field name')
except KeyError as e:
self.lgr.warning(
'problem while trying to identify field names, skipping traffic analysis :: [ '
+ str(e) + ' ]')
return
self.initTrafficNodeAnalysisCalculations()
# get counts for src and dst node fields in raw logs in log store
# check for raw logs
if self.logStore:
# raw logs present, initialize occurrence count calculations for all indiv. nodes for each node type
self.calculateNodeOccurrenceCounts(nodeFieldName=srcNodeFieldName,
nodeFieldType='src')
self.calculateNodeOccurrenceCounts(nodeFieldName=dstNodeFieldName,
nodeFieldType='dst')
# take previously calculated OCCs and calculate OPS of each node set type
for nodeType in self.nodeTypes:
self.lgr.info('calculating OPS for [ ' + nodeType +
' ] node set')
self.calculateOPSResultsForNodeSet(nodeType)
# read in OPS results from db that were calculated on previous runs
self.lgr.info(
'fetching previously-calculated OPS results from Inquisition database'
)
self.fetchNodeOPSRecordInDB()
# run anomaly analysis
self.lgr.info('performing anomaly analysis on OPS results')
self.analyzeOPSResultsForAnomalies()
# update database with OPS results
self.lgr.info('syncing new traffic node stats to inquisition DB')
self.syncOPSResultsToDB()
self.lgr.info('completed traffic node analysis')
else:
self.lgr.info(
'no raw logs available, skipping traffic node analysis')
def startAnomalyDetectionEngine(self):
"""
Starts anomaly detection engine and its components
:return: void
"""
self.runStartTime = time()
analysisHasRan = False
# check if this is a test run
testRun = self.getCfgValue(section='cli',
name='test_run',
defaultVal=False,
dataType=bool)
newAnalyzerPID = 0
if not testRun:
# fork process before beginning analysis
self.lgr.debug(
'forking off host and traffic anomaly detection engine to child process'
)
newAnalyzerPID = fork()
if newAnalyzerPID == 0 or testRun:
# in child process, bounce inquisition DB handle (see issue #66)
try:
self.bounceInquisitionDbConnection()
except OperationalError as e:
self.lgr.critical(
'could not create database connection :: [ ' + str(e) +
' ]')
if self.sentryClient:
self.sentryClient.captureException()
exit(1)
# run analysis after every $sleepTime seconds
sleepTime = self.getCfgValue(section='learning',
name='anomalyDetectionSleepTime',
defaultVal=15,
dataType=int)
while True:
# fetch raw logs
self.lgr.info('fetching raw logs for anomaly detection')
# get baseline logs if in baseline mode, raw if not
logType = 'raw'
if self.getCfgValue(section='learning',
name='enableBaselineMode',
defaultVal=False,
dataType=bool):
logType = 'baseline'
self.logStore = self.fetchLogData(logType=logType)
# set end time and cached start time right after logs are read in so no time is missed (we use this later)
self.runEndTime = time()
cachedStartTime = time()
if self.logStore:
# raw logs available
# start unknown host analysis
self.performUnknownHostAnalysis()
# start traffic node analysis
# only run analysis if we have logs and have ran an analysis before
# we get bad results on the first run, see Issue #54
if analysisHasRan:
self.performTrafficNodeAnalysis()
else:
self.lgr.debug(
'skipping traffic node analysis since this is the first run (see Issue #54)'
)
else:
self.lgr.info(
'no raw logs to perform host anomaly detection on - sleeping...'
)
# check if running a test run
if testRun:
self.lgr.debug('test run, exiting erudite loop')
break
# update orig start time with cached start time now that we're done with the orig time
self.runStartTime = cachedStartTime
analysisHasRan = True
# sleep for determined time
self.lgr.debug('anomaly detection engine is sleeping for [ ' +
str(sleepTime) +
' ] seconds before restarting routines')
sleep(sleepTime)
def setUp(self):
# generate config
cfg = configparser.ConfigParser()
cfg.read('build/tests/unit_tests_GOOD.cfg')
self.revelation = Revelation(cfg=cfg)
class RevelationTestCase(unittest.TestCase):
def setUp(self):
# generate config
cfg = configparser.ConfigParser()
cfg.read('build/tests/unit_tests_GOOD.cfg')
self.revelation = Revelation(cfg=cfg)
def test_addAlert_defaultParams(self):
self.revelation.addAlert(addAlertToDb=False)
self.assertGreater(len(self.revelation.alertStore), 0)
def test_addAlert_allParamsSet(self):
self.revelation.addAlert(timestamp=time(),
alertType=1,
status=1,
host='HOSTNAME1',
srcNode='1.1.1.1',
dstNode='2.2.2.2',
alertDetails='test details',
addAlertToDb=False)
self.assertGreater(len(self.revelation.alertStore), 0)
def test_addAlert_invalidTimestamp(self):
try:
self.revelation.addAlert(timestamp=-1, addAlertToDb=False)
except ValueError:
self.assertTrue(True)
try:
self.revelation.addAlert(timestamp=-1000000000, addAlertToDb=False)
except ValueError:
self.assertTrue(True)
def test_addAlert_invalidAlertType(self):
try:
self.revelation.addAlert(alertType=-1, addAlertToDb=False)
except ValueError:
self.assertTrue(True)
try:
self.revelation.addAlert(alertType=-1000000000, addAlertToDb=False)
except ValueError:
self.assertTrue(True)
def test_addAlert_invalidStatus(self):
try:
self.revelation.addAlert(status=-1, addAlertToDb=False)
except ValueError:
self.assertTrue(True)
try:
self.revelation.addAlert(status=-1000000000, addAlertToDb=False)
except ValueError:
self.assertTrue(True)
try:
self.revelation.addAlert(status=3, addAlertToDb=False)
except ValueError:
self.assertTrue(True)
try:
self.revelation.addAlert(status=30000000000000000,
addAlertToDb=False)
except ValueError:
self.assertTrue(True)