def _fillupDataSource(self):
# Fillup database.
with io.open(MODSECURITY_AUDIT_LOG_SAMPLE_PATH, 'rt', errors = 'replace') as stream:
iterable = ModsecurityAuditLogParser().parseStream(stream)
dataSource = ModsecurityAuditDataSourceSQL(MODSECURITY_AUDIT_ENTRY_DATA_SOURCE_SQLITE_URL)
dataSource.insertModsecurityAuditEntryIterable(iterable)
def setUp(self):
cleanUp()
self._stream = io.open(MODSECURITY_AUDIT_LOG_SAMPLE_PATH, 'rt')
self._fillUpDataSource()
self._dataSource = ModsecurityAuditDataSourceSQL(MODSECURITY_AUDIT_ENTRY_DATA_SOURCE_SQLITE_URL)
self._itemDictIterableOriginal = self._dataSource.itemDictIterable(self._VARIABLE_NAME_LIST)
def _fillupDataSource(self):
# Fillup database.
with io.open(MODSECURITY_AUDIT_LOG_SAMPLE_PATH, 'rt',
errors='replace') as stream:
iterable = ModsecurityAuditLogParser().parseStream(stream)
dataSource = ModsecurityAuditDataSourceSQL(
MODSECURITY_AUDIT_ENTRY_DATA_SOURCE_SQLITE_URL)
dataSource.insertModsecurityAuditEntryIterable(iterable)
def testCorrelate(self):
# Fillup database.
dataSource = ModsecurityAuditDataSourceSQL(
MODSECURITY_AUDIT_ENTRY_DATA_SOURCE_SQLITE_URL)
# Making correlation engine.
correlationEngine = CorrelationEngine(self._VARIABLE_NAME_LIST)
# Testing progress listener.
progressListener = Mock(ICorrelationProgressListener)
correlationEngine.addProgressListener(progressListener)
# Correlating.
correlationList = map(lambda correlation: repr(correlation),
correlationEngine.correlate(dataSource))
self.assertEqual(self._EXPECTED_CORRELATION_LIST, correlationList)
self.assertEqual([
call.progress(217, 723),
call.progress(434, 723),
call.progress(651, 723),
call.progress(656, 723),
call.progress(674, 723),
call.progress(692, 723),
call.progress(710, 723),
call.progress(712, 723),
call.progress(714, 723),
call.progress(715, 723),
call.progress(717, 723),
call.progress(719, 723),
call.progress(721, 723),
call.progress(723, 723)
], progressListener.mock_calls)
def testCorrelationWithMaximumValueCount(self):
# Fillup database.
dataSource = ModsecurityAuditDataSourceSQL(
MODSECURITY_AUDIT_ENTRY_DATA_SOURCE_SQLITE_URL)
correlationEngine = CorrelationEngine(self._VARIABLE_NAME_LIST,
maximumValueCountThreshold=5)
correlationList = map(lambda correlation: repr(correlation),
correlationEngine.correlate(dataSource))
self.assertEqual(
self._EXPECTED_CORRELATION_LIST_WITH_MAXIMUM_VALUE_COUNT,
correlationList)
def testCorrelateWithIgnoredVariableDict(self):
# Fillup database.
dataSource = ModsecurityAuditDataSourceSQL(
MODSECURITY_AUDIT_ENTRY_DATA_SOURCE_SQLITE_URL)
ignoredVariableDict = {
'host_name': [u"1.1.1.1"],
'rule_id': [u"111111", u"981174"]
}
correlationEngine = CorrelationEngine(self._VARIABLE_NAME_LIST,
ignoredVariableDict)
correlationList = map(lambda correlation: repr(correlation),
correlationEngine.correlate(dataSource))
self.assertEqual(
self._EXPECTED_CORRELATION_LIST_WITH_IGNORED_VARIABLE_DICT,
correlationList)
class TestModsecurityAuditDataSourceSQL(unittest.TestCase):
_VARIABLE_NAME_LIST = ['host_name', 'request_file_name', 'payload_container', 'rule_id']
def setUp(self):
cleanUp()
self._stream = io.open(MODSECURITY_AUDIT_LOG_SAMPLE_PATH, 'rt')
self._fillUpDataSource()
self._dataSource = ModsecurityAuditDataSourceSQL(MODSECURITY_AUDIT_ENTRY_DATA_SOURCE_SQLITE_URL)
self._itemDictIterableOriginal = self._dataSource.itemDictIterable(self._VARIABLE_NAME_LIST)
def tearDown(self):
self._stream.close()
cleanUp()
def testInsertModsecurityAuditEntryIterable(self):
cursor = sqlite3.connect(MODSECURITY_AUDIT_ENTRY_DATA_SOURCE_SQLITE_FILE_PATH).cursor()
self.assertEqual(723, cursor.execute(u"SELECT count(*) FROM messages").fetchone()[0])
self.assertEqual((5, None, u'/agilefant/login.jsp', u'ARGS:a', u'111111'),
cursor.execute(u"SELECT * FROM messages LIMIT 4, 1").fetchone())
self.assertEqual((9,
u"test.domain.com",
u"/agilefant/login.jsp",
u"ARGS:a",
u"111111"),
cursor.execute(u"SELECT * FROM messages LIMIT 8, 1").fetchone())
def testModsecurityEntryMessageIterable(self):
self.assertEqual(723, len(self._itemDictIterableOriginal))
# Checking some items values.
itemDictList = list(self._itemDictIterableOriginal)
message = itemDictList[4]
self.assertEqual(None, message['host_name'])
self.assertEqual(u"/agilefant/login.jsp", message['request_file_name'])
self.assertEqual(u"ARGS:a", message['payload_container'])
self.assertEqual(u"111111", message['rule_id'])
message = itemDictList[68]
self.assertEqual(u"1.1.1.1", message['host_name'])
self.assertEqual(u"/agilefant/static/js/jquery.jstree.js", message['request_file_name'])
self.assertEqual(u"TX:anomaly_score", message['payload_container'])
self.assertEqual(u"981174", message['rule_id'])
message = itemDictList[100]
self.assertEqual(u"1.1.1.1", message['host_name'])
self.assertEqual(u"/agilefant/static/js/dynamics/controller/MenuController.js", message['request_file_name'])
self.assertEqual(u"REQUEST_HEADERS:Host", message['payload_container'])
self.assertEqual(u"960017", message['rule_id'])
def testModsecurityEntryMessageIterableDistinct(self):
itemDictDistinctIterable = self._itemDictIterableOriginal.distinct()
self.assertEqual(545, len(itemDictDistinctIterable))
self.assertEqual(723, len(self._itemDictIterableOriginal))
# Checking some item values.
itemDictDistinctList = list(itemDictDistinctIterable)
message = itemDictDistinctList[4]
self.assertEqual(None, message['host_name'])
self.assertEqual(u"/agilefant/login.jsp", message['request_file_name'])
self.assertEqual(u"ARGS:a", message['payload_container'])
self.assertEqual(u"111111", message['rule_id'])
message = itemDictDistinctList[68]
self.assertEqual(u"1.1.1.1", message['host_name'])
self.assertEqual(u"/agilefant/static/js/jquery.autoSuggest.minified.js", message['request_file_name'])
self.assertEqual(u"TX:anomaly_score", message['payload_container'])
self.assertEqual(u"981174", message['rule_id'])
message = itemDictDistinctList[100]
self.assertEqual(u"1.1.1.1", message['host_name'])
self.assertEqual(u"/agilefant/static/js/utils/ArrayUtils.js", message['request_file_name'])
self.assertEqual(u"REQUEST_HEADERS:Host", message['payload_container'])
self.assertEqual(u"960017", message['rule_id'])
def testMostFrequentAttribute(self):
self.assertEqual({'host_name': u'1.1.1.1'},
self._itemDictIterableOriginal.mostFrequentVariableAndValue(['host_name', 'request_file_name', 'payload_container', 'rule_id']))
self.assertEqual({'rule_id': u'960017'},
self._itemDictIterableOriginal.mostFrequentVariableAndValue(['rule_id']))
def testFilterByVariable(self):
# Filtering by request file name.
itemDictIterable = self._itemDictIterableOriginal.filterByVariable('host_name',
u"test.domain.com")
self.assertEqual(59, len(itemDictIterable))
def testFilterByVariableMultiple(self):
# Multiple filters.
itemDictIterable = self._itemDictIterableOriginal.filterByVariable('host_name',
u"test.domain.com")
itemDictIterable = itemDictIterable.filterByVariable('rule_id',
u"960017")
self.assertEqual(18, len(itemDictIterable))
def testFilterByVariableReverse(self):
# Reverse filtering by host name and rule id.
itemDictIterable = self._itemDictIterableOriginal.filterByVariable('host_name',
u"test.domain.com")
itemDictIterable = itemDictIterable.filterByVariable('request_file_name',
u"/agilefant/static/js/jquery.hotkeys.js",
negate = True)
self.assertEqual(56, len(itemDictIterable))
def testFilterByVariableReverseMultiple(self):
# Reverse filtering by host name and rule id.
itemDictIterable = self._itemDictIterableOriginal.filterByVariable('host_name',
u"test.domain.com")
itemDictIterable = itemDictIterable.filterByVariable('request_file_name',
u"/agilefant/static/js/jquery.hotkeys.js",
negate = True)
itemDictIterable = itemDictIterable.filterByVariable('request_file_name',
u"/agilefant/static/js/backlogSelector.js",
negate = True)
itemDictIterable = itemDictIterable.filterByVariable('rule_id',
u"981203",
negate = True)
self.assertEqual(36, len(itemDictIterable))
def testFilterByVariableNull(self):
itemDictIterable = self._itemDictIterableOriginal.filterByVariable('host_name', None)
self.assertEqual(8, len(itemDictIterable))
def testFilterByVariableReverseNull(self):
itemDictIterable = self._itemDictIterableOriginal.filterByVariable('host_name', None, negate = True)
self.assertEqual(715, len(itemDictIterable))
def testDistinctWithFilter(self):
# Testing that 'distinct' works with filters.
itemDictIterable = self._itemDictIterableOriginal.filterByVariable('host_name',
u"test.domain.com")
itemDictIterable = itemDictIterable.filterByVariable('request_file_name',
u"/agilefant/static/js/jquery.hotkeys.js",
negate = True)
itemDictIterable = itemDictIterable.filterByVariable('request_file_name',
u"/agilefant/static/js/backlogSelector.js",
negate = True)
itemDictIterable = itemDictIterable.filterByVariable('rule_id',
u"981203",
negate = True)
itemDictIterableDistinct = itemDictIterable.distinct()
self.assertEqual(32, len(itemDictIterableDistinct))
# Checking that the original iterable has not been modified.
self.assertEqual(36, len(itemDictIterable))
# Checking that other methods work with filters.
self.assertEqual({'host_name': u'test.domain.com'},
itemDictIterable.mostFrequentVariableAndValue(self._VARIABLE_NAME_LIST))
# Testing iterator.
self.assertEqual(36, len(list(itemDictIterable)))
def testFilterByVariableMany(self):
itemDictIterable = self._itemDictIterableOriginal
for i in range(1000):
itemDictIterable = itemDictIterable.filterByVariable('host_name',
unicode(i),
negate = True)
self.assertEqual(723, len(itemDictIterable))
def _fillUpDataSource(self):
iterable = ModsecurityAuditLogParser().parseStream(self._stream)
dataSource = ModsecurityAuditDataSourceSQL(MODSECURITY_AUDIT_ENTRY_DATA_SOURCE_SQLITE_URL)
dataSource.insertModsecurityAuditEntryIterable(iterable)
def _fillUpDataSource(self):
iterable = ModsecurityAuditLogParser().parseStream(self._stream)
dataSource = ModsecurityAuditDataSourceSQL(MODSECURITY_AUDIT_ENTRY_DATA_SOURCE_SQLITE_URL)
dataSource.insertModsecurityAuditEntryIterable(iterable)
