def _fillupDataSource(self): # Fillup database. with io.open(MODSECURITY_AUDIT_LOG_SAMPLE_PATH, 'rt', errors = 'replace') as stream: iterable = ModsecurityAuditLogParser().parseStream(stream) dataSource = ModsecurityAuditDataSourceSQL(MODSECURITY_AUDIT_ENTRY_DATA_SOURCE_SQLITE_URL) dataSource.insertModsecurityAuditEntryIterable(iterable)
def setUp(self): cleanUp() self._stream = io.open(MODSECURITY_AUDIT_LOG_SAMPLE_PATH, 'rt') self._fillUpDataSource() self._dataSource = ModsecurityAuditDataSourceSQL(MODSECURITY_AUDIT_ENTRY_DATA_SOURCE_SQLITE_URL) self._itemDictIterableOriginal = self._dataSource.itemDictIterable(self._VARIABLE_NAME_LIST)
def _fillupDataSource(self): # Fillup database. with io.open(MODSECURITY_AUDIT_LOG_SAMPLE_PATH, 'rt', errors='replace') as stream: iterable = ModsecurityAuditLogParser().parseStream(stream) dataSource = ModsecurityAuditDataSourceSQL( MODSECURITY_AUDIT_ENTRY_DATA_SOURCE_SQLITE_URL) dataSource.insertModsecurityAuditEntryIterable(iterable)
def testCorrelate(self): # Fillup database. dataSource = ModsecurityAuditDataSourceSQL( MODSECURITY_AUDIT_ENTRY_DATA_SOURCE_SQLITE_URL) # Making correlation engine. correlationEngine = CorrelationEngine(self._VARIABLE_NAME_LIST) # Testing progress listener. progressListener = Mock(ICorrelationProgressListener) correlationEngine.addProgressListener(progressListener) # Correlating. correlationList = map(lambda correlation: repr(correlation), correlationEngine.correlate(dataSource)) self.assertEqual(self._EXPECTED_CORRELATION_LIST, correlationList) self.assertEqual([ call.progress(217, 723), call.progress(434, 723), call.progress(651, 723), call.progress(656, 723), call.progress(674, 723), call.progress(692, 723), call.progress(710, 723), call.progress(712, 723), call.progress(714, 723), call.progress(715, 723), call.progress(717, 723), call.progress(719, 723), call.progress(721, 723), call.progress(723, 723) ], progressListener.mock_calls)
def testCorrelationWithMaximumValueCount(self): # Fillup database. dataSource = ModsecurityAuditDataSourceSQL( MODSECURITY_AUDIT_ENTRY_DATA_SOURCE_SQLITE_URL) correlationEngine = CorrelationEngine(self._VARIABLE_NAME_LIST, maximumValueCountThreshold=5) correlationList = map(lambda correlation: repr(correlation), correlationEngine.correlate(dataSource)) self.assertEqual( self._EXPECTED_CORRELATION_LIST_WITH_MAXIMUM_VALUE_COUNT, correlationList)
def testCorrelateWithIgnoredVariableDict(self): # Fillup database. dataSource = ModsecurityAuditDataSourceSQL( MODSECURITY_AUDIT_ENTRY_DATA_SOURCE_SQLITE_URL) ignoredVariableDict = { 'host_name': [u"1.1.1.1"], 'rule_id': [u"111111", u"981174"] } correlationEngine = CorrelationEngine(self._VARIABLE_NAME_LIST, ignoredVariableDict) correlationList = map(lambda correlation: repr(correlation), correlationEngine.correlate(dataSource)) self.assertEqual( self._EXPECTED_CORRELATION_LIST_WITH_IGNORED_VARIABLE_DICT, correlationList)
class TestModsecurityAuditDataSourceSQL(unittest.TestCase): _VARIABLE_NAME_LIST = ['host_name', 'request_file_name', 'payload_container', 'rule_id'] def setUp(self): cleanUp() self._stream = io.open(MODSECURITY_AUDIT_LOG_SAMPLE_PATH, 'rt') self._fillUpDataSource() self._dataSource = ModsecurityAuditDataSourceSQL(MODSECURITY_AUDIT_ENTRY_DATA_SOURCE_SQLITE_URL) self._itemDictIterableOriginal = self._dataSource.itemDictIterable(self._VARIABLE_NAME_LIST) def tearDown(self): self._stream.close() cleanUp() def testInsertModsecurityAuditEntryIterable(self): cursor = sqlite3.connect(MODSECURITY_AUDIT_ENTRY_DATA_SOURCE_SQLITE_FILE_PATH).cursor() self.assertEqual(723, cursor.execute(u"SELECT count(*) FROM messages").fetchone()[0]) self.assertEqual((5, None, u'/agilefant/login.jsp', u'ARGS:a', u'111111'), cursor.execute(u"SELECT * FROM messages LIMIT 4, 1").fetchone()) self.assertEqual((9, u"test.domain.com", u"/agilefant/login.jsp", u"ARGS:a", u"111111"), cursor.execute(u"SELECT * FROM messages LIMIT 8, 1").fetchone()) def testModsecurityEntryMessageIterable(self): self.assertEqual(723, len(self._itemDictIterableOriginal)) # Checking some items values. itemDictList = list(self._itemDictIterableOriginal) message = itemDictList[4] self.assertEqual(None, message['host_name']) self.assertEqual(u"/agilefant/login.jsp", message['request_file_name']) self.assertEqual(u"ARGS:a", message['payload_container']) self.assertEqual(u"111111", message['rule_id']) message = itemDictList[68] self.assertEqual(u"1.1.1.1", message['host_name']) self.assertEqual(u"/agilefant/static/js/jquery.jstree.js", message['request_file_name']) self.assertEqual(u"TX:anomaly_score", message['payload_container']) self.assertEqual(u"981174", message['rule_id']) message = itemDictList[100] self.assertEqual(u"1.1.1.1", message['host_name']) self.assertEqual(u"/agilefant/static/js/dynamics/controller/MenuController.js", message['request_file_name']) self.assertEqual(u"REQUEST_HEADERS:Host", message['payload_container']) self.assertEqual(u"960017", message['rule_id']) def testModsecurityEntryMessageIterableDistinct(self): itemDictDistinctIterable = self._itemDictIterableOriginal.distinct() self.assertEqual(545, len(itemDictDistinctIterable)) self.assertEqual(723, len(self._itemDictIterableOriginal)) # Checking some item values. itemDictDistinctList = list(itemDictDistinctIterable) message = itemDictDistinctList[4] self.assertEqual(None, message['host_name']) self.assertEqual(u"/agilefant/login.jsp", message['request_file_name']) self.assertEqual(u"ARGS:a", message['payload_container']) self.assertEqual(u"111111", message['rule_id']) message = itemDictDistinctList[68] self.assertEqual(u"1.1.1.1", message['host_name']) self.assertEqual(u"/agilefant/static/js/jquery.autoSuggest.minified.js", message['request_file_name']) self.assertEqual(u"TX:anomaly_score", message['payload_container']) self.assertEqual(u"981174", message['rule_id']) message = itemDictDistinctList[100] self.assertEqual(u"1.1.1.1", message['host_name']) self.assertEqual(u"/agilefant/static/js/utils/ArrayUtils.js", message['request_file_name']) self.assertEqual(u"REQUEST_HEADERS:Host", message['payload_container']) self.assertEqual(u"960017", message['rule_id']) def testMostFrequentAttribute(self): self.assertEqual({'host_name': u'1.1.1.1'}, self._itemDictIterableOriginal.mostFrequentVariableAndValue(['host_name', 'request_file_name', 'payload_container', 'rule_id'])) self.assertEqual({'rule_id': u'960017'}, self._itemDictIterableOriginal.mostFrequentVariableAndValue(['rule_id'])) def testFilterByVariable(self): # Filtering by request file name. itemDictIterable = self._itemDictIterableOriginal.filterByVariable('host_name', u"test.domain.com") self.assertEqual(59, len(itemDictIterable)) def testFilterByVariableMultiple(self): # Multiple filters. itemDictIterable = self._itemDictIterableOriginal.filterByVariable('host_name', u"test.domain.com") itemDictIterable = itemDictIterable.filterByVariable('rule_id', u"960017") self.assertEqual(18, len(itemDictIterable)) def testFilterByVariableReverse(self): # Reverse filtering by host name and rule id. itemDictIterable = self._itemDictIterableOriginal.filterByVariable('host_name', u"test.domain.com") itemDictIterable = itemDictIterable.filterByVariable('request_file_name', u"/agilefant/static/js/jquery.hotkeys.js", negate = True) self.assertEqual(56, len(itemDictIterable)) def testFilterByVariableReverseMultiple(self): # Reverse filtering by host name and rule id. itemDictIterable = self._itemDictIterableOriginal.filterByVariable('host_name', u"test.domain.com") itemDictIterable = itemDictIterable.filterByVariable('request_file_name', u"/agilefant/static/js/jquery.hotkeys.js", negate = True) itemDictIterable = itemDictIterable.filterByVariable('request_file_name', u"/agilefant/static/js/backlogSelector.js", negate = True) itemDictIterable = itemDictIterable.filterByVariable('rule_id', u"981203", negate = True) self.assertEqual(36, len(itemDictIterable)) def testFilterByVariableNull(self): itemDictIterable = self._itemDictIterableOriginal.filterByVariable('host_name', None) self.assertEqual(8, len(itemDictIterable)) def testFilterByVariableReverseNull(self): itemDictIterable = self._itemDictIterableOriginal.filterByVariable('host_name', None, negate = True) self.assertEqual(715, len(itemDictIterable)) def testDistinctWithFilter(self): # Testing that 'distinct' works with filters. itemDictIterable = self._itemDictIterableOriginal.filterByVariable('host_name', u"test.domain.com") itemDictIterable = itemDictIterable.filterByVariable('request_file_name', u"/agilefant/static/js/jquery.hotkeys.js", negate = True) itemDictIterable = itemDictIterable.filterByVariable('request_file_name', u"/agilefant/static/js/backlogSelector.js", negate = True) itemDictIterable = itemDictIterable.filterByVariable('rule_id', u"981203", negate = True) itemDictIterableDistinct = itemDictIterable.distinct() self.assertEqual(32, len(itemDictIterableDistinct)) # Checking that the original iterable has not been modified. self.assertEqual(36, len(itemDictIterable)) # Checking that other methods work with filters. self.assertEqual({'host_name': u'test.domain.com'}, itemDictIterable.mostFrequentVariableAndValue(self._VARIABLE_NAME_LIST)) # Testing iterator. self.assertEqual(36, len(list(itemDictIterable))) def testFilterByVariableMany(self): itemDictIterable = self._itemDictIterableOriginal for i in range(1000): itemDictIterable = itemDictIterable.filterByVariable('host_name', unicode(i), negate = True) self.assertEqual(723, len(itemDictIterable)) def _fillUpDataSource(self): iterable = ModsecurityAuditLogParser().parseStream(self._stream) dataSource = ModsecurityAuditDataSourceSQL(MODSECURITY_AUDIT_ENTRY_DATA_SOURCE_SQLITE_URL) dataSource.insertModsecurityAuditEntryIterable(iterable)
def _fillUpDataSource(self): iterable = ModsecurityAuditLogParser().parseStream(self._stream) dataSource = ModsecurityAuditDataSourceSQL(MODSECURITY_AUDIT_ENTRY_DATA_SOURCE_SQLITE_URL) dataSource.insertModsecurityAuditEntryIterable(iterable)