def testCorrelate(self):
# Fillup database.
dataSource = ModsecurityAuditDataSourceSQL(MODSECURITY_AUDIT_ENTRY_DATA_SOURCE_SQLITE_URL)
# Making correlation engine.
correlationEngine = CorrelationEngine(self._VARIABLE_NAME_LIST)
# Testing progress listener.
progressListener = Mock(ICorrelationProgressListener)
correlationEngine.addProgressListener(progressListener)
# Correlating.
correlationList = map(lambda correlation: repr(correlation), correlationEngine.correlate(dataSource))
self.assertEqual(self._EXPECTED_CORRELATION_LIST, correlationList)
self.assertEqual([call.progress(217, 723),
call.progress(434, 723),
call.progress(651, 723),
call.progress(656, 723),
call.progress(674, 723),
call.progress(692, 723),
call.progress(710, 723),
call.progress(712, 723),
call.progress(714, 723),
call.progress(715, 723),
call.progress(717, 723),
call.progress(719, 723),
call.progress(721, 723),
call.progress(723, 723)],
progressListener.mock_calls)
def testCorrelate(self):
# Fillup database.
dataSource = ModsecurityAuditDataSourceSQL(
MODSECURITY_AUDIT_ENTRY_DATA_SOURCE_SQLITE_URL)
# Making correlation engine.
correlationEngine = CorrelationEngine(self._VARIABLE_NAME_LIST)
# Testing progress listener.
progressListener = Mock(ICorrelationProgressListener)
correlationEngine.addProgressListener(progressListener)
# Correlating.
correlationList = map(lambda correlation: repr(correlation),
correlationEngine.correlate(dataSource))
self.assertEqual(self._EXPECTED_CORRELATION_LIST, correlationList)
self.assertEqual([
call.progress(217, 723),
call.progress(434, 723),
call.progress(651, 723),
call.progress(656, 723),
call.progress(674, 723),
call.progress(692, 723),
call.progress(710, 723),
call.progress(712, 723),
call.progress(714, 723),
call.progress(715, 723),
call.progress(717, 723),
call.progress(719, 723),
call.progress(721, 723),
call.progress(723, 723)
], progressListener.mock_calls)
def main(self, argumentList):
# Disabling contracts solves some performance issues.
contracts.disable_all()
argumentParser = argparse.ArgumentParser(
description=u"Make ModSecurity exceptions.")
argumentParser.add_argument(
u"-i",
u"--input",
metavar=u"MODSEC_AUDIT_LOG_FILE",
dest='modsecurityAuditLogPath',
type=unicode,
default=None,
help=
u"Modsecurity audit log file path or '-' to read from standard input."
)
argumentParser.add_argument(
u"-d",
u"--data-url",
dest='dataURL',
type=unicode,
required=True,
default=None,
help=u"Example: 'sqlite:////tmp/modsecurity-exception-factory.db'")
argumentParser.add_argument(u"-c",
u"--config-file",
dest='configFilePath',
type=unicode,
default=None)
argumentObject = argumentParser.parse_args(argumentList)
# Try to parse config.
config = Config(argumentObject.configFilePath)
variableNameList = config.variableNameList()
ignoredVariableDict = config.ignoredVariableDict()
minimumOccurrenceCountThreshold = config.minimumOccurrenceCountThreshold(
)
maximumValueCountThreshold = config.maximumValueCountThreshold()
# Initialize data source object.
dataSource = ModsecurityAuditDataSourceSQL(argumentObject.dataURL)
# Parse log if given.
if argumentObject.modsecurityAuditLogPath is not None:
self._parseFile(argumentObject.modsecurityAuditLogPath, dataSource)
# Preparing correlation engine.
correlationEngine = CorrelationEngine(variableNameList,
ignoredVariableDict,
minimumOccurrenceCountThreshold,
maximumValueCountThreshold)
correlationEngine.addProgressListener(
CorrelationProgressListenerConsole(sys.stderr))
# Correlating and writing exceptions progressively using the power of Python generators.
ModsecurityExceptionWriter(stream=sys.stdout).write(
correlationEngine.correlate(dataSource))
return 0
def testCorrelationWithMaximumValueCount(self):
# Fillup database.
dataSource = ModsecurityAuditDataSourceSQL(MODSECURITY_AUDIT_ENTRY_DATA_SOURCE_SQLITE_URL)
correlationEngine = CorrelationEngine(self._VARIABLE_NAME_LIST,
maximumValueCountThreshold = 5)
correlationList = map(lambda correlation: repr(correlation), correlationEngine.correlate(dataSource))
self.assertEqual(self._EXPECTED_CORRELATION_LIST_WITH_MAXIMUM_VALUE_COUNT, correlationList)
def testCorrelateWithIgnoredVariableDict(self):
# Fillup database.
dataSource = ModsecurityAuditDataSourceSQL(MODSECURITY_AUDIT_ENTRY_DATA_SOURCE_SQLITE_URL)
ignoredVariableDict = {'host_name': [u"1.1.1.1"],
'rule_id': [u"111111", u"981174"]}
correlationEngine = CorrelationEngine(self._VARIABLE_NAME_LIST, ignoredVariableDict)
correlationList = map(lambda correlation: repr(correlation), correlationEngine.correlate(dataSource))
self.assertEqual(self._EXPECTED_CORRELATION_LIST_WITH_IGNORED_VARIABLE_DICT, correlationList)
def testCorrelationWithMaximumValueCount(self):
# Fillup database.
dataSource = ModsecurityAuditDataSourceSQL(
MODSECURITY_AUDIT_ENTRY_DATA_SOURCE_SQLITE_URL)
correlationEngine = CorrelationEngine(self._VARIABLE_NAME_LIST,
maximumValueCountThreshold=5)
correlationList = map(lambda correlation: repr(correlation),
correlationEngine.correlate(dataSource))
self.assertEqual(
self._EXPECTED_CORRELATION_LIST_WITH_MAXIMUM_VALUE_COUNT,
correlationList)
def main(self, argumentList):
# Disabling contracts solves some performance issues.
contracts.disable_all()
argumentParser = argparse.ArgumentParser(description = u"Make ModSecurity exceptions.")
argumentParser.add_argument(u"-i",
u"--input",
metavar = u"MODSEC_AUDIT_LOG_FILE",
dest = 'modsecurityAuditLogPath',
type = unicode,
default = None,
help = u"Modsecurity audit log file path or '-' to read from standard input.")
argumentParser.add_argument(u"-d",
u"--data-url",
dest = 'dataURL',
type = unicode,
required = True,
default = None,
help = u"Example: 'sqlite:////tmp/modsecurity-exception-factory.db'")
argumentParser.add_argument(u"-c",
u"--config-file",
dest = 'configFilePath',
type = unicode,
default = None)
argumentObject = argumentParser.parse_args(argumentList)
# Try to parse config.
config = Config(argumentObject.configFilePath)
variableNameList = config.variableNameList()
ignoredVariableDict = config.ignoredVariableDict()
minimumOccurrenceCountThreshold = config.minimumOccurrenceCountThreshold()
maximumValueCountThreshold = config.maximumValueCountThreshold()
# Initialize data source object.
dataSource = ModsecurityAuditDataSourceSQL(argumentObject.dataURL)
# Parse log if given.
if argumentObject.modsecurityAuditLogPath is not None:
self._parseFile(argumentObject.modsecurityAuditLogPath, dataSource)
# Preparing correlation engine.
correlationEngine = CorrelationEngine(variableNameList,
ignoredVariableDict,
minimumOccurrenceCountThreshold,
maximumValueCountThreshold)
correlationEngine.addProgressListener(CorrelationProgressListenerConsole(sys.stderr))
# Correlating and writing exceptions progressively using the power of Python generators.
ModsecurityExceptionWriter(stream = sys.stdout).write(correlationEngine.correlate(dataSource))
return 0
def testCorrelateWithIgnoredVariableDict(self):
# Fillup database.
dataSource = ModsecurityAuditDataSourceSQL(
MODSECURITY_AUDIT_ENTRY_DATA_SOURCE_SQLITE_URL)
ignoredVariableDict = {
'host_name': [u"1.1.1.1"],
'rule_id': [u"111111", u"981174"]
}
correlationEngine = CorrelationEngine(self._VARIABLE_NAME_LIST,
ignoredVariableDict)
correlationList = map(lambda correlation: repr(correlation),
correlationEngine.correlate(dataSource))
self.assertEqual(
self._EXPECTED_CORRELATION_LIST_WITH_IGNORED_VARIABLE_DICT,
correlationList)
