def setUp(self):
self.flt = DataObjectFilter()
self.psr = PynspectFilterParser()
self.psr.build()
self.cpl = IDEAFilterCompiler()
self.msg_idea = lite.Idea(self.test_msg1)
def __init__(self, rule, actions, addrGroups, parser=None, compiler=None):
# Check is we got parser instance
if parser is None:
self.parser = PynspectFilterParser()
self.parser.build()
else:
self.parser = parser
# Check is we got compiler instance
if compiler is None:
self.compiler = IDEAFilterCompiler()
else:
self.compiler = compiler
# Instantiate filter
self.__filter = DataObjectFilter()
# Store rule condition in raw form
self.__conditionRaw = rule["condition"]
if not self.__matchvar(rule["condition"], addrGroups):
self.__condition = self.__conditionRaw
# Set inner rule ID
self.id = rule["id"]
if (self.__condition != None):
self.parseRule()
self.__actions = list()
self.__elseactions = list()
# Associate actions
if "actions" in rule:
for actionId in rule["actions"]:
try:
logger.debug("Rule %s inserting %s", self.id, actionId)
self.__actions.append(actions[actionId])
except KeyError as e:
raise Exception("Missing action with ID " + str(e))
# Associate elseactions
if "elseactions" in rule:
for actionId in rule["elseactions"]:
try:
self.__elseactions.append(actions[actionId])
except KeyError as e:
raise Exception("Missing elseaction with ID " + str(e))
def test_05_non_existent_nodes(self):
"""
Perform advanced filtering tests.
"""
self.maxDiff = None
flt = DataObjectFilter()
psr = MentatFilterParser()
psr.build()
rule = psr.parse('(ConnCounts + 10) > 11')
self.assertEqual(flt.filter(rule, self.test_msg1), None)
rule = psr.parse('ConnCount > ConnCounts')
self.assertEqual(flt.filter(rule, self.test_msg1), None)
rule = psr.parse('DetectTime < InspectionTime')
self.assertEqual(flt.filter(rule, self.test_msg1), None)
def test_04_advanced_filters(self):
"""
Perform advanced filtering tests.
"""
self.maxDiff = None
flt = DataObjectFilter()
psr = MentatFilterParser()
psr.build()
rule = psr.parse('(ConnCount + 10) > 11')
self.assertEqual(flt.filter(rule, self.test_msg1), True)
rule = psr.parse('((ConnCount + 3) < 5) or ((ConnCount + 10) > 11)')
self.assertEqual(flt.filter(rule, self.test_msg1), True)
rule = psr.parse('1')
self.assertEqual(flt.filter(rule, self.test_msg1), True)
def test_06_shortcuts(self):
"""
Perform tests of shortcut methods.
"""
self.maxDiff = None
# Let the shortcut method initialize everything.
flt = DataObjectFilter(
parser = PynspectFilterParser,
compiler = IDEAFilterCompiler
)
rule = flt.prepare('(Source.IP4 == 188.14.166.39)')
self.assertEqual(repr(rule), "COMPBINOP(VARIABLE('Source.IP4') OP_EQ IPV4(IP4('188.14.166.39')))")
self.assertEqual(self.check_rule(rule), True)
# Create parser and compiler instances by hand, but register them into filter.
cpl = IDEAFilterCompiler()
psr = PynspectFilterParser()
psr.build()
flt = DataObjectFilter(
parser = psr,
compiler = cpl
)
rule = flt.prepare('(Source.IP4 == 188.14.166.39)')
self.assertEqual(repr(rule), "COMPBINOP(VARIABLE('Source.IP4') OP_EQ IPV4(IP4('188.14.166.39')))")
self.assertEqual(self.check_rule(rule), True)
class Rule():
def __init__(self, rule, actions, addrGroups, parser=None, compiler=None):
# Check is we got parser instance
if parser is None:
self.parser = PynspectFilterParser()
self.parser.build()
else:
self.parser = parser
# Check is we got compiler instance
if compiler is None:
self.compiler = IDEAFilterCompiler()
else:
self.compiler = compiler
# Instantiate filter
self.__filter = DataObjectFilter()
# Store rule condition in raw form
self.__conditionRaw = rule["condition"]
if not self.__matchvar(rule["condition"], addrGroups):
self.__condition = self.__conditionRaw
# Set inner rule ID
self.id = rule["id"]
if (self.__condition != None):
self.parseRule()
self.__actions = list()
self.__elseactions = list()
# Associate actions
if "actions" in rule:
for actionId in rule["actions"]:
try:
logger.debug("Rule %s inserting %s", self.id, actionId)
self.__actions.append(actions[actionId])
except KeyError as e:
raise Exception("Missing action with ID " + str(e))
# Associate elseactions
if "elseactions" in rule:
for actionId in rule["elseactions"]:
try:
self.__elseactions.append(actions[actionId])
except KeyError as e:
raise Exception("Missing elseaction with ID " + str(e))
def parseRule(self):
cond = str(self.__condition).lower()
if cond in ["none", "null", "true"]:
self.__condition = True
elif cond == "false":
self.__condition = False
else:
try:
self.__condition = self.parser.parse(self.__condition)
self.__condition = self.compiler.compile(self.__condition)
except Exception as e:
print(
"Error while parsing condition: {0}\nOriginal exception: {1}"
.format(self.__condition, e))
def filter(self, record):
"""
Filter given record based on rule's condition
@note If the rule's condition is empty, record is always matched.
@return Boolean
"""
# The message must be converted via idea module
if not isinstance(record, lite.Idea):
logger.info("Converting message to IDEA")
record = lite.Idea(record)
logger.debug(record)
logger.debug(self.__condition)
if self.__condition == None or self.__condition == True:
# Rule condition is empty (tautology) - should always match the record
res = True
elif self.__condition == False:
res = False
else:
# Match the record with non-empty rule's condition
res = self.__filter.filter(self.__condition, record)
logger.debug("RESULT: %s", res)
return res
def actions(self, record):
for action in self.__actions:
action.run(record)
def elseactions(self, record):
for action in self.__elseactions:
action.run(record)
def __repr__(self):
return self.__conditionRaw
def __str__(self):
actions = []
for i in self.__actions:
actions.append("{0} ({1})".format(i.actionId, i.actionType))
elseactions = []
for i in self.__elseactions:
elseactions.append("{0} ({1})".format(i.actionId, i.actionType))
return "{0}: {1}\n{2}{3}".format(
self.id,
" ".join([i.strip()
for i in str(self.__conditionRaw).split("\n")]),
"\tActions: " + (", ".join(actions)) + "\n" if actions else "",
"\tElse Actions: " + (", ".join(elseactions)) +
"\n" if elseactions else "")
def rule(self):
return str(self.__condition)
def __matchvar(self, rule, addrGroups):
"""
Since pyncspect doesn't support variables yet we had to provide
a solution where every address group's name is matched against
the rule's condition and eventually replaced with address group's values
"""
matched = False
"""
Tautology - empty rule should always match
Don't try to match or replace any address group
"""
if rule == None or isinstance(rule, bool):
return False
for key in addrGroups:
if key in rule:
rule = re.sub(r"\b{0}\b".format(re.escape(key)),
addrGroups[key].iplist(), rule)
matched = True
self.__condition = rule
return matched
class TestDataObjectFilterIDEA(unittest.TestCase):
"""
Unit test class for testing the :py:mod:`pynspect.filters` module.
"""
test_msg1 = {
"ID" : "e214d2d9-359b-443d-993d-3cc5637107a0",
"WinEndTime" : "2016-06-21 11:25:01Z",
"ConnCount" : 2,
"Source" : [
{
"IP4" : [
"188.14.166.39"
]
}
],
"Format" : "IDEA0",
"WinStartTime" : "2016-06-21 11:20:01Z",
"_CESNET" : {
"StorageTime" : 1466508305
},
"Target" : [
{
"IP4" : [
"195.113.165.128/25"
],
"Port" : [
"22"
],
"Proto" : [
"tcp",
"ssh"
],
"Anonymised" : True
}
],
"Note" : "SSH login attempt",
"DetectTime" : "2016-06-21 13:08:27Z",
"Node" : [
{
"Name" : "cz.cesnet.mentat.warden_filer",
"Type" : [
"Relay"
]
},
{
"AggrWin" : "00:05:00",
"Type" : [
"Connection",
"Honeypot",
"Recon"
],
"SW" : [
"Kippo"
],
"Name" : "cz.uhk.apate.cowrie"
}
],
"Category" : [
"Attempt.Login"
]
}
def setUp(self):
self.flt = DataObjectFilter()
self.psr = PynspectFilterParser()
self.psr.build()
self.cpl = IDEAFilterCompiler()
self.msg_idea = lite.Idea(self.test_msg1)
def build_rule(self, rule_str):
"""
Build and compile rule tree from given rule string.
"""
rule = self.psr.parse(rule_str)
rule = self.cpl.compile(rule)
return rule
def check_rule(self, rule):
"""
Check given rule against internal test message and filter.
"""
return self.flt.filter(rule, self.msg_idea)
def test_01_basic_logical(self):
"""
Perform filtering tests with basic logical expressions.
"""
self.maxDiff = None
rule = LogicalBinOpRule('OP_AND', ConstantRule(True), ConstantRule(True))
self.assertEqual(self.check_rule(rule), True)
rule = LogicalBinOpRule('OP_AND', ConstantRule(True), ConstantRule(False))
self.assertEqual(self.check_rule(rule), False)
rule = LogicalBinOpRule('OP_AND', ConstantRule(False), ConstantRule(True))
self.assertEqual(self.check_rule(rule), False)
rule = LogicalBinOpRule('OP_AND', ConstantRule(False), ConstantRule(False))
self.assertEqual(self.check_rule(rule), False)
rule = LogicalBinOpRule('OP_OR', ConstantRule(True), ConstantRule(True))
self.assertEqual(self.check_rule(rule), True)
rule = LogicalBinOpRule('OP_OR', ConstantRule(True), ConstantRule(False))
self.assertEqual(self.check_rule(rule), True)
rule = LogicalBinOpRule('OP_OR', ConstantRule(False), ConstantRule(True))
self.assertEqual(self.check_rule(rule), True)
rule = LogicalBinOpRule('OP_OR', ConstantRule(False), ConstantRule(False))
self.assertEqual(self.check_rule(rule), False)
rule = LogicalBinOpRule('OP_XOR', ConstantRule(True), ConstantRule(True))
self.assertEqual(self.check_rule(rule), False)
rule = LogicalBinOpRule('OP_XOR', ConstantRule(True), ConstantRule(False))
self.assertEqual(self.check_rule(rule), True)
rule = LogicalBinOpRule('OP_XOR', ConstantRule(False), ConstantRule(True))
self.assertEqual(self.check_rule(rule), True)
rule = LogicalBinOpRule('OP_XOR', ConstantRule(False), ConstantRule(False))
self.assertEqual(self.check_rule(rule), False)
rule = UnaryOperationRule('OP_NOT', ConstantRule(True))
self.assertEqual(self.check_rule(rule), False)
rule = UnaryOperationRule('OP_NOT', ConstantRule(False))
self.assertEqual(self.check_rule(rule), True)
rule = UnaryOperationRule('OP_NOT', VariableRule("Target.Anonymised"))
self.assertEqual(self.check_rule(rule), False)
def test_02_basic_comparison(self):
"""
Perform filtering tests with basic comparison operations.
"""
self.maxDiff = None
rule = ComparisonBinOpRule('OP_EQ', VariableRule("ID"), ConstantRule("e214d2d9-359b-443d-993d-3cc5637107a0"))
self.assertEqual(self.check_rule(rule), True)
rule = ComparisonBinOpRule('OP_EQ', VariableRule("ID"), ConstantRule("e214d2d9-359b-443d-993d-3cc5637107"))
self.assertEqual(self.check_rule(rule), False)
rule = ComparisonBinOpRule('OP_NE', VariableRule("ID"), ConstantRule("e214d2d9-359b-443d-993d-3cc5637107a0"))
self.assertEqual(self.check_rule(rule), False)
rule = ComparisonBinOpRule('OP_NE', VariableRule("ID"), ConstantRule("e214d2d9-359b-443d-993d-3cc5637107"))
self.assertEqual(self.check_rule(rule), True)
rule = ComparisonBinOpRule('OP_LIKE', VariableRule("ID"), ConstantRule("e214d2d9"))
self.assertEqual(self.check_rule(rule), True)
rule = ComparisonBinOpRule('OP_LIKE', VariableRule("ID"), ConstantRule("xxxxxxxx"))
self.assertEqual(self.check_rule(rule), False)
rule = ComparisonBinOpRule('OP_IN', VariableRule("Category"), ListRule(ConstantRule("Phishing"), ListRule(ConstantRule("Attempt.Login"))))
self.assertEqual(self.check_rule(rule), True)
rule = ComparisonBinOpRule('OP_IN', VariableRule("Category"), ListRule(ConstantRule("Phishing"), ListRule(ConstantRule("Spam"))))
self.assertEqual(self.check_rule(rule), False)
rule = ComparisonBinOpRule('OP_IS', VariableRule("Category"), ListRule(ConstantRule("Attempt.Login")))
self.assertEqual(self.check_rule(rule), True)
rule = ComparisonBinOpRule('OP_IS', VariableRule("Category"), ListRule(ConstantRule("Phishing"), ListRule(ConstantRule("Attempt.Login"))))
self.assertEqual(self.check_rule(rule), False)
rule = ComparisonBinOpRule('OP_EQ', VariableRule("ConnCount"), IntegerRule(2))
self.assertEqual(self.check_rule(rule), True)
rule = ComparisonBinOpRule('OP_EQ', VariableRule("ConnCount"), IntegerRule(4))
self.assertEqual(self.check_rule(rule), False)
rule = ComparisonBinOpRule('OP_NE', VariableRule("ConnCount"), IntegerRule(2))
self.assertEqual(self.check_rule(rule), False)
rule = ComparisonBinOpRule('OP_NE', VariableRule("ConnCount"), IntegerRule(4))
self.assertEqual(self.check_rule(rule), True)
rule = ComparisonBinOpRule('OP_GT', VariableRule("ConnCount"), IntegerRule(2))
self.assertEqual(self.check_rule(rule), False)
rule = ComparisonBinOpRule('OP_GT', VariableRule("ConnCount"), IntegerRule(1))
self.assertEqual(self.check_rule(rule), True)
rule = ComparisonBinOpRule('OP_GE', VariableRule("ConnCount"), IntegerRule(2))
self.assertEqual(self.check_rule(rule), True)
rule = ComparisonBinOpRule('OP_GE', VariableRule("ConnCount"), IntegerRule(1))
self.assertEqual(self.check_rule(rule), True)
rule = ComparisonBinOpRule('OP_GE', VariableRule("ConnCount"), IntegerRule(3))
self.assertEqual(self.check_rule(rule), False)
rule = ComparisonBinOpRule('OP_LT', VariableRule("ConnCount"), IntegerRule(2))
self.assertEqual(self.check_rule(rule), False)
rule = ComparisonBinOpRule('OP_LT', VariableRule("ConnCount"), IntegerRule(3))
self.assertEqual(self.check_rule(rule), True)
rule = ComparisonBinOpRule('OP_LE', VariableRule("ConnCount"), IntegerRule(2))
self.assertEqual(self.check_rule(rule), True)
rule = ComparisonBinOpRule('OP_LE', VariableRule("ConnCount"), IntegerRule(3))
self.assertEqual(self.check_rule(rule), True)
rule = ComparisonBinOpRule('OP_LE', VariableRule("ConnCount"), IntegerRule(1))
self.assertEqual(self.check_rule(rule), False)
def test_03_parsed_comparison(self):
"""
Perform filtering tests with basic parsed comparison operations.
"""
self.maxDiff = None
rule = self.build_rule('ID == "e214d2d9-359b-443d-993d-3cc5637107a0"')
self.assertEqual(self.check_rule(rule), True)
rule = self.build_rule('ID eq "e214d2d9-359b-443d-993d-3cc5637107"')
self.assertEqual(self.check_rule(rule), False)
rule = self.build_rule('ID != "e214d2d9-359b-443d-993d-3cc5637107a0"')
self.assertEqual(self.check_rule(rule), False)
rule = self.build_rule('ID ne "e214d2d9-359b-443d-993d-3cc5637107"')
self.assertEqual(self.check_rule(rule), True)
rule = self.build_rule('ID like "e214d2d9"')
self.assertEqual(self.check_rule(rule), True)
rule = self.build_rule('ID LIKE "xxxxxxxx"')
self.assertEqual(self.check_rule(rule), False)
rule = self.build_rule('Category in ["Phishing" , "Attempt.Login"]')
self.assertEqual(self.check_rule(rule), True)
rule = self.build_rule('Category IN ["Phishing" , "Spam"]')
self.assertEqual(self.check_rule(rule), False)
rule = self.build_rule('Category is ["Attempt.Login"]')
self.assertEqual(self.check_rule(rule), True)
rule = self.build_rule('Category IS ["Phishing" , "Attempt.Login"]')
self.assertEqual(self.check_rule(rule), False)
rule = self.build_rule('ConnCount == 2')
self.assertEqual(self.check_rule(rule), True)
rule = self.build_rule('ConnCount eq 4')
self.assertEqual(self.check_rule(rule), False)
rule = self.build_rule('ConnCount != 2')
self.assertEqual(self.check_rule(rule), False)
rule = self.build_rule('ConnCount ne 4')
self.assertEqual(self.check_rule(rule), True)
rule = self.build_rule('ConnCount > 2')
self.assertEqual(self.check_rule(rule), False)
rule = self.build_rule('ConnCount gt 1')
self.assertEqual(self.check_rule(rule), True)
rule = self.build_rule('ConnCount >= 2')
self.assertEqual(self.check_rule(rule), True)
rule = self.build_rule('ConnCount ge 1')
self.assertEqual(self.check_rule(rule), True)
rule = self.build_rule('ConnCount GE 3')
self.assertEqual(self.check_rule(rule), False)
rule = self.build_rule('ConnCount < 2')
self.assertEqual(self.check_rule(rule), False)
rule = self.build_rule('ConnCount lt 3')
self.assertEqual(self.check_rule(rule), True)
rule = self.build_rule('ConnCount <= 2')
self.assertEqual(self.check_rule(rule), True)
rule = self.build_rule('ConnCount le 3')
self.assertEqual(self.check_rule(rule), True)
rule = self.build_rule('ConnCount LE 1')
self.assertEqual(self.check_rule(rule), False)
def test_04_basic_math(self):
"""
Perform filtering tests with basic math operations.
"""
self.maxDiff = None
rule = MathBinOpRule('OP_PLUS', VariableRule("ConnCount"), IntegerRule(1))
self.assertEqual(self.check_rule(rule), 3)
rule = MathBinOpRule('OP_MINUS', VariableRule("ConnCount"), IntegerRule(1))
self.assertEqual(self.check_rule(rule), 1)
rule = MathBinOpRule('OP_TIMES', VariableRule("ConnCount"), IntegerRule(5))
self.assertEqual(self.check_rule(rule), 10)
rule = MathBinOpRule('OP_DIVIDE', VariableRule("ConnCount"), IntegerRule(2))
self.assertEqual(self.check_rule(rule), 1)
rule = MathBinOpRule('OP_MODULO', VariableRule("ConnCount"), IntegerRule(2))
self.assertEqual(self.check_rule(rule), 0)
def test_05_parsed_math(self):
"""
Perform filtering tests with parsed math operations.
"""
self.maxDiff = None
rule = self.build_rule('ConnCount + 1')
self.assertEqual(self.check_rule(rule), 3)
rule = self.build_rule('ConnCount - 1')
self.assertEqual(self.check_rule(rule), 1)
rule = self.build_rule('ConnCount * 5')
self.assertEqual(self.check_rule(rule), 10)
rule = self.build_rule('ConnCount / 2')
self.assertEqual(self.check_rule(rule), 1)
rule = self.build_rule('ConnCount % 2')
self.assertEqual(self.check_rule(rule), 0)
def test_06_advanced_filters(self):
"""
Perform advanced filtering tests.
"""
self.maxDiff = None
rule = self.build_rule('DetectTime + 3600')
self.assertEqual(repr(rule), "MATHBINOP(VARIABLE('DetectTime') OP_PLUS TIMEDELTA(datetime.timedelta(0, 3600)))")
expected_res = (datetime.datetime(2016, 6, 21, 13, 8, 27) + datetime.timedelta(seconds = 3600))
self.assertEqual(self.check_rule(rule), expected_res)
rule = self.build_rule('(ConnCount + 10) > 11')
self.assertEqual(repr(rule), "COMPBINOP(MATHBINOP(VARIABLE('ConnCount') OP_PLUS INTEGER(10)) OP_GT INTEGER(11))")
self.assertEqual(self.check_rule(rule), True)
rule = self.build_rule('(ConnCount + 3) < 5')
self.assertEqual(repr(rule), "COMPBINOP(MATHBINOP(VARIABLE('ConnCount') OP_PLUS INTEGER(3)) OP_LT INTEGER(5))")
self.assertEqual(self.check_rule(rule), False)
rule = self.build_rule('((ConnCount + 3) < 5) or ((ConnCount + 10) > 11)')
self.assertEqual(repr(rule), "LOGBINOP(COMPBINOP(MATHBINOP(VARIABLE('ConnCount') OP_PLUS INTEGER(3)) OP_LT INTEGER(5)) OP_OR COMPBINOP(MATHBINOP(VARIABLE('ConnCount') OP_PLUS INTEGER(10)) OP_GT INTEGER(11)))")
self.assertEqual(self.check_rule(rule), True)
rule = self.build_rule('(DetectTime == 2016-06-21T13:08:27Z)')
self.assertEqual(repr(rule), "COMPBINOP(VARIABLE('DetectTime') OP_EQ DATETIME(datetime.datetime(2016, 6, 21, 13, 8, 27)))")
self.assertEqual(self.check_rule(rule), True)
rule = self.build_rule('(DetectTime != 2016-06-21T13:08:27Z)')
self.assertEqual(repr(rule), "COMPBINOP(VARIABLE('DetectTime') OP_NE DATETIME(datetime.datetime(2016, 6, 21, 13, 8, 27)))")
self.assertEqual(self.check_rule(rule), False)
rule = self.build_rule('(DetectTime >= 2016-06-21T14:08:27Z)')
self.assertEqual(repr(rule), "COMPBINOP(VARIABLE('DetectTime') OP_GE DATETIME(datetime.datetime(2016, 6, 21, 14, 8, 27)))")
self.assertEqual(self.check_rule(rule), False)
rule = self.build_rule('(DetectTime <= 2016-06-21T14:08:27Z)')
self.assertEqual(repr(rule), "COMPBINOP(VARIABLE('DetectTime') OP_LE DATETIME(datetime.datetime(2016, 6, 21, 14, 8, 27)))")
self.assertEqual(self.check_rule(rule), True)
rule = self.build_rule('DetectTime < (utcnow() + 05:00:00)')
self.assertEqual(repr(rule), "COMPBINOP(VARIABLE('DetectTime') OP_LT MATHBINOP(FUNCTION(utcnow()) OP_PLUS TIMEDELTA(datetime.timedelta(0, 18000))))")
self.assertEqual(self.check_rule(rule), True)
rule = self.build_rule('DetectTime > (utcnow() - 05:00:00)')
self.assertEqual(repr(rule), "COMPBINOP(VARIABLE('DetectTime') OP_GT MATHBINOP(FUNCTION(utcnow()) OP_MINUS TIMEDELTA(datetime.timedelta(0, 18000))))")
self.assertEqual(self.check_rule(rule), False)
rule = self.build_rule('(Source.IP4 == 188.14.166.39)')
self.assertEqual(repr(rule), "COMPBINOP(VARIABLE('Source.IP4') OP_EQ IPV4(IP4('188.14.166.39')))")
self.assertEqual(self.check_rule(rule), True)
rule = self.build_rule('(Source.IP4 in ["188.14.166.39","188.14.166.40","188.14.166.41"])')
self.assertEqual(repr(rule), "COMPBINOP(VARIABLE('Source.IP4') OP_IN IPLIST(IPV4(IP4('188.14.166.39')), IPV4(IP4('188.14.166.40')), IPV4(IP4('188.14.166.41'))))")
self.assertEqual(self.check_rule(rule), True)
# list with CIDR addresses
rule = self.build_rule('(Source.IP4 in ["188.14.166.0/24","10.0.0.0/8","189.14.166.41"])')
self.assertEqual(repr(rule), "COMPBINOP(VARIABLE('Source.IP4') OP_IN IPLIST(IPV4(IP4Net('188.14.166.0/24')), IPV4(IP4Net('10.0.0.0/8')), IPV4(IP4('189.14.166.41'))))")
self.assertEqual(self.check_rule(rule), True)
def test_06_shortcuts(self):
"""
Perform tests of shortcut methods.
"""
self.maxDiff = None
# Let the shortcut method initialize everything.
flt = DataObjectFilter(
parser = PynspectFilterParser,
compiler = IDEAFilterCompiler
)
rule = flt.prepare('(Source.IP4 == 188.14.166.39)')
self.assertEqual(repr(rule), "COMPBINOP(VARIABLE('Source.IP4') OP_EQ IPV4(IP4('188.14.166.39')))")
self.assertEqual(self.check_rule(rule), True)
# Create parser and compiler instances by hand, but register them into filter.
cpl = IDEAFilterCompiler()
psr = PynspectFilterParser()
psr.build()
flt = DataObjectFilter(
parser = psr,
compiler = cpl
)
rule = flt.prepare('(Source.IP4 == 188.14.166.39)')
self.assertEqual(repr(rule), "COMPBINOP(VARIABLE('Source.IP4') OP_EQ IPV4(IP4('188.14.166.39')))")
self.assertEqual(self.check_rule(rule), True)
def test_01_basic_logical(self):
"""
Perform basic filtering tests.
"""
self.maxDiff = None
flt = DataObjectFilter()
rule = LogicalBinOpRule('OP_AND', ConstantRule(True),
ConstantRule(True))
self.assertEqual(flt.filter(rule, self.test_msg1), True)
rule = LogicalBinOpRule('OP_AND', ConstantRule(True),
ConstantRule(False))
self.assertEqual(flt.filter(rule, self.test_msg1), False)
rule = LogicalBinOpRule('OP_AND', ConstantRule(False),
ConstantRule(True))
self.assertEqual(flt.filter(rule, self.test_msg1), False)
rule = LogicalBinOpRule('OP_AND', ConstantRule(False),
ConstantRule(False))
self.assertEqual(flt.filter(rule, self.test_msg1), False)
rule = LogicalBinOpRule('OP_OR', ConstantRule(True),
ConstantRule(True))
self.assertEqual(flt.filter(rule, self.test_msg1), True)
rule = LogicalBinOpRule('OP_OR', ConstantRule(True),
ConstantRule(False))
self.assertEqual(flt.filter(rule, self.test_msg1), True)
rule = LogicalBinOpRule('OP_OR', ConstantRule(False),
ConstantRule(True))
self.assertEqual(flt.filter(rule, self.test_msg1), True)
rule = LogicalBinOpRule('OP_OR', ConstantRule(False),
ConstantRule(False))
self.assertEqual(flt.filter(rule, self.test_msg1), False)
rule = LogicalBinOpRule('OP_XOR', ConstantRule(True),
ConstantRule(True))
self.assertEqual(flt.filter(rule, self.test_msg1), False)
rule = LogicalBinOpRule('OP_XOR', ConstantRule(True),
ConstantRule(False))
self.assertEqual(flt.filter(rule, self.test_msg1), True)
rule = LogicalBinOpRule('OP_XOR', ConstantRule(False),
ConstantRule(True))
self.assertEqual(flt.filter(rule, self.test_msg1), True)
rule = LogicalBinOpRule('OP_XOR', ConstantRule(False),
ConstantRule(False))
self.assertEqual(flt.filter(rule, self.test_msg1), False)
rule = UnaryOperationRule('OP_NOT', ConstantRule(True))
self.assertEqual(flt.filter(rule, self.test_msg1), False)
rule = UnaryOperationRule('OP_NOT', ConstantRule(False))
self.assertEqual(flt.filter(rule, self.test_msg1), True)
rule = UnaryOperationRule('OP_NOT', VariableRule("Target.Anonymised"))
self.assertEqual(flt.filter(rule, self.test_msg1), False)
def test_03_basic_math(self):
"""
Perform basic math tests.
"""
self.maxDiff = None
flt = DataObjectFilter()
psr = MentatFilterParser()
psr.build()
rule = MathBinOpRule('OP_PLUS', VariableRule("ConnCount"),
IntegerRule(1))
self.assertEqual(flt.filter(rule, self.test_msg1), 3)
rule = MathBinOpRule('OP_MINUS', VariableRule("ConnCount"),
IntegerRule(1))
self.assertEqual(flt.filter(rule, self.test_msg1), 1)
rule = MathBinOpRule('OP_TIMES', VariableRule("ConnCount"),
IntegerRule(5))
self.assertEqual(flt.filter(rule, self.test_msg1), 10)
rule = MathBinOpRule('OP_DIVIDE', VariableRule("ConnCount"),
IntegerRule(2))
self.assertEqual(flt.filter(rule, self.test_msg1), 1)
rule = MathBinOpRule('OP_MODULO', VariableRule("ConnCount"),
IntegerRule(2))
self.assertEqual(flt.filter(rule, self.test_msg1), 0)
rule = psr.parse('ConnCount + 1')
self.assertEqual(flt.filter(rule, self.test_msg1), 3)
rule = psr.parse('ConnCount - 1')
self.assertEqual(flt.filter(rule, self.test_msg1), 1)
rule = psr.parse('ConnCount * 5')
self.assertEqual(flt.filter(rule, self.test_msg1), 10)
rule = psr.parse('ConnCount / 2')
self.assertEqual(flt.filter(rule, self.test_msg1), 1)
rule = psr.parse('ConnCount % 2')
self.assertEqual(flt.filter(rule, self.test_msg1), 0)
def test_02_basic_comparison(self):
"""
Perform basic filtering tests.
"""
self.maxDiff = None
flt = DataObjectFilter()
psr = MentatFilterParser()
psr.build()
rule = ComparisonBinOpRule(
'OP_EQ', VariableRule("ID"),
ConstantRule("e214d2d9-359b-443d-993d-3cc5637107a0"))
self.assertEqual(flt.filter(rule, self.test_msg1), True)
rule = ComparisonBinOpRule(
'OP_EQ', VariableRule("ID"),
ConstantRule("e214d2d9-359b-443d-993d-3cc5637107"))
self.assertEqual(flt.filter(rule, self.test_msg1), False)
rule = ComparisonBinOpRule(
'OP_NE', VariableRule("ID"),
ConstantRule("e214d2d9-359b-443d-993d-3cc5637107a0"))
self.assertEqual(flt.filter(rule, self.test_msg1), False)
rule = ComparisonBinOpRule(
'OP_NE', VariableRule("ID"),
ConstantRule("e214d2d9-359b-443d-993d-3cc5637107"))
self.assertEqual(flt.filter(rule, self.test_msg1), True)
rule = ComparisonBinOpRule('OP_LIKE', VariableRule("ID"),
ConstantRule("e214d2d9"))
self.assertEqual(flt.filter(rule, self.test_msg1), True)
rule = ComparisonBinOpRule('OP_LIKE', VariableRule("ID"),
ConstantRule("xxxxxxxx"))
self.assertEqual(flt.filter(rule, self.test_msg1), False)
rule = ComparisonBinOpRule(
'OP_IN', VariableRule("Category"),
ListRule(ConstantRule("Phishing"),
ListRule(ConstantRule("Attempt.Login"))))
self.assertEqual(flt.filter(rule, self.test_msg1), True)
rule = ComparisonBinOpRule(
'OP_IN', VariableRule("Category"),
ListRule(ConstantRule("Phishing"), ListRule(ConstantRule("Spam"))))
self.assertEqual(flt.filter(rule, self.test_msg1), False)
rule = ComparisonBinOpRule('OP_IS', VariableRule("Category"),
ListRule(ConstantRule("Attempt.Login")))
self.assertEqual(flt.filter(rule, self.test_msg1), True)
rule = ComparisonBinOpRule(
'OP_IS', VariableRule("Category"),
ListRule(ConstantRule("Phishing"),
ListRule(ConstantRule("Attempt.Login"))))
self.assertEqual(flt.filter(rule, self.test_msg1), False)
rule = ComparisonBinOpRule('OP_EQ', VariableRule("ConnCount"),
IntegerRule(2))
self.assertEqual(flt.filter(rule, self.test_msg1), True)
rule = ComparisonBinOpRule('OP_EQ', VariableRule("ConnCount"),
IntegerRule(4))
self.assertEqual(flt.filter(rule, self.test_msg1), False)
rule = ComparisonBinOpRule('OP_NE', VariableRule("ConnCount"),
IntegerRule(2))
self.assertEqual(flt.filter(rule, self.test_msg1), False)
rule = ComparisonBinOpRule('OP_NE', VariableRule("ConnCount"),
IntegerRule(4))
self.assertEqual(flt.filter(rule, self.test_msg1), True)
rule = ComparisonBinOpRule('OP_GT', VariableRule("ConnCount"),
IntegerRule(2))
self.assertEqual(flt.filter(rule, self.test_msg1), False)
rule = ComparisonBinOpRule('OP_GT', VariableRule("ConnCount"),
IntegerRule(1))
self.assertEqual(flt.filter(rule, self.test_msg1), True)
rule = ComparisonBinOpRule('OP_GE', VariableRule("ConnCount"),
IntegerRule(2))
self.assertEqual(flt.filter(rule, self.test_msg1), True)
rule = ComparisonBinOpRule('OP_GE', VariableRule("ConnCount"),
IntegerRule(1))
self.assertEqual(flt.filter(rule, self.test_msg1), True)
rule = ComparisonBinOpRule('OP_GE', VariableRule("ConnCount"),
IntegerRule(3))
self.assertEqual(flt.filter(rule, self.test_msg1), False)
rule = ComparisonBinOpRule('OP_LT', VariableRule("ConnCount"),
IntegerRule(2))
self.assertEqual(flt.filter(rule, self.test_msg1), False)
rule = ComparisonBinOpRule('OP_LT', VariableRule("ConnCount"),
IntegerRule(3))
self.assertEqual(flt.filter(rule, self.test_msg1), True)
rule = ComparisonBinOpRule('OP_LE', VariableRule("ConnCount"),
IntegerRule(2))
self.assertEqual(flt.filter(rule, self.test_msg1), True)
rule = ComparisonBinOpRule('OP_LE', VariableRule("ConnCount"),
IntegerRule(3))
self.assertEqual(flt.filter(rule, self.test_msg1), True)
rule = ComparisonBinOpRule('OP_LE', VariableRule("ConnCount"),
IntegerRule(1))
self.assertEqual(flt.filter(rule, self.test_msg1), False)
rule = psr.parse('ID == "e214d2d9-359b-443d-993d-3cc5637107a0"')
self.assertEqual(flt.filter(rule, self.test_msg1), True)
rule = psr.parse('ID eq "e214d2d9-359b-443d-993d-3cc5637107"')
self.assertEqual(flt.filter(rule, self.test_msg1), False)
rule = psr.parse('ID != "e214d2d9-359b-443d-993d-3cc5637107a0"')
self.assertEqual(flt.filter(rule, self.test_msg1), False)
rule = psr.parse('ID ne "e214d2d9-359b-443d-993d-3cc5637107"')
self.assertEqual(flt.filter(rule, self.test_msg1), True)
rule = psr.parse('ID like "e214d2d9"')
self.assertEqual(flt.filter(rule, self.test_msg1), True)
rule = psr.parse('ID LIKE "xxxxxxxx"')
self.assertEqual(flt.filter(rule, self.test_msg1), False)
rule = psr.parse('Category in ["Phishing" , "Attempt.Login"]')
self.assertEqual(flt.filter(rule, self.test_msg1), True)
rule = psr.parse('Category IN ["Phishing" , "Spam"]')
self.assertEqual(flt.filter(rule, self.test_msg1), False)
rule = psr.parse('Category is ["Attempt.Login"]')
self.assertEqual(flt.filter(rule, self.test_msg1), True)
rule = psr.parse('Category IS ["Phishing" , "Attempt.Login"]')
self.assertEqual(flt.filter(rule, self.test_msg1), False)
rule = psr.parse('ConnCount == 2')
self.assertEqual(flt.filter(rule, self.test_msg1), True)
rule = psr.parse('ConnCount eq 4')
self.assertEqual(flt.filter(rule, self.test_msg1), False)
rule = psr.parse('ConnCount != 2')
self.assertEqual(flt.filter(rule, self.test_msg1), False)
rule = psr.parse('ConnCount ne 4')
self.assertEqual(flt.filter(rule, self.test_msg1), True)
rule = psr.parse('ConnCount > 2')
self.assertEqual(flt.filter(rule, self.test_msg1), False)
rule = psr.parse('ConnCount gt 1')
self.assertEqual(flt.filter(rule, self.test_msg1), True)
rule = psr.parse('ConnCount >= 2')
self.assertEqual(flt.filter(rule, self.test_msg1), True)
rule = psr.parse('ConnCount ge 1')
self.assertEqual(flt.filter(rule, self.test_msg1), True)
rule = psr.parse('ConnCount GE 3')
self.assertEqual(flt.filter(rule, self.test_msg1), False)
rule = psr.parse('ConnCount < 2')
self.assertEqual(flt.filter(rule, self.test_msg1), False)
rule = psr.parse('ConnCount lt 3')
self.assertEqual(flt.filter(rule, self.test_msg1), True)
rule = psr.parse('ConnCount <= 2')
self.assertEqual(flt.filter(rule, self.test_msg1), True)
rule = psr.parse('ConnCount le 3')
self.assertEqual(flt.filter(rule, self.test_msg1), True)
rule = psr.parse('ConnCount LE 1')
self.assertEqual(flt.filter(rule, self.test_msg1), False)
rule = psr.parse('ConnCounts LE 1')
self.assertEqual(flt.filter(rule, self.test_msg1), None)
def setUp(self):
self.flt = DataObjectFilter()
self.psr = PynspectFilterParser()
self.psr.build()
class TestDataObjectFilter(unittest.TestCase):
"""
Unit test class for testing the :py:mod:`pynspect.filters` module.
"""
test_msg1 = {
"ID" : "e214d2d9-359b-443d-993d-3cc5637107a0",
"WinEndTime" : "2016-06-21 11:25:01Z",
"ConnCount" : 2,
"Source" : [
{
"IP4" : [
"188.14.166.39"
]
}
],
"Format" : "IDEA0",
"WinStartTime" : "2016-06-21 11:20:01Z",
"_CESNET" : {
"StorageTime" : 1466508305
},
"Target" : [
{
"IP4" : [
"195.113.165.128/25"
],
"Port" : [
"22"
],
"Proto" : [
"tcp",
"ssh"
],
"Anonymised" : True
}
],
"Note" : "SSH login attempt",
"DetectTime" : "2016-06-21 13:08:27Z",
"Node" : [
{
"Name" : "cz.cesnet.mentat.warden_filer",
"Type" : [
"Relay"
]
},
{
"AggrWin" : "00:05:00",
"Type" : [
"Connection",
"Honeypot",
"Recon"
],
"SW" : [
"Kippo"
],
"Name" : "cz.uhk.apate.cowrie"
}
],
"Category" : [
"Attempt.Login"
]
}
def setUp(self):
self.flt = DataObjectFilter()
self.psr = PynspectFilterParser()
self.psr.build()
def test_01_basic_logical(self):
"""
Perform filtering tests with basic logical expressions.
"""
self.maxDiff = None
rule = LogicalBinOpRule('OP_AND', ConstantRule(True), ConstantRule(True))
self.assertEqual(self.flt.filter(rule, self.test_msg1), True)
rule = LogicalBinOpRule('OP_AND', ConstantRule(True), ConstantRule(False))
self.assertEqual(self.flt.filter(rule, self.test_msg1), False)
rule = LogicalBinOpRule('OP_AND', ConstantRule(False), ConstantRule(True))
self.assertEqual(self.flt.filter(rule, self.test_msg1), False)
rule = LogicalBinOpRule('OP_AND', ConstantRule(False), ConstantRule(False))
self.assertEqual(self.flt.filter(rule, self.test_msg1), False)
rule = LogicalBinOpRule('OP_OR', ConstantRule(True), ConstantRule(True))
self.assertEqual(self.flt.filter(rule, self.test_msg1), True)
rule = LogicalBinOpRule('OP_OR', ConstantRule(True), ConstantRule(False))
self.assertEqual(self.flt.filter(rule, self.test_msg1), True)
rule = LogicalBinOpRule('OP_OR', ConstantRule(False), ConstantRule(True))
self.assertEqual(self.flt.filter(rule, self.test_msg1), True)
rule = LogicalBinOpRule('OP_OR', ConstantRule(False), ConstantRule(False))
self.assertEqual(self.flt.filter(rule, self.test_msg1), False)
rule = LogicalBinOpRule('OP_XOR', ConstantRule(True), ConstantRule(True))
self.assertEqual(self.flt.filter(rule, self.test_msg1), False)
rule = LogicalBinOpRule('OP_XOR', ConstantRule(True), ConstantRule(False))
self.assertEqual(self.flt.filter(rule, self.test_msg1), True)
rule = LogicalBinOpRule('OP_XOR', ConstantRule(False), ConstantRule(True))
self.assertEqual(self.flt.filter(rule, self.test_msg1), True)
rule = LogicalBinOpRule('OP_XOR', ConstantRule(False), ConstantRule(False))
self.assertEqual(self.flt.filter(rule, self.test_msg1), False)
rule = UnaryOperationRule('OP_NOT', ConstantRule(True))
self.assertEqual(self.flt.filter(rule, self.test_msg1), False)
rule = UnaryOperationRule('OP_NOT', ConstantRule(False))
self.assertEqual(self.flt.filter(rule, self.test_msg1), True)
rule = UnaryOperationRule('OP_NOT', VariableRule("Target.Anonymised"))
self.assertEqual(self.flt.filter(rule, self.test_msg1), False)
def test_02_basic_comparison(self):
"""
Perform filtering tests with basic comparison expressions.
"""
self.maxDiff = None
rule = ComparisonBinOpRule('OP_EQ', VariableRule("ID"), ConstantRule("e214d2d9-359b-443d-993d-3cc5637107a0"))
self.assertEqual(self.flt.filter(rule, self.test_msg1), True)
rule = ComparisonBinOpRule('OP_EQ', VariableRule("ID"), ConstantRule("e214d2d9-359b-443d-993d-3cc5637107"))
self.assertEqual(self.flt.filter(rule, self.test_msg1), False)
rule = ComparisonBinOpRule('OP_NE', VariableRule("ID"), ConstantRule("e214d2d9-359b-443d-993d-3cc5637107a0"))
self.assertEqual(self.flt.filter(rule, self.test_msg1), False)
rule = ComparisonBinOpRule('OP_NE', VariableRule("ID"), ConstantRule("e214d2d9-359b-443d-993d-3cc5637107"))
self.assertEqual(self.flt.filter(rule, self.test_msg1), True)
rule = ComparisonBinOpRule('OP_LIKE', VariableRule("ID"), ConstantRule("e214d2d9"))
self.assertEqual(self.flt.filter(rule, self.test_msg1), True)
rule = ComparisonBinOpRule('OP_LIKE', VariableRule("ID"), ConstantRule("xxxxxxxx"))
self.assertEqual(self.flt.filter(rule, self.test_msg1), False)
rule = ComparisonBinOpRule('OP_IN', VariableRule("Category"), ListRule(ConstantRule("Phishing"), ListRule(ConstantRule("Attempt.Login"))))
self.assertEqual(self.flt.filter(rule, self.test_msg1), True)
rule = ComparisonBinOpRule('OP_IN', VariableRule("Category"), ListRule(ConstantRule("Phishing"), ListRule(ConstantRule("Spam"))))
self.assertEqual(self.flt.filter(rule, self.test_msg1), False)
rule = ComparisonBinOpRule('OP_IS', VariableRule("Category"), ListRule(ConstantRule("Attempt.Login")))
self.assertEqual(self.flt.filter(rule, self.test_msg1), True)
rule = ComparisonBinOpRule('OP_IS', VariableRule("Category"), ListRule(ConstantRule("Phishing"), ListRule(ConstantRule("Attempt.Login"))))
self.assertEqual(self.flt.filter(rule, self.test_msg1), False)
rule = ComparisonBinOpRule('OP_EQ', VariableRule("ConnCount"), IntegerRule(2))
self.assertEqual(self.flt.filter(rule, self.test_msg1), True)
rule = ComparisonBinOpRule('OP_EQ', VariableRule("ConnCount"), IntegerRule(4))
self.assertEqual(self.flt.filter(rule, self.test_msg1), False)
rule = ComparisonBinOpRule('OP_NE', VariableRule("ConnCount"), IntegerRule(2))
self.assertEqual(self.flt.filter(rule, self.test_msg1), False)
rule = ComparisonBinOpRule('OP_NE', VariableRule("ConnCount"), IntegerRule(4))
self.assertEqual(self.flt.filter(rule, self.test_msg1), True)
rule = ComparisonBinOpRule('OP_GT', VariableRule("ConnCount"), IntegerRule(2))
self.assertEqual(self.flt.filter(rule, self.test_msg1), False)
rule = ComparisonBinOpRule('OP_GT', VariableRule("ConnCount"), IntegerRule(1))
self.assertEqual(self.flt.filter(rule, self.test_msg1), True)
rule = ComparisonBinOpRule('OP_GE', VariableRule("ConnCount"), IntegerRule(2))
self.assertEqual(self.flt.filter(rule, self.test_msg1), True)
rule = ComparisonBinOpRule('OP_GE', VariableRule("ConnCount"), IntegerRule(1))
self.assertEqual(self.flt.filter(rule, self.test_msg1), True)
rule = ComparisonBinOpRule('OP_GE', VariableRule("ConnCount"), IntegerRule(3))
self.assertEqual(self.flt.filter(rule, self.test_msg1), False)
rule = ComparisonBinOpRule('OP_LT', VariableRule("ConnCount"), IntegerRule(2))
self.assertEqual(self.flt.filter(rule, self.test_msg1), False)
rule = ComparisonBinOpRule('OP_LT', VariableRule("ConnCount"), IntegerRule(3))
self.assertEqual(self.flt.filter(rule, self.test_msg1), True)
rule = ComparisonBinOpRule('OP_LE', VariableRule("ConnCount"), IntegerRule(2))
self.assertEqual(self.flt.filter(rule, self.test_msg1), True)
rule = ComparisonBinOpRule('OP_LE', VariableRule("ConnCount"), IntegerRule(3))
self.assertEqual(self.flt.filter(rule, self.test_msg1), True)
rule = ComparisonBinOpRule('OP_LE', VariableRule("ConnCount"), IntegerRule(1))
self.assertEqual(self.flt.filter(rule, self.test_msg1), False)
def test_03_parsed_comparison(self):
"""
Perform filtering tests with parsed comparison expressions.
"""
self.maxDiff = None
rule = self.psr.parse('ID == "e214d2d9-359b-443d-993d-3cc5637107a0"')
self.assertEqual(self.flt.filter(rule, self.test_msg1), True)
rule = self.psr.parse('ID eq "e214d2d9-359b-443d-993d-3cc5637107"')
self.assertEqual(self.flt.filter(rule, self.test_msg1), False)
rule = self.psr.parse('ID != "e214d2d9-359b-443d-993d-3cc5637107a0"')
self.assertEqual(self.flt.filter(rule, self.test_msg1), False)
rule = self.psr.parse('ID ne "e214d2d9-359b-443d-993d-3cc5637107"')
self.assertEqual(self.flt.filter(rule, self.test_msg1), True)
rule = self.psr.parse('ID like "e214d2d9"')
self.assertEqual(self.flt.filter(rule, self.test_msg1), True)
rule = self.psr.parse('ID LIKE "xxxxxxxx"')
self.assertEqual(self.flt.filter(rule, self.test_msg1), False)
rule = self.psr.parse('Category in ["Phishing" , "Attempt.Login"]')
self.assertEqual(self.flt.filter(rule, self.test_msg1), True)
rule = self.psr.parse('Category IN ["Phishing" , "Spam"]')
self.assertEqual(self.flt.filter(rule, self.test_msg1), False)
rule = self.psr.parse('Category is ["Attempt.Login"]')
self.assertEqual(self.flt.filter(rule, self.test_msg1), True)
rule = self.psr.parse('Category IS ["Phishing" , "Attempt.Login"]')
self.assertEqual(self.flt.filter(rule, self.test_msg1), False)
rule = self.psr.parse('ConnCount == 2')
self.assertEqual(self.flt.filter(rule, self.test_msg1), True)
rule = self.psr.parse('ConnCount eq 4')
self.assertEqual(self.flt.filter(rule, self.test_msg1), False)
rule = self.psr.parse('ConnCount != 2')
self.assertEqual(self.flt.filter(rule, self.test_msg1), False)
rule = self.psr.parse('ConnCount ne 4')
self.assertEqual(self.flt.filter(rule, self.test_msg1), True)
rule = self.psr.parse('ConnCount > 2')
self.assertEqual(self.flt.filter(rule, self.test_msg1), False)
rule = self.psr.parse('ConnCount gt 1')
self.assertEqual(self.flt.filter(rule, self.test_msg1), True)
rule = self.psr.parse('ConnCount >= 2')
self.assertEqual(self.flt.filter(rule, self.test_msg1), True)
rule = self.psr.parse('ConnCount ge 1')
self.assertEqual(self.flt.filter(rule, self.test_msg1), True)
rule = self.psr.parse('ConnCount GE 3')
self.assertEqual(self.flt.filter(rule, self.test_msg1), False)
rule = self.psr.parse('ConnCount < 2')
self.assertEqual(self.flt.filter(rule, self.test_msg1), False)
rule = self.psr.parse('ConnCount lt 3')
self.assertEqual(self.flt.filter(rule, self.test_msg1), True)
rule = self.psr.parse('ConnCount <= 2')
self.assertEqual(self.flt.filter(rule, self.test_msg1), True)
rule = self.psr.parse('ConnCount le 3')
self.assertEqual(self.flt.filter(rule, self.test_msg1), True)
rule = self.psr.parse('ConnCount LE 1')
self.assertEqual(self.flt.filter(rule, self.test_msg1), False)
rule = self.psr.parse('ConnCounts LE 1')
self.assertEqual(self.flt.filter(rule, self.test_msg1), None)
def test_04_basic_math(self):
"""
Perform filtering tests with basic math expressions.
"""
self.maxDiff = None
rule = MathBinOpRule('OP_PLUS', VariableRule("ConnCount"), IntegerRule(1))
self.assertEqual(self.flt.filter(rule, self.test_msg1), 3)
rule = MathBinOpRule('OP_MINUS', VariableRule("ConnCount"), IntegerRule(1))
self.assertEqual(self.flt.filter(rule, self.test_msg1), 1)
rule = MathBinOpRule('OP_TIMES', VariableRule("ConnCount"), IntegerRule(5))
self.assertEqual(self.flt.filter(rule, self.test_msg1), 10)
rule = MathBinOpRule('OP_DIVIDE', VariableRule("ConnCount"), IntegerRule(2))
self.assertEqual(self.flt.filter(rule, self.test_msg1), 1)
rule = MathBinOpRule('OP_MODULO', VariableRule("ConnCount"), IntegerRule(2))
self.assertEqual(self.flt.filter(rule, self.test_msg1), 0)
def test_05_parsed_math(self):
"""
Perform filtering tests with parsed math expressions.
"""
self.maxDiff = None
rule = self.psr.parse('ConnCount + 1')
self.assertEqual(self.flt.filter(rule, self.test_msg1), 3)
rule = self.psr.parse('ConnCount - 1')
self.assertEqual(self.flt.filter(rule, self.test_msg1), 1)
rule = self.psr.parse('ConnCount * 5')
self.assertEqual(self.flt.filter(rule, self.test_msg1), 10)
rule = self.psr.parse('ConnCount / 2')
self.assertEqual(self.flt.filter(rule, self.test_msg1), 1)
rule = self.psr.parse('ConnCount % 2')
self.assertEqual(self.flt.filter(rule, self.test_msg1), 0)
def test_06_advanced_filters(self):
"""
Perform advanced filtering tests.
"""
self.maxDiff = None
rule = self.psr.parse('(ConnCount + 10) > 11')
self.assertEqual(repr(rule), "COMPBINOP(MATHBINOP(VARIABLE('ConnCount') OP_PLUS INTEGER(10)) OP_GT INTEGER(11))")
self.assertEqual(self.flt.filter(rule, self.test_msg1), True)
rule = self.psr.parse('((ConnCount + 3) < 5) or ((ConnCount + 10) > 11)')
self.assertEqual(repr(rule), "LOGBINOP(COMPBINOP(MATHBINOP(VARIABLE('ConnCount') OP_PLUS INTEGER(3)) OP_LT INTEGER(5)) OP_OR COMPBINOP(MATHBINOP(VARIABLE('ConnCount') OP_PLUS INTEGER(10)) OP_GT INTEGER(11)))")
self.assertEqual(self.flt.filter(rule, self.test_msg1), True)
rule = self.psr.parse('1')
self.assertEqual(repr(rule), "INTEGER(1)")
self.assertEqual(self.flt.filter(rule, self.test_msg1), True)
def test_07_non_existent_nodes(self):
"""
Perform advanced filtering tests.
"""
self.maxDiff = None
rule = self.psr.parse('(ConnCounts + 10) > 11')
self.assertEqual(self.flt.filter(rule, self.test_msg1), None)
rule = self.psr.parse('ConnCount > ConnCounts')
self.assertEqual(self.flt.filter(rule, self.test_msg1), None)
rule = self.psr.parse('DetectTime < InspectionTime')
self.assertEqual(self.flt.filter(rule, self.test_msg1), None)
def test_08_functions(self):
"""
Perform advanced filtering tests.
"""
self.maxDiff = None
rule = self.psr.parse('(size(Node.Type) > 2)')
self.assertEqual(repr(rule), "COMPBINOP(FUNCTION(size(VARIABLE('Node.Type'),)) OP_GT INTEGER(2))")
self.assertEqual(self.flt.filter(rule, self.test_msg1), True)
rule = self.psr.parse('(size(Source.IP4) > 4)')
self.assertEqual(repr(rule), "COMPBINOP(FUNCTION(size(VARIABLE('Source.IP4'),)) OP_GT INTEGER(4))")
self.assertEqual(self.flt.filter(rule, self.test_msg1), False)
rule = self.psr.parse('(strlen(ID) > 8)')
self.assertEqual(repr(rule), "COMPBINOP(FUNCTION(strlen(VARIABLE('ID'),)) OP_GT INTEGER(8))")
self.assertEqual(self.flt.filter(rule, self.test_msg1), True)
rule = self.psr.parse('(strlen(ID) < 8)')
self.assertEqual(repr(rule), "COMPBINOP(FUNCTION(strlen(VARIABLE('ID'),)) OP_LT INTEGER(8))")
self.assertEqual(self.flt.filter(rule, self.test_msg1), False)
rule = self.psr.parse('(strlen(Description) > 8)')
self.assertEqual(repr(rule), "COMPBINOP(FUNCTION(strlen(VARIABLE('Description'),)) OP_GT INTEGER(8))")
self.assertEqual(self.flt.filter(rule, self.test_msg1), None)
rule = self.psr.parse('(strlen(Description) < 8)')
self.assertEqual(repr(rule), "COMPBINOP(FUNCTION(strlen(VARIABLE('Description'),)) OP_LT INTEGER(8))")
self.assertEqual(self.flt.filter(rule, self.test_msg1), None)
rule = self.psr.parse('(strlen(Node.Name) > 20)')
self.assertEqual(repr(rule), "COMPBINOP(FUNCTION(strlen(VARIABLE('Node.Name'),)) OP_GT INTEGER(20))")
self.assertEqual(self.flt.filter(rule, self.test_msg1), True)
rule = self.psr.parse('(strlen(Node.Name) > 30)')
self.assertEqual(repr(rule), "COMPBINOP(FUNCTION(strlen(VARIABLE('Node.Name'),)) OP_GT INTEGER(30))")
self.assertEqual(self.flt.filter(rule, self.test_msg1), False)
rule = self.psr.parse('(strlen(Node.Name) < 20)')
self.assertEqual(repr(rule), "COMPBINOP(FUNCTION(strlen(VARIABLE('Node.Name'),)) OP_LT INTEGER(20))")
self.assertEqual(self.flt.filter(rule, self.test_msg1), True)
rule = self.psr.parse('(strlen(Node.Name) < 10)')
self.assertEqual(repr(rule), "COMPBINOP(FUNCTION(strlen(VARIABLE('Node.Name'),)) OP_LT INTEGER(10))")
self.assertEqual(self.flt.filter(rule, self.test_msg1), False)
rule = self.psr.parse('(time() > 500.12)')
self.assertEqual(repr(rule), "COMPBINOP(FUNCTION(time()) OP_GT FLOAT(500.12))")
self.assertEqual(self.flt.filter(rule, self.test_msg1), True)
def setUp(self):
self.flt = DataObjectFilter()
self.psr = PynspectFilterParser()
self.psr.build()
self.cpl = IDEAFilterCompiler()
class TestDataObjectFilterInspector(unittest.TestCase):
"""
Unit test class for testing the :py:mod:`pynspect.filters` module.
"""
def setUp(self):
self.flt = DataObjectFilter()
self.psr = PynspectFilterParser()
self.psr.build()
self.cpl = IDEAFilterCompiler()
def build_rule(self, rule_str):
"""
Build and compile rule tree from given rule string.
"""
rule = self.psr.parse(rule_str)
rule = self.cpl.compile(rule)
return rule
def test_01_inspector_filters(self):
"""
Perform tests of filters currently used in mentat-inspector.py.
"""
self.maxDiff = None
inspection_rules = [{
"filter":
"Category in ['Attempt.Login'] and Target.Port in [3389]",
"str":
'((Category OP_IN ["Attempt.Login"]) OP_AND (Target.Port OP_IN [3389]))',
"tests": [[{
"Format":
"IDEA0",
"ID":
"e214d2d9-359b-443d-993d-3cc5637107a0",
"Source": [{
"IP4": ["188.14.166.39"]
}],
"Target": [{
"IP4": ["195.113.165.128/25"],
"Port": ["3389"],
"Proto": ["tcp", "ssh"]
}],
"Note":
"SSH login attempt",
"DetectTime":
"2016-06-21 13:08:27Z",
"Node": [{
"Type": ["Connection", "Honeypot"],
"SW": ["Kippo"],
"Name": "cz.uhk.apate.cowrie"
}],
"Category": ["Attempt.Login"]
}, True],
[{
"Format":
"IDEA0",
"ID":
"e214d2d9-359b-443d-993d-3cc5637107a0",
"Source": [{
"IP4": ["188.14.166.39"]
}],
"Target": [{
"IP4": ["195.113.165.128/25"],
"Port": ["338"],
"Proto": ["tcp", "ssh"]
}],
"Note":
"SSH login attempt",
"DetectTime":
"2016-06-21 13:08:27Z",
"Node": [{
"Type": ["Connection", "Honeypot"],
"SW": ["Kippo"],
"Name": "cz.uhk.apate.cowrie"
}],
"Category": ["Attempt.Login"]
}, False]],
}, {
"filter":
"Category in ['Attempt.Login'] and (Target.Proto in ['telnet'] or Source.Proto in ['telnet'] or Target.Port in [23])",
"str":
'((Category OP_IN ["Attempt.Login"]) OP_AND ((Target.Proto OP_IN ["telnet"]) OP_OR ((Source.Proto OP_IN ["telnet"]) OP_OR (Target.Port OP_IN [23]))))'
}, {
"filter":
"Category in ['Attempt.Login'] and (Target.Proto in ['ssh'] or Source.Proto in ['ssh'] or Target.Port in [22])",
"str":
'((Category OP_IN ["Attempt.Login"]) OP_AND ((Target.Proto OP_IN ["ssh"]) OP_OR ((Source.Proto OP_IN ["ssh"]) OP_OR (Target.Port OP_IN [22]))))'
}, {
"filter":
"Category in ['Attempt.Login'] and (Target.Proto in ['sip', 'sip-tls'] or Source.Proto in ['sip', 'sip-tls'] or Target.Port in [5060])",
"str":
'((Category OP_IN ["Attempt.Login"]) OP_AND ((Target.Proto OP_IN ["sip", "sip-tls"]) OP_OR ((Source.Proto OP_IN ["sip", "sip-tls"]) OP_OR (Target.Port OP_IN [5060]))))'
}, {
"filter":
"Category in ['Attempt.Exploit'] and (Target.Proto in ['sip', 'sip-tls'] or Source.Proto in ['sip', 'sip-tls'] or Target.Port in [5060])",
"str":
'((Category OP_IN ["Attempt.Exploit"]) OP_AND ((Target.Proto OP_IN ["sip", "sip-tls"]) OP_OR ((Source.Proto OP_IN ["sip", "sip-tls"]) OP_OR (Target.Port OP_IN [5060]))))'
}, {
"filter":
"Category in ['Attempt.Exploit'] and Target.Port in [23]",
"str":
'((Category OP_IN ["Attempt.Exploit"]) OP_AND (Target.Port OP_IN [23]))'
}, {
"filter":
"Category in ['Attempt.Exploit'] and (Target.Port in [80, 443] or Source.Proto in ['http', 'https', 'http-alt'] or Target.Proto in ['http', 'https', 'http-alt'])",
"str":
'((Category OP_IN ["Attempt.Exploit"]) OP_AND ((Target.Port OP_IN [80, 443]) OP_OR ((Source.Proto OP_IN ["http", "https", "http-alt"]) OP_OR (Target.Proto OP_IN ["http", "https", "http-alt"]))))'
}, {
"filter":
"Category in ['Attempt.Exploit'] and (Target.Port in [3306] or Source.Proto in ['mysql'] or Target.Proto in ['mysql'])",
"str":
'((Category OP_IN ["Attempt.Exploit"]) OP_AND ((Target.Port OP_IN [3306]) OP_OR ((Source.Proto OP_IN ["mysql"]) OP_OR (Target.Proto OP_IN ["mysql"]))))'
}, {
"filter":
"Category in ['Attempt.Exploit'] and (Target.Port in [445] or Source.Proto in ['microsoft-ds', 'smb'] or Target.Proto in ['microsoft-ds', 'smb'])",
"str":
'((Category OP_IN ["Attempt.Exploit"]) OP_AND ((Target.Port OP_IN [445]) OP_OR ((Source.Proto OP_IN ["microsoft-ds", "smb"]) OP_OR (Target.Proto OP_IN ["microsoft-ds", "smb"]))))'
}, {
"filter":
"Category in ['Attempt.Exploit'] and (Target.Port in [135] or Source.Proto in ['loc-srv', 'epmap'] or Target.Proto in ['loc-srv', 'epmap'])",
"str":
'((Category OP_IN ["Attempt.Exploit"]) OP_AND ((Target.Port OP_IN [135]) OP_OR ((Source.Proto OP_IN ["loc-srv", "epmap"]) OP_OR (Target.Proto OP_IN ["loc-srv", "epmap"]))))'
}, {
"filter":
"Category in ['Attempt.Exploit'] and (Target.Port in [1900] or Source.Proto in ['upnp', 'ssdp'] or Target.Proto in ['upnp', 'ssdp'])",
"str":
'((Category OP_IN ["Attempt.Exploit"]) OP_AND ((Target.Port OP_IN [1900]) OP_OR ((Source.Proto OP_IN ["upnp", "ssdp"]) OP_OR (Target.Proto OP_IN ["upnp", "ssdp"]))))'
}, {
"filter":
"Category in ['Attempt.Exploit'] and (Target.Port in [20, 21, 989, 990] or Source.Proto in ['ftp', 'ftp-data', 'ftps', 'ftps-data'] or Target.Proto in ['ftp', 'ftp-data', 'ftps', 'ftps-data'])",
"str":
'((Category OP_IN ["Attempt.Exploit"]) OP_AND ((Target.Port OP_IN [20, 21, 989, 990]) OP_OR ((Source.Proto OP_IN ["ftp", "ftp-data", "ftps", "ftps-data"]) OP_OR (Target.Proto OP_IN ["ftp", "ftp-data", "ftps", "ftps-data"]))))'
}, {
"filter":
"Category in ['Attempt.Exploit'] and (Target.Port in [1433, 1434] or Source.Proto in ['ms-sql-s', 'ms-sql-m'] or Target.Proto in ['ms-sql-s', 'ms-sql-m'])",
"str":
'((Category OP_IN ["Attempt.Exploit"]) OP_AND ((Target.Port OP_IN [1433, 1434]) OP_OR ((Source.Proto OP_IN ["ms-sql-s", "ms-sql-m"]) OP_OR (Target.Proto OP_IN ["ms-sql-s", "ms-sql-m"]))))'
}, {
"filter":
"Category in ['Attempt.Exploit'] and (Target.Port in [42] or Source.Proto in ['nameserver'] or Target.Proto in ['nameserver'])",
"str":
'((Category OP_IN ["Attempt.Exploit"]) OP_AND ((Target.Port OP_IN [42]) OP_OR ((Source.Proto OP_IN ["nameserver"]) OP_OR (Target.Proto OP_IN ["nameserver"]))))'
}, {
"filter":
"Category in ['Attempt.Exploit'] and Node.SW in ['Dionaea']",
"str":
'((Category OP_IN ["Attempt.Exploit"]) OP_AND (Node.SW OP_IN ["Dionaea"]))'
}, {
"filter":
"Category in ['Availability.DoS', 'Availability.DDoS'] and (Target.Proto in ['dns', 'domain'] or Source.Proto in ['dns', 'domain'] or Target.Port in [53] or Source.Port in [53])",
"str":
'((Category OP_IN ["Availability.DoS", "Availability.DDoS"]) OP_AND ((Target.Proto OP_IN ["dns", "domain"]) OP_OR ((Source.Proto OP_IN ["dns", "domain"]) OP_OR ((Target.Port OP_IN [53]) OP_OR (Source.Port OP_IN [53])))))'
}, {
"filter":
"Category in ['Availability.DDoS'] and Node.Type in ['Flow'] and Node.Type in ['Statistical']",
"str":
'((Category OP_IN ["Availability.DDoS"]) OP_AND ((Node.Type OP_IN ["Flow"]) OP_AND (Node.Type OP_IN ["Statistical"])))'
}, {
"filter":
"Category in ['Abusive.Spam'] and Node.SW in ['UCEPROT']",
"str":
'((Category OP_IN ["Abusive.Spam"]) OP_AND (Node.SW OP_IN ["UCEPROT"]))'
}, {
"filter":
"Category in ['Abusive.Spam'] and Node.SW in ['Fail2Ban', 'IntelMQ']",
"str":
'((Category OP_IN ["Abusive.Spam"]) OP_AND (Node.SW OP_IN ["Fail2Ban", "IntelMQ"]))'
}, {
"filter":
"Category in ['Vulnerable.Config'] and (Source.Proto in ['qotd'] or Source.Port in [17])",
"str":
'((Category OP_IN ["Vulnerable.Config"]) OP_AND ((Source.Proto OP_IN ["qotd"]) OP_OR (Source.Port OP_IN [17])))'
}, {
"filter":
"Category in ['Vulnerable.Config'] and Source.Proto in ['ssdp']",
"str":
'((Category OP_IN ["Vulnerable.Config"]) OP_AND (Source.Proto OP_IN ["ssdp"]))'
}, {
"filter":
"Category in ['Vulnerable.Config'] and (Source.Proto in ['ntp'] or Source.Port in [123])",
"str":
'((Category OP_IN ["Vulnerable.Config"]) OP_AND ((Source.Proto OP_IN ["ntp"]) OP_OR (Source.Port OP_IN [123])))'
}, {
"filter":
"Category in ['Vulnerable.Config'] and (Source.Proto in ['domain'] or Source.Port in [53])",
"str":
'((Category OP_IN ["Vulnerable.Config"]) OP_AND ((Source.Proto OP_IN ["domain"]) OP_OR (Source.Port OP_IN [53])))'
}, {
"filter":
"Category in ['Vulnerable.Config'] and (Source.Proto in ['netbios-ns'] or Source.Port in [137])",
"str":
'((Category OP_IN ["Vulnerable.Config"]) OP_AND ((Source.Proto OP_IN ["netbios-ns"]) OP_OR (Source.Port OP_IN [137])))'
}, {
"filter":
"Category in ['Vulnerable.Config'] and (Source.Proto in ['ipmi'] or Source.Port in [623])",
"str":
'((Category OP_IN ["Vulnerable.Config"]) OP_AND ((Source.Proto OP_IN ["ipmi"]) OP_OR (Source.Port OP_IN [623])))'
}, {
"filter":
"Category in ['Vulnerable.Config'] and (Source.Proto in ['chargen'] or Source.Port in [19])",
"str":
'((Category OP_IN ["Vulnerable.Config"]) OP_AND ((Source.Proto OP_IN ["chargen"]) OP_OR (Source.Port OP_IN [19])))'
}, {
"filter": "Category in ['Anomaly.Traffic']",
"str": '(Category OP_IN ["Anomaly.Traffic"])'
}, {
"filter":
"Category in ['Anomaly.Connection'] and Source.Type in ['Booter']",
"str":
'((Category OP_IN ["Anomaly.Connection"]) OP_AND (Source.Type OP_IN ["Booter"]))'
}, {
"filter":
"Category in ['Intrusion.Botnet'] and Source.Type in ['Botnet']",
"str":
'((Category OP_IN ["Intrusion.Botnet"]) OP_AND (Source.Type OP_IN ["Botnet"]))'
}, {
"filter": "Category in ['Recon.Scanning']",
"str": '(Category OP_IN ["Recon.Scanning"])'
}]
for insr in inspection_rules:
rule = self.build_rule(insr['filter'])
self.assertEqual(str(rule), insr['str'])
if 'tests' in insr:
for itemt in insr['tests']:
msg_idea = lite.Idea(itemt[0])
self.assertEqual(
[insr['filter'],
self.flt.filter(rule, msg_idea)],
[insr['filter'], itemt[1]])