def setUp(self):
self.flt = DataObjectFilter()
self.psr = PynspectFilterParser()
self.psr.build()
self.cpl = IDEAFilterCompiler()
self.msg_idea = lite.Idea(self.test_msg1)
def test_06_shortcuts(self):
"""
Perform tests of shortcut methods.
"""
self.maxDiff = None
# Let the shortcut method initialize everything.
flt = DataObjectFilter(
parser = PynspectFilterParser,
compiler = IDEAFilterCompiler
)
rule = flt.prepare('(Source.IP4 == 188.14.166.39)')
self.assertEqual(repr(rule), "COMPBINOP(VARIABLE('Source.IP4') OP_EQ IPV4(IP4('188.14.166.39')))")
self.assertEqual(self.check_rule(rule), True)
# Create parser and compiler instances by hand, but register them into filter.
cpl = IDEAFilterCompiler()
psr = PynspectFilterParser()
psr.build()
flt = DataObjectFilter(
parser = psr,
compiler = cpl
)
rule = flt.prepare('(Source.IP4 == 188.14.166.39)')
self.assertEqual(repr(rule), "COMPBINOP(VARIABLE('Source.IP4') OP_EQ IPV4(IP4('188.14.166.39')))")
self.assertEqual(self.check_rule(rule), True)
def __init__(self, rule, actions, addrGroups, parser=None, compiler=None):
# Check is we got parser instance
if parser is None:
self.parser = PynspectFilterParser()
self.parser.build()
else:
self.parser = parser
# Check is we got compiler instance
if compiler is None:
self.compiler = IDEAFilterCompiler()
else:
self.compiler = compiler
# Instantiate filter
self.__filter = DataObjectFilter()
# Store rule condition in raw form
self.__conditionRaw = rule["condition"]
if not self.__matchvar(rule["condition"], addrGroups):
self.__condition = self.__conditionRaw
# Set inner rule ID
self.id = rule["id"]
if (self.__condition != None):
self.parseRule()
self.__actions = list()
self.__elseactions = list()
# Associate actions
if "actions" in rule:
for actionId in rule["actions"]:
try:
logger.debug("Rule %s inserting %s", self.id, actionId)
self.__actions.append(actions[actionId])
except KeyError as e:
raise Exception("Missing action with ID " + str(e))
# Associate elseactions
if "elseactions" in rule:
for actionId in rule["elseactions"]:
try:
self.__elseactions.append(actions[actionId])
except KeyError as e:
raise Exception("Missing elseaction with ID " + str(e))
def test_03_idea_time_compilations(self):
"""
Perform IDEA datetime compilation tests.
"""
self.maxDiff = None
cpl = IDEAFilterCompiler()
psr = PynspectFilterParser()
psr.build()
rule = psr.parse('(DetectTime == "2016-06-21T13:08:27Z")')
self.assertEqual(repr(rule), "COMPBINOP(VARIABLE('DetectTime') OP_EQ CONSTANT('2016-06-21T13:08:27Z'))")
res = cpl.compile(rule)
self.assertEqual(repr(res), "COMPBINOP(VARIABLE('DetectTime') OP_EQ DATETIME(datetime.datetime(2016, 6, 21, 13, 8, 27)))")
rule = psr.parse('(CreateTime == "2016-06-21T13:08:27Z")')
self.assertEqual(repr(rule), "COMPBINOP(VARIABLE('CreateTime') OP_EQ CONSTANT('2016-06-21T13:08:27Z'))")
res = cpl.compile(rule)
self.assertEqual(repr(res), "COMPBINOP(VARIABLE('CreateTime') OP_EQ DATETIME(datetime.datetime(2016, 6, 21, 13, 8, 27)))")
rule = psr.parse('(EventTime == "2016-06-21T13:08:27Z")')
self.assertEqual(repr(rule), "COMPBINOP(VARIABLE('EventTime') OP_EQ CONSTANT('2016-06-21T13:08:27Z'))")
res = cpl.compile(rule)
self.assertEqual(repr(res), "COMPBINOP(VARIABLE('EventTime') OP_EQ DATETIME(datetime.datetime(2016, 6, 21, 13, 8, 27)))")
rule = psr.parse('(CeaseTime == "2016-06-21T13:08:27Z")')
self.assertEqual(repr(rule), "COMPBINOP(VARIABLE('CeaseTime') OP_EQ CONSTANT('2016-06-21T13:08:27Z'))")
res = cpl.compile(rule)
self.assertEqual(repr(res), "COMPBINOP(VARIABLE('CeaseTime') OP_EQ DATETIME(datetime.datetime(2016, 6, 21, 13, 8, 27)))")
rule = psr.parse('(WinStartTime == "2016-06-21T13:08:27Z")')
self.assertEqual(repr(rule), "COMPBINOP(VARIABLE('WinStartTime') OP_EQ CONSTANT('2016-06-21T13:08:27Z'))")
res = cpl.compile(rule)
self.assertEqual(repr(res), "COMPBINOP(VARIABLE('WinStartTime') OP_EQ DATETIME(datetime.datetime(2016, 6, 21, 13, 8, 27)))")
rule = psr.parse('(WinEndTime == "2016-06-21T13:08:27Z")')
self.assertEqual(repr(rule), "COMPBINOP(VARIABLE('WinEndTime') OP_EQ CONSTANT('2016-06-21T13:08:27Z'))")
res = cpl.compile(rule)
self.assertEqual(repr(res), "COMPBINOP(VARIABLE('WinEndTime') OP_EQ DATETIME(datetime.datetime(2016, 6, 21, 13, 8, 27)))")
def test_05_idea_func_compilations(self):
"""
Perform IDEA function compilation tests.
"""
self.maxDiff = None
cpl = IDEAFilterCompiler()
psr = PynspectFilterParser()
psr.build()
rule = psr.parse('(DetectTime < utcnow())')
self.assertEqual(repr(rule), "COMPBINOP(VARIABLE('DetectTime') OP_LT FUNCTION(utcnow()))")
res = cpl.compile(rule)
self.assertEqual(repr(res), "COMPBINOP(VARIABLE('DetectTime') OP_LT FUNCTION(utcnow()))")
rule = psr.parse('(DetectTime < (utcnow() - 3600))')
self.assertEqual(repr(rule), "COMPBINOP(VARIABLE('DetectTime') OP_LT MATHBINOP(FUNCTION(utcnow()) OP_MINUS INTEGER(3600)))")
res = cpl.compile(rule)
self.assertEqual(repr(res), "COMPBINOP(VARIABLE('DetectTime') OP_LT MATHBINOP(FUNCTION(utcnow()) OP_MINUS TIMEDELTA(datetime.timedelta(0, 3600))))")
rule = psr.parse('(DetectTime + 3600) > utcnow()')
self.assertEqual(repr(rule), "COMPBINOP(MATHBINOP(VARIABLE('DetectTime') OP_PLUS INTEGER(3600)) OP_GT FUNCTION(utcnow()))")
res = cpl.compile(rule)
self.assertEqual(repr(res), "COMPBINOP(MATHBINOP(VARIABLE('DetectTime') OP_PLUS TIMEDELTA(datetime.timedelta(0, 3600))) OP_GT FUNCTION(utcnow()))")
class Rule():
def __init__(self, rule, actions, addrGroups, parser=None, compiler=None):
# Check is we got parser instance
if parser is None:
self.parser = PynspectFilterParser()
self.parser.build()
else:
self.parser = parser
# Check is we got compiler instance
if compiler is None:
self.compiler = IDEAFilterCompiler()
else:
self.compiler = compiler
# Instantiate filter
self.__filter = DataObjectFilter()
# Store rule condition in raw form
self.__conditionRaw = rule["condition"]
if not self.__matchvar(rule["condition"], addrGroups):
self.__condition = self.__conditionRaw
# Set inner rule ID
self.id = rule["id"]
if (self.__condition != None):
self.parseRule()
self.__actions = list()
self.__elseactions = list()
# Associate actions
if "actions" in rule:
for actionId in rule["actions"]:
try:
logger.debug("Rule %s inserting %s", self.id, actionId)
self.__actions.append(actions[actionId])
except KeyError as e:
raise Exception("Missing action with ID " + str(e))
# Associate elseactions
if "elseactions" in rule:
for actionId in rule["elseactions"]:
try:
self.__elseactions.append(actions[actionId])
except KeyError as e:
raise Exception("Missing elseaction with ID " + str(e))
def parseRule(self):
cond = str(self.__condition).lower()
if cond in ["none", "null", "true"]:
self.__condition = True
elif cond == "false":
self.__condition = False
else:
try:
self.__condition = self.parser.parse(self.__condition)
self.__condition = self.compiler.compile(self.__condition)
except Exception as e:
print(
"Error while parsing condition: {0}\nOriginal exception: {1}"
.format(self.__condition, e))
def filter(self, record):
"""
Filter given record based on rule's condition
@note If the rule's condition is empty, record is always matched.
@return Boolean
"""
# The message must be converted via idea module
if not isinstance(record, lite.Idea):
logger.info("Converting message to IDEA")
record = lite.Idea(record)
logger.debug(record)
logger.debug(self.__condition)
if self.__condition == None or self.__condition == True:
# Rule condition is empty (tautology) - should always match the record
res = True
elif self.__condition == False:
res = False
else:
# Match the record with non-empty rule's condition
res = self.__filter.filter(self.__condition, record)
logger.debug("RESULT: %s", res)
return res
def actions(self, record):
for action in self.__actions:
action.run(record)
def elseactions(self, record):
for action in self.__elseactions:
action.run(record)
def __repr__(self):
return self.__conditionRaw
def __str__(self):
actions = []
for i in self.__actions:
actions.append("{0} ({1})".format(i.actionId, i.actionType))
elseactions = []
for i in self.__elseactions:
elseactions.append("{0} ({1})".format(i.actionId, i.actionType))
return "{0}: {1}\n{2}{3}".format(
self.id,
" ".join([i.strip()
for i in str(self.__conditionRaw).split("\n")]),
"\tActions: " + (", ".join(actions)) + "\n" if actions else "",
"\tElse Actions: " + (", ".join(elseactions)) +
"\n" if elseactions else "")
def rule(self):
return str(self.__condition)
def __matchvar(self, rule, addrGroups):
"""
Since pyncspect doesn't support variables yet we had to provide
a solution where every address group's name is matched against
the rule's condition and eventually replaced with address group's values
"""
matched = False
"""
Tautology - empty rule should always match
Don't try to match or replace any address group
"""
if rule == None or isinstance(rule, bool):
return False
for key in addrGroups:
if key in rule:
rule = re.sub(r"\b{0}\b".format(re.escape(key)),
addrGroups[key].iplist(), rule)
matched = True
self.__condition = rule
return matched
def test_04_idea_ip_compilations(self):
"""
Perform IDEA IP address compilation tests.
"""
self.maxDiff = None
cpl = IDEAFilterCompiler()
psr = PynspectFilterParser()
psr.build()
rule = psr.parse('(Source.IP4 == "192.168.1.1")')
self.assertEqual(repr(rule), "COMPBINOP(VARIABLE('Source.IP4') OP_EQ CONSTANT('192.168.1.1'))")
res = cpl.compile(rule)
self.assertEqual(repr(res), "COMPBINOP(VARIABLE('Source.IP4') OP_EQ IPV4(IP4('192.168.1.1')))")
rule = psr.parse('(Target.IP4 == "192.168.1.1")')
self.assertEqual(repr(rule), "COMPBINOP(VARIABLE('Target.IP4') OP_EQ CONSTANT('192.168.1.1'))")
res = cpl.compile(rule)
self.assertEqual(repr(res), "COMPBINOP(VARIABLE('Target.IP4') OP_EQ IPV4(IP4('192.168.1.1')))")
rule = psr.parse('(Source.IP6 == "::1")')
self.assertEqual(repr(rule), "COMPBINOP(VARIABLE('Source.IP6') OP_EQ CONSTANT('::1'))")
res = cpl.compile(rule)
self.assertEqual(repr(res), "COMPBINOP(VARIABLE('Source.IP6') OP_EQ IPV6(IP6('::1')))")
rule = psr.parse('(Target.IP6 == "::1")')
self.assertEqual(repr(rule), "COMPBINOP(VARIABLE('Target.IP6') OP_EQ CONSTANT('::1'))")
res = cpl.compile(rule)
self.assertEqual(repr(res), "COMPBINOP(VARIABLE('Target.IP6') OP_EQ IPV6(IP6('::1')))")
rule = psr.parse('(Source.IP4 IN ["192.168.1.1","192.168.1.2"])')
self.assertEqual(repr(rule), "COMPBINOP(VARIABLE('Source.IP4') OP_IN LIST(CONSTANT('192.168.1.1'), CONSTANT('192.168.1.2')))")
res = cpl.compile(rule)
self.assertEqual(repr(res), "COMPBINOP(VARIABLE('Source.IP4') OP_IN IPLIST(IPV4(IP4('192.168.1.1')), IPV4(IP4('192.168.1.2'))))")
rule = psr.parse('(Target.IP4 IN ["192.168.1.1","192.168.1.2"])')
self.assertEqual(repr(rule), "COMPBINOP(VARIABLE('Target.IP4') OP_IN LIST(CONSTANT('192.168.1.1'), CONSTANT('192.168.1.2')))")
res = cpl.compile(rule)
self.assertEqual(repr(res), "COMPBINOP(VARIABLE('Target.IP4') OP_IN IPLIST(IPV4(IP4('192.168.1.1')), IPV4(IP4('192.168.1.2'))))")
rule = psr.parse('(Source.IP6 IN ["::1","::2"])')
self.assertEqual(repr(rule), "COMPBINOP(VARIABLE('Source.IP6') OP_IN LIST(CONSTANT('::1'), CONSTANT('::2')))")
res = cpl.compile(rule)
self.assertEqual(repr(res), "COMPBINOP(VARIABLE('Source.IP6') OP_IN IPLIST(IPV6(IP6('::1')), IPV6(IP6('::2'))))")
rule = psr.parse('(Target.IP6 IN ["::1","::2"])')
self.assertEqual(repr(rule), "COMPBINOP(VARIABLE('Target.IP6') OP_IN LIST(CONSTANT('::1'), CONSTANT('::2')))")
res = cpl.compile(rule)
self.assertEqual(repr(res), "COMPBINOP(VARIABLE('Target.IP6') OP_IN IPLIST(IPV6(IP6('::1')), IPV6(IP6('::2'))))")
def test_03_basic_compilations(self):
"""
Perform basic compilation tests.
"""
self.maxDiff = None
cpl = IDEAFilterCompiler()
psr = PynspectFilterParser()
psr.build()
rule = psr.parse('(DetectTime == "2016-06-21T13:08:27Z")')
self.assertEqual(repr(rule), "COMPBINOP(VARIABLE('DetectTime') OP_EQ CONSTANT('2016-06-21T13:08:27Z'))")
res = cpl.compile(rule)
self.assertEqual(repr(res), "COMPBINOP(VARIABLE('DetectTime') OP_EQ DATETIME(datetime.datetime(2016, 6, 21, 13, 8, 27)))")
rule = psr.parse('(DetectTime == 2016-06-21T13:08:27Z)')
self.assertEqual(repr(rule), "COMPBINOP(VARIABLE('DetectTime') OP_EQ DATETIME('2016-06-21T13:08:27Z'))")
res = cpl.compile(rule)
self.assertEqual(repr(res), "COMPBINOP(VARIABLE('DetectTime') OP_EQ DATETIME(datetime.datetime(2016, 6, 21, 13, 8, 27)))")
rule = psr.parse('(Source.IP4 == "188.14.166.39")')
self.assertEqual(repr(rule), "COMPBINOP(VARIABLE('Source.IP4') OP_EQ CONSTANT('188.14.166.39'))")
res = cpl.compile(rule)
self.assertEqual(repr(res), "COMPBINOP(VARIABLE('Source.IP4') OP_EQ IPV4(IP4('188.14.166.39')))")
rule = psr.parse('(Source.IP4 == 188.14.166.39)')
self.assertEqual(repr(rule), "COMPBINOP(VARIABLE('Source.IP4') OP_EQ IPV4('188.14.166.39'))")
res = cpl.compile(rule)
self.assertEqual(repr(res), "COMPBINOP(VARIABLE('Source.IP4') OP_EQ IPV4(IP4('188.14.166.39')))")
rule = psr.parse('5 + 6 - 9')
self.assertEqual(repr(rule), "MATHBINOP(INTEGER(5) OP_PLUS MATHBINOP(INTEGER(6) OP_MINUS INTEGER(9)))")
res = cpl.compile(rule)
self.assertEqual(repr(res), "INTEGER(2)")
rule = psr.parse('Test + 10 - 9')
self.assertEqual(repr(rule), "MATHBINOP(VARIABLE('Test') OP_PLUS MATHBINOP(INTEGER(10) OP_MINUS INTEGER(9)))")
res = cpl.compile(rule)
self.assertEqual(repr(res), "MATHBINOP(VARIABLE('Test') OP_PLUS INTEGER(1))")
rule = psr.parse('Test + (10 - 9)')
self.assertEqual(repr(rule), "MATHBINOP(VARIABLE('Test') OP_PLUS MATHBINOP(INTEGER(10) OP_MINUS INTEGER(9)))")
res = cpl.compile(rule)
self.assertEqual(repr(res), "MATHBINOP(VARIABLE('Test') OP_PLUS INTEGER(1))")
rule = psr.parse('(Test + 10) - 9')
self.assertEqual(repr(rule), "MATHBINOP(MATHBINOP(VARIABLE('Test') OP_PLUS INTEGER(10)) OP_MINUS INTEGER(9))")
res = cpl.compile(rule)
self.assertEqual(repr(res), "MATHBINOP(MATHBINOP(VARIABLE('Test') OP_PLUS INTEGER(10)) OP_MINUS INTEGER(9))")
rule = psr.parse('9 - 6 + Test')
self.assertEqual(repr(rule), "MATHBINOP(INTEGER(9) OP_MINUS MATHBINOP(INTEGER(6) OP_PLUS VARIABLE('Test')))")
res = cpl.compile(rule)
self.assertEqual(repr(res), "MATHBINOP(INTEGER(9) OP_MINUS MATHBINOP(VARIABLE('Test') OP_PLUS INTEGER(6)))")
rule = psr.parse('9 - (6 + Test)')
self.assertEqual(repr(rule), "MATHBINOP(INTEGER(9) OP_MINUS MATHBINOP(INTEGER(6) OP_PLUS VARIABLE('Test')))")
res = cpl.compile(rule)
self.assertEqual(repr(res), "MATHBINOP(INTEGER(9) OP_MINUS MATHBINOP(VARIABLE('Test') OP_PLUS INTEGER(6)))")
rule = psr.parse('(9 - 6) + Test')
self.assertEqual(repr(rule), "MATHBINOP(MATHBINOP(INTEGER(9) OP_MINUS INTEGER(6)) OP_PLUS VARIABLE('Test'))")
res = cpl.compile(rule)
self.assertEqual(repr(res), "MATHBINOP(VARIABLE('Test') OP_PLUS INTEGER(3))")
class TestDataObjectFilterInspector(unittest.TestCase):
"""
Unit test class for testing the :py:mod:`pynspect.filters` module.
"""
def setUp(self):
self.flt = DataObjectFilter()
self.psr = PynspectFilterParser()
self.psr.build()
self.cpl = IDEAFilterCompiler()
def build_rule(self, rule_str):
"""
Build and compile rule tree from given rule string.
"""
rule = self.psr.parse(rule_str)
rule = self.cpl.compile(rule)
return rule
def test_01_inspector_filters(self):
"""
Perform tests of filters currently used in mentat-inspector.py.
"""
self.maxDiff = None
inspection_rules = [{
"filter":
"Category in ['Attempt.Login'] and Target.Port in [3389]",
"str":
'((Category OP_IN ["Attempt.Login"]) OP_AND (Target.Port OP_IN [3389]))',
"tests": [[{
"Format":
"IDEA0",
"ID":
"e214d2d9-359b-443d-993d-3cc5637107a0",
"Source": [{
"IP4": ["188.14.166.39"]
}],
"Target": [{
"IP4": ["195.113.165.128/25"],
"Port": ["3389"],
"Proto": ["tcp", "ssh"]
}],
"Note":
"SSH login attempt",
"DetectTime":
"2016-06-21 13:08:27Z",
"Node": [{
"Type": ["Connection", "Honeypot"],
"SW": ["Kippo"],
"Name": "cz.uhk.apate.cowrie"
}],
"Category": ["Attempt.Login"]
}, True],
[{
"Format":
"IDEA0",
"ID":
"e214d2d9-359b-443d-993d-3cc5637107a0",
"Source": [{
"IP4": ["188.14.166.39"]
}],
"Target": [{
"IP4": ["195.113.165.128/25"],
"Port": ["338"],
"Proto": ["tcp", "ssh"]
}],
"Note":
"SSH login attempt",
"DetectTime":
"2016-06-21 13:08:27Z",
"Node": [{
"Type": ["Connection", "Honeypot"],
"SW": ["Kippo"],
"Name": "cz.uhk.apate.cowrie"
}],
"Category": ["Attempt.Login"]
}, False]],
}, {
"filter":
"Category in ['Attempt.Login'] and (Target.Proto in ['telnet'] or Source.Proto in ['telnet'] or Target.Port in [23])",
"str":
'((Category OP_IN ["Attempt.Login"]) OP_AND ((Target.Proto OP_IN ["telnet"]) OP_OR ((Source.Proto OP_IN ["telnet"]) OP_OR (Target.Port OP_IN [23]))))'
}, {
"filter":
"Category in ['Attempt.Login'] and (Target.Proto in ['ssh'] or Source.Proto in ['ssh'] or Target.Port in [22])",
"str":
'((Category OP_IN ["Attempt.Login"]) OP_AND ((Target.Proto OP_IN ["ssh"]) OP_OR ((Source.Proto OP_IN ["ssh"]) OP_OR (Target.Port OP_IN [22]))))'
}, {
"filter":
"Category in ['Attempt.Login'] and (Target.Proto in ['sip', 'sip-tls'] or Source.Proto in ['sip', 'sip-tls'] or Target.Port in [5060])",
"str":
'((Category OP_IN ["Attempt.Login"]) OP_AND ((Target.Proto OP_IN ["sip", "sip-tls"]) OP_OR ((Source.Proto OP_IN ["sip", "sip-tls"]) OP_OR (Target.Port OP_IN [5060]))))'
}, {
"filter":
"Category in ['Attempt.Exploit'] and (Target.Proto in ['sip', 'sip-tls'] or Source.Proto in ['sip', 'sip-tls'] or Target.Port in [5060])",
"str":
'((Category OP_IN ["Attempt.Exploit"]) OP_AND ((Target.Proto OP_IN ["sip", "sip-tls"]) OP_OR ((Source.Proto OP_IN ["sip", "sip-tls"]) OP_OR (Target.Port OP_IN [5060]))))'
}, {
"filter":
"Category in ['Attempt.Exploit'] and Target.Port in [23]",
"str":
'((Category OP_IN ["Attempt.Exploit"]) OP_AND (Target.Port OP_IN [23]))'
}, {
"filter":
"Category in ['Attempt.Exploit'] and (Target.Port in [80, 443] or Source.Proto in ['http', 'https', 'http-alt'] or Target.Proto in ['http', 'https', 'http-alt'])",
"str":
'((Category OP_IN ["Attempt.Exploit"]) OP_AND ((Target.Port OP_IN [80, 443]) OP_OR ((Source.Proto OP_IN ["http", "https", "http-alt"]) OP_OR (Target.Proto OP_IN ["http", "https", "http-alt"]))))'
}, {
"filter":
"Category in ['Attempt.Exploit'] and (Target.Port in [3306] or Source.Proto in ['mysql'] or Target.Proto in ['mysql'])",
"str":
'((Category OP_IN ["Attempt.Exploit"]) OP_AND ((Target.Port OP_IN [3306]) OP_OR ((Source.Proto OP_IN ["mysql"]) OP_OR (Target.Proto OP_IN ["mysql"]))))'
}, {
"filter":
"Category in ['Attempt.Exploit'] and (Target.Port in [445] or Source.Proto in ['microsoft-ds', 'smb'] or Target.Proto in ['microsoft-ds', 'smb'])",
"str":
'((Category OP_IN ["Attempt.Exploit"]) OP_AND ((Target.Port OP_IN [445]) OP_OR ((Source.Proto OP_IN ["microsoft-ds", "smb"]) OP_OR (Target.Proto OP_IN ["microsoft-ds", "smb"]))))'
}, {
"filter":
"Category in ['Attempt.Exploit'] and (Target.Port in [135] or Source.Proto in ['loc-srv', 'epmap'] or Target.Proto in ['loc-srv', 'epmap'])",
"str":
'((Category OP_IN ["Attempt.Exploit"]) OP_AND ((Target.Port OP_IN [135]) OP_OR ((Source.Proto OP_IN ["loc-srv", "epmap"]) OP_OR (Target.Proto OP_IN ["loc-srv", "epmap"]))))'
}, {
"filter":
"Category in ['Attempt.Exploit'] and (Target.Port in [1900] or Source.Proto in ['upnp', 'ssdp'] or Target.Proto in ['upnp', 'ssdp'])",
"str":
'((Category OP_IN ["Attempt.Exploit"]) OP_AND ((Target.Port OP_IN [1900]) OP_OR ((Source.Proto OP_IN ["upnp", "ssdp"]) OP_OR (Target.Proto OP_IN ["upnp", "ssdp"]))))'
}, {
"filter":
"Category in ['Attempt.Exploit'] and (Target.Port in [20, 21, 989, 990] or Source.Proto in ['ftp', 'ftp-data', 'ftps', 'ftps-data'] or Target.Proto in ['ftp', 'ftp-data', 'ftps', 'ftps-data'])",
"str":
'((Category OP_IN ["Attempt.Exploit"]) OP_AND ((Target.Port OP_IN [20, 21, 989, 990]) OP_OR ((Source.Proto OP_IN ["ftp", "ftp-data", "ftps", "ftps-data"]) OP_OR (Target.Proto OP_IN ["ftp", "ftp-data", "ftps", "ftps-data"]))))'
}, {
"filter":
"Category in ['Attempt.Exploit'] and (Target.Port in [1433, 1434] or Source.Proto in ['ms-sql-s', 'ms-sql-m'] or Target.Proto in ['ms-sql-s', 'ms-sql-m'])",
"str":
'((Category OP_IN ["Attempt.Exploit"]) OP_AND ((Target.Port OP_IN [1433, 1434]) OP_OR ((Source.Proto OP_IN ["ms-sql-s", "ms-sql-m"]) OP_OR (Target.Proto OP_IN ["ms-sql-s", "ms-sql-m"]))))'
}, {
"filter":
"Category in ['Attempt.Exploit'] and (Target.Port in [42] or Source.Proto in ['nameserver'] or Target.Proto in ['nameserver'])",
"str":
'((Category OP_IN ["Attempt.Exploit"]) OP_AND ((Target.Port OP_IN [42]) OP_OR ((Source.Proto OP_IN ["nameserver"]) OP_OR (Target.Proto OP_IN ["nameserver"]))))'
}, {
"filter":
"Category in ['Attempt.Exploit'] and Node.SW in ['Dionaea']",
"str":
'((Category OP_IN ["Attempt.Exploit"]) OP_AND (Node.SW OP_IN ["Dionaea"]))'
}, {
"filter":
"Category in ['Availability.DoS', 'Availability.DDoS'] and (Target.Proto in ['dns', 'domain'] or Source.Proto in ['dns', 'domain'] or Target.Port in [53] or Source.Port in [53])",
"str":
'((Category OP_IN ["Availability.DoS", "Availability.DDoS"]) OP_AND ((Target.Proto OP_IN ["dns", "domain"]) OP_OR ((Source.Proto OP_IN ["dns", "domain"]) OP_OR ((Target.Port OP_IN [53]) OP_OR (Source.Port OP_IN [53])))))'
}, {
"filter":
"Category in ['Availability.DDoS'] and Node.Type in ['Flow'] and Node.Type in ['Statistical']",
"str":
'((Category OP_IN ["Availability.DDoS"]) OP_AND ((Node.Type OP_IN ["Flow"]) OP_AND (Node.Type OP_IN ["Statistical"])))'
}, {
"filter":
"Category in ['Abusive.Spam'] and Node.SW in ['UCEPROT']",
"str":
'((Category OP_IN ["Abusive.Spam"]) OP_AND (Node.SW OP_IN ["UCEPROT"]))'
}, {
"filter":
"Category in ['Abusive.Spam'] and Node.SW in ['Fail2Ban', 'IntelMQ']",
"str":
'((Category OP_IN ["Abusive.Spam"]) OP_AND (Node.SW OP_IN ["Fail2Ban", "IntelMQ"]))'
}, {
"filter":
"Category in ['Vulnerable.Config'] and (Source.Proto in ['qotd'] or Source.Port in [17])",
"str":
'((Category OP_IN ["Vulnerable.Config"]) OP_AND ((Source.Proto OP_IN ["qotd"]) OP_OR (Source.Port OP_IN [17])))'
}, {
"filter":
"Category in ['Vulnerable.Config'] and Source.Proto in ['ssdp']",
"str":
'((Category OP_IN ["Vulnerable.Config"]) OP_AND (Source.Proto OP_IN ["ssdp"]))'
}, {
"filter":
"Category in ['Vulnerable.Config'] and (Source.Proto in ['ntp'] or Source.Port in [123])",
"str":
'((Category OP_IN ["Vulnerable.Config"]) OP_AND ((Source.Proto OP_IN ["ntp"]) OP_OR (Source.Port OP_IN [123])))'
}, {
"filter":
"Category in ['Vulnerable.Config'] and (Source.Proto in ['domain'] or Source.Port in [53])",
"str":
'((Category OP_IN ["Vulnerable.Config"]) OP_AND ((Source.Proto OP_IN ["domain"]) OP_OR (Source.Port OP_IN [53])))'
}, {
"filter":
"Category in ['Vulnerable.Config'] and (Source.Proto in ['netbios-ns'] or Source.Port in [137])",
"str":
'((Category OP_IN ["Vulnerable.Config"]) OP_AND ((Source.Proto OP_IN ["netbios-ns"]) OP_OR (Source.Port OP_IN [137])))'
}, {
"filter":
"Category in ['Vulnerable.Config'] and (Source.Proto in ['ipmi'] or Source.Port in [623])",
"str":
'((Category OP_IN ["Vulnerable.Config"]) OP_AND ((Source.Proto OP_IN ["ipmi"]) OP_OR (Source.Port OP_IN [623])))'
}, {
"filter":
"Category in ['Vulnerable.Config'] and (Source.Proto in ['chargen'] or Source.Port in [19])",
"str":
'((Category OP_IN ["Vulnerable.Config"]) OP_AND ((Source.Proto OP_IN ["chargen"]) OP_OR (Source.Port OP_IN [19])))'
}, {
"filter": "Category in ['Anomaly.Traffic']",
"str": '(Category OP_IN ["Anomaly.Traffic"])'
}, {
"filter":
"Category in ['Anomaly.Connection'] and Source.Type in ['Booter']",
"str":
'((Category OP_IN ["Anomaly.Connection"]) OP_AND (Source.Type OP_IN ["Booter"]))'
}, {
"filter":
"Category in ['Intrusion.Botnet'] and Source.Type in ['Botnet']",
"str":
'((Category OP_IN ["Intrusion.Botnet"]) OP_AND (Source.Type OP_IN ["Botnet"]))'
}, {
"filter": "Category in ['Recon.Scanning']",
"str": '(Category OP_IN ["Recon.Scanning"])'
}]
for insr in inspection_rules:
rule = self.build_rule(insr['filter'])
self.assertEqual(str(rule), insr['str'])
if 'tests' in insr:
for itemt in insr['tests']:
msg_idea = lite.Idea(itemt[0])
self.assertEqual(
[insr['filter'],
self.flt.filter(rule, msg_idea)],
[insr['filter'], itemt[1]])
class Config():
addrGroups = dict()
actions = dict()
rules = list()
parser = None
compiler = None
def __init__(self, parsed_config, trap=None, warden=None):
"""
:param parsed_config Parser with parsed Yaml configuration
:param trap Instance of TRAP client used in TrapAction
:param warden Instance of Warden Client used in WardenAction
"""
self.conf = parsed_config
if not self.conf:
raise Exception(
"Loading YAML file ({0}) failed. Isn't it empty?".format(path))
# Build parser
self.parser = PynspectFilterParser()
self.parser.build()
self.compiler = IDEAFilterCompiler()
self.addrGroups = dict()
if "namespace" not in self.conf:
logger.error(
"ERROR: 'namespace' is required but is missing. Please specify 'namespace' in YAML config to identify names of reporters (e.g., com.example.collectornemea)."
)
# Create all address groups if there are any
if "addressgroups" in self.conf:
for i in self.conf["addressgroups"]:
self.addrGroups[i["id"]] = AddressGroup(i)
self.smtp_conns = dict()
# Check if "smtp_connections" exists when there is some "email" action in "custom_actions"
if ("custom_actions" in self.conf and any(
[i.__contains__("email") for i in self.conf["custom_actions"]])
and "smtp_connections" not in self.conf):
raise LookupError(
"'smtp_connections' is required when there is at least one 'email' action but it is missing in YAML config. Check your YAML config."
)
# Parse parameters for all smtp connections
if "smtp_connections" in self.conf:
for i in self.conf["smtp_connections"]:
self.smtp_conns[i["id"]] = i
self.actions = dict()
# Parse and instantiate all custom actions
if "custom_actions" in self.conf:
for i in self.conf["custom_actions"]:
if "mark" in i:
from .actions.Mark import MarkAction
self.actions[i["id"]] = MarkAction(i)
elif "mongo" in i:
from .actions.Mongo import MongoAction
self.actions[i["id"]] = MongoAction(i)
elif "email" in i:
from .actions.Email import EmailAction
self.actions[i["id"]] = EmailAction(
i, self.smtp_conns[i['email']['smtp_connection']])
elif "file" in i:
from .actions.File import FileAction
self.actions[i["id"]] = FileAction(i)
elif "warden" in i:
"""
Pass Warden Client instance to the Warden action
"""
from .actions.Warden import WardenAction
self.actions[i["id"]] = WardenAction(i, warden)
elif "trap" in i:
"""
Pass TRAP context instance to the TRAP action
"""
from .actions.Trap import TrapAction
self.actions[i["id"]] = TrapAction(i, trap)
elif "drop" in i:
logger.warning(
"Drop action mustn't be specified in custom_actions!")
continue
else:
raise Exception("Undefined action: " + str(i))
self.actions["drop"] = DropAction()
self.rules = list()
# Parse all rules and match them with actions and address groups
# There must be at least one rule (mandatory field)
if "rules" in self.conf:
if self.conf["rules"]:
for i in self.conf["rules"]:
self.rules.append(
Rule(i,
self.actions,
self.addrGroups,
parser=self.parser,
compiler=self.compiler))
if not self.rules:
raise Exception(
"YAML file should contain at least one `rule` in `rules`.")
else:
raise Exception("YAML file must contain `rules`.")
def match(self, msg):
"""
Check if msg matches rules from config file.
Return a list of bool values representing results of all tested rules.
"""
results = []
actionsDone = []
try:
for rule in self.rules:
self.clearActionLog()
res = rule.filter(msg)
#logger.debug("Filter by rule: %s \n message: %s\n\nresult: %s", rule, msg, res)
#print("Filter by rule: %s \n message: %s\n\nresult: %s" % (rule.rule(), msg, res))
results.append(res)
tmp_msg = copy.deepcopy(msg)
if res:
# condition is True
rule.actions(tmp_msg)
logger.info("action running")
else:
# condition is False
rule.elseactions(tmp_msg)
logger.info("else action running")
actionsDone.append(self.getActionLog())
except DropMsg:
# This exception breaks the processing of rule list.
pass
return (results, actionsDone)
def getActionLog(self):
return Action.actionLog
def clearActionLog(self):
Action.actionLog = []
def loglevel(self):
"""Get logging level
CRITICAL 50
ERROR 40
WARNING 30
INFO 20
DEBUG 10
NOTSET 0
"""
try:
return (self.conf["reporter"]["loglevel"]) * 10
except:
return 30
def __str__(self):
smtp = dict()
for smtp_id, conns in self.smtp_conns.items():
smtp[smtp_id] = "\n".join([
"\t{}: {}".format(param, val) for param, val in conns.items()
])
s = "\n".join("{}:\n{}\n".format(smtp_id, params)
for smtp_id, params in smtp.items())
ag = "\n".join([str(self.addrGroups[key]) for key in self.addrGroups])
a = "\n".join(
[key + ":\n\t" + str(self.actions[key]) for key in self.actions])
r = "\n".join([str(val) for val in self.rules])
string = "Namespace: {0}\n----------------\nSmtp connections:\n{1}\n----------------\nAddress Groups:\n{2}\n"\
"----------------\nCustom Actions:\n{3}\n----------------\nRules:\n{4}\n".format(self.conf.get("namespace"), s, ag, a, r)
return string
def __init__(self, parsed_config, trap=None, warden=None):
"""
:param parsed_config Parser with parsed Yaml configuration
:param trap Instance of TRAP client used in TrapAction
:param warden Instance of Warden Client used in WardenAction
"""
self.conf = parsed_config
if not self.conf:
raise Exception(
"Loading YAML file ({0}) failed. Isn't it empty?".format(path))
# Build parser
self.parser = PynspectFilterParser()
self.parser.build()
self.compiler = IDEAFilterCompiler()
self.addrGroups = dict()
if "namespace" not in self.conf:
logger.error(
"ERROR: 'namespace' is required but is missing. Please specify 'namespace' in YAML config to identify names of reporters (e.g., com.example.collectornemea)."
)
# Create all address groups if there are any
if "addressgroups" in self.conf:
for i in self.conf["addressgroups"]:
self.addrGroups[i["id"]] = AddressGroup(i)
self.smtp_conns = dict()
# Check if "smtp_connections" exists when there is some "email" action in "custom_actions"
if ("custom_actions" in self.conf and any(
[i.__contains__("email") for i in self.conf["custom_actions"]])
and "smtp_connections" not in self.conf):
raise LookupError(
"'smtp_connections' is required when there is at least one 'email' action but it is missing in YAML config. Check your YAML config."
)
# Parse parameters for all smtp connections
if "smtp_connections" in self.conf:
for i in self.conf["smtp_connections"]:
self.smtp_conns[i["id"]] = i
self.actions = dict()
# Parse and instantiate all custom actions
if "custom_actions" in self.conf:
for i in self.conf["custom_actions"]:
if "mark" in i:
from .actions.Mark import MarkAction
self.actions[i["id"]] = MarkAction(i)
elif "mongo" in i:
from .actions.Mongo import MongoAction
self.actions[i["id"]] = MongoAction(i)
elif "email" in i:
from .actions.Email import EmailAction
self.actions[i["id"]] = EmailAction(
i, self.smtp_conns[i['email']['smtp_connection']])
elif "file" in i:
from .actions.File import FileAction
self.actions[i["id"]] = FileAction(i)
elif "warden" in i:
"""
Pass Warden Client instance to the Warden action
"""
from .actions.Warden import WardenAction
self.actions[i["id"]] = WardenAction(i, warden)
elif "trap" in i:
"""
Pass TRAP context instance to the TRAP action
"""
from .actions.Trap import TrapAction
self.actions[i["id"]] = TrapAction(i, trap)
elif "drop" in i:
logger.warning(
"Drop action mustn't be specified in custom_actions!")
continue
else:
raise Exception("Undefined action: " + str(i))
self.actions["drop"] = DropAction()
self.rules = list()
# Parse all rules and match them with actions and address groups
# There must be at least one rule (mandatory field)
if "rules" in self.conf:
if self.conf["rules"]:
for i in self.conf["rules"]:
self.rules.append(
Rule(i,
self.actions,
self.addrGroups,
parser=self.parser,
compiler=self.compiler))
if not self.rules:
raise Exception(
"YAML file should contain at least one `rule` in `rules`.")
else:
raise Exception("YAML file must contain `rules`.")
def setUp(self):
self.flt = DataObjectFilter()
self.psr = PynspectFilterParser()
self.psr.build()
class TestDataObjectFilter(unittest.TestCase):
"""
Unit test class for testing the :py:mod:`pynspect.filters` module.
"""
test_msg1 = {
"ID" : "e214d2d9-359b-443d-993d-3cc5637107a0",
"WinEndTime" : "2016-06-21 11:25:01Z",
"ConnCount" : 2,
"Source" : [
{
"IP4" : [
"188.14.166.39"
]
}
],
"Format" : "IDEA0",
"WinStartTime" : "2016-06-21 11:20:01Z",
"_CESNET" : {
"StorageTime" : 1466508305
},
"Target" : [
{
"IP4" : [
"195.113.165.128/25"
],
"Port" : [
"22"
],
"Proto" : [
"tcp",
"ssh"
],
"Anonymised" : True
}
],
"Note" : "SSH login attempt",
"DetectTime" : "2016-06-21 13:08:27Z",
"Node" : [
{
"Name" : "cz.cesnet.mentat.warden_filer",
"Type" : [
"Relay"
]
},
{
"AggrWin" : "00:05:00",
"Type" : [
"Connection",
"Honeypot",
"Recon"
],
"SW" : [
"Kippo"
],
"Name" : "cz.uhk.apate.cowrie"
}
],
"Category" : [
"Attempt.Login"
]
}
def setUp(self):
self.flt = DataObjectFilter()
self.psr = PynspectFilterParser()
self.psr.build()
def test_01_basic_logical(self):
"""
Perform filtering tests with basic logical expressions.
"""
self.maxDiff = None
rule = LogicalBinOpRule('OP_AND', ConstantRule(True), ConstantRule(True))
self.assertEqual(self.flt.filter(rule, self.test_msg1), True)
rule = LogicalBinOpRule('OP_AND', ConstantRule(True), ConstantRule(False))
self.assertEqual(self.flt.filter(rule, self.test_msg1), False)
rule = LogicalBinOpRule('OP_AND', ConstantRule(False), ConstantRule(True))
self.assertEqual(self.flt.filter(rule, self.test_msg1), False)
rule = LogicalBinOpRule('OP_AND', ConstantRule(False), ConstantRule(False))
self.assertEqual(self.flt.filter(rule, self.test_msg1), False)
rule = LogicalBinOpRule('OP_OR', ConstantRule(True), ConstantRule(True))
self.assertEqual(self.flt.filter(rule, self.test_msg1), True)
rule = LogicalBinOpRule('OP_OR', ConstantRule(True), ConstantRule(False))
self.assertEqual(self.flt.filter(rule, self.test_msg1), True)
rule = LogicalBinOpRule('OP_OR', ConstantRule(False), ConstantRule(True))
self.assertEqual(self.flt.filter(rule, self.test_msg1), True)
rule = LogicalBinOpRule('OP_OR', ConstantRule(False), ConstantRule(False))
self.assertEqual(self.flt.filter(rule, self.test_msg1), False)
rule = LogicalBinOpRule('OP_XOR', ConstantRule(True), ConstantRule(True))
self.assertEqual(self.flt.filter(rule, self.test_msg1), False)
rule = LogicalBinOpRule('OP_XOR', ConstantRule(True), ConstantRule(False))
self.assertEqual(self.flt.filter(rule, self.test_msg1), True)
rule = LogicalBinOpRule('OP_XOR', ConstantRule(False), ConstantRule(True))
self.assertEqual(self.flt.filter(rule, self.test_msg1), True)
rule = LogicalBinOpRule('OP_XOR', ConstantRule(False), ConstantRule(False))
self.assertEqual(self.flt.filter(rule, self.test_msg1), False)
rule = UnaryOperationRule('OP_NOT', ConstantRule(True))
self.assertEqual(self.flt.filter(rule, self.test_msg1), False)
rule = UnaryOperationRule('OP_NOT', ConstantRule(False))
self.assertEqual(self.flt.filter(rule, self.test_msg1), True)
rule = UnaryOperationRule('OP_NOT', VariableRule("Target.Anonymised"))
self.assertEqual(self.flt.filter(rule, self.test_msg1), False)
def test_02_basic_comparison(self):
"""
Perform filtering tests with basic comparison expressions.
"""
self.maxDiff = None
rule = ComparisonBinOpRule('OP_EQ', VariableRule("ID"), ConstantRule("e214d2d9-359b-443d-993d-3cc5637107a0"))
self.assertEqual(self.flt.filter(rule, self.test_msg1), True)
rule = ComparisonBinOpRule('OP_EQ', VariableRule("ID"), ConstantRule("e214d2d9-359b-443d-993d-3cc5637107"))
self.assertEqual(self.flt.filter(rule, self.test_msg1), False)
rule = ComparisonBinOpRule('OP_NE', VariableRule("ID"), ConstantRule("e214d2d9-359b-443d-993d-3cc5637107a0"))
self.assertEqual(self.flt.filter(rule, self.test_msg1), False)
rule = ComparisonBinOpRule('OP_NE', VariableRule("ID"), ConstantRule("e214d2d9-359b-443d-993d-3cc5637107"))
self.assertEqual(self.flt.filter(rule, self.test_msg1), True)
rule = ComparisonBinOpRule('OP_LIKE', VariableRule("ID"), ConstantRule("e214d2d9"))
self.assertEqual(self.flt.filter(rule, self.test_msg1), True)
rule = ComparisonBinOpRule('OP_LIKE', VariableRule("ID"), ConstantRule("xxxxxxxx"))
self.assertEqual(self.flt.filter(rule, self.test_msg1), False)
rule = ComparisonBinOpRule('OP_IN', VariableRule("Category"), ListRule(ConstantRule("Phishing"), ListRule(ConstantRule("Attempt.Login"))))
self.assertEqual(self.flt.filter(rule, self.test_msg1), True)
rule = ComparisonBinOpRule('OP_IN', VariableRule("Category"), ListRule(ConstantRule("Phishing"), ListRule(ConstantRule("Spam"))))
self.assertEqual(self.flt.filter(rule, self.test_msg1), False)
rule = ComparisonBinOpRule('OP_IS', VariableRule("Category"), ListRule(ConstantRule("Attempt.Login")))
self.assertEqual(self.flt.filter(rule, self.test_msg1), True)
rule = ComparisonBinOpRule('OP_IS', VariableRule("Category"), ListRule(ConstantRule("Phishing"), ListRule(ConstantRule("Attempt.Login"))))
self.assertEqual(self.flt.filter(rule, self.test_msg1), False)
rule = ComparisonBinOpRule('OP_EQ', VariableRule("ConnCount"), IntegerRule(2))
self.assertEqual(self.flt.filter(rule, self.test_msg1), True)
rule = ComparisonBinOpRule('OP_EQ', VariableRule("ConnCount"), IntegerRule(4))
self.assertEqual(self.flt.filter(rule, self.test_msg1), False)
rule = ComparisonBinOpRule('OP_NE', VariableRule("ConnCount"), IntegerRule(2))
self.assertEqual(self.flt.filter(rule, self.test_msg1), False)
rule = ComparisonBinOpRule('OP_NE', VariableRule("ConnCount"), IntegerRule(4))
self.assertEqual(self.flt.filter(rule, self.test_msg1), True)
rule = ComparisonBinOpRule('OP_GT', VariableRule("ConnCount"), IntegerRule(2))
self.assertEqual(self.flt.filter(rule, self.test_msg1), False)
rule = ComparisonBinOpRule('OP_GT', VariableRule("ConnCount"), IntegerRule(1))
self.assertEqual(self.flt.filter(rule, self.test_msg1), True)
rule = ComparisonBinOpRule('OP_GE', VariableRule("ConnCount"), IntegerRule(2))
self.assertEqual(self.flt.filter(rule, self.test_msg1), True)
rule = ComparisonBinOpRule('OP_GE', VariableRule("ConnCount"), IntegerRule(1))
self.assertEqual(self.flt.filter(rule, self.test_msg1), True)
rule = ComparisonBinOpRule('OP_GE', VariableRule("ConnCount"), IntegerRule(3))
self.assertEqual(self.flt.filter(rule, self.test_msg1), False)
rule = ComparisonBinOpRule('OP_LT', VariableRule("ConnCount"), IntegerRule(2))
self.assertEqual(self.flt.filter(rule, self.test_msg1), False)
rule = ComparisonBinOpRule('OP_LT', VariableRule("ConnCount"), IntegerRule(3))
self.assertEqual(self.flt.filter(rule, self.test_msg1), True)
rule = ComparisonBinOpRule('OP_LE', VariableRule("ConnCount"), IntegerRule(2))
self.assertEqual(self.flt.filter(rule, self.test_msg1), True)
rule = ComparisonBinOpRule('OP_LE', VariableRule("ConnCount"), IntegerRule(3))
self.assertEqual(self.flt.filter(rule, self.test_msg1), True)
rule = ComparisonBinOpRule('OP_LE', VariableRule("ConnCount"), IntegerRule(1))
self.assertEqual(self.flt.filter(rule, self.test_msg1), False)
def test_03_parsed_comparison(self):
"""
Perform filtering tests with parsed comparison expressions.
"""
self.maxDiff = None
rule = self.psr.parse('ID == "e214d2d9-359b-443d-993d-3cc5637107a0"')
self.assertEqual(self.flt.filter(rule, self.test_msg1), True)
rule = self.psr.parse('ID eq "e214d2d9-359b-443d-993d-3cc5637107"')
self.assertEqual(self.flt.filter(rule, self.test_msg1), False)
rule = self.psr.parse('ID != "e214d2d9-359b-443d-993d-3cc5637107a0"')
self.assertEqual(self.flt.filter(rule, self.test_msg1), False)
rule = self.psr.parse('ID ne "e214d2d9-359b-443d-993d-3cc5637107"')
self.assertEqual(self.flt.filter(rule, self.test_msg1), True)
rule = self.psr.parse('ID like "e214d2d9"')
self.assertEqual(self.flt.filter(rule, self.test_msg1), True)
rule = self.psr.parse('ID LIKE "xxxxxxxx"')
self.assertEqual(self.flt.filter(rule, self.test_msg1), False)
rule = self.psr.parse('Category in ["Phishing" , "Attempt.Login"]')
self.assertEqual(self.flt.filter(rule, self.test_msg1), True)
rule = self.psr.parse('Category IN ["Phishing" , "Spam"]')
self.assertEqual(self.flt.filter(rule, self.test_msg1), False)
rule = self.psr.parse('Category is ["Attempt.Login"]')
self.assertEqual(self.flt.filter(rule, self.test_msg1), True)
rule = self.psr.parse('Category IS ["Phishing" , "Attempt.Login"]')
self.assertEqual(self.flt.filter(rule, self.test_msg1), False)
rule = self.psr.parse('ConnCount == 2')
self.assertEqual(self.flt.filter(rule, self.test_msg1), True)
rule = self.psr.parse('ConnCount eq 4')
self.assertEqual(self.flt.filter(rule, self.test_msg1), False)
rule = self.psr.parse('ConnCount != 2')
self.assertEqual(self.flt.filter(rule, self.test_msg1), False)
rule = self.psr.parse('ConnCount ne 4')
self.assertEqual(self.flt.filter(rule, self.test_msg1), True)
rule = self.psr.parse('ConnCount > 2')
self.assertEqual(self.flt.filter(rule, self.test_msg1), False)
rule = self.psr.parse('ConnCount gt 1')
self.assertEqual(self.flt.filter(rule, self.test_msg1), True)
rule = self.psr.parse('ConnCount >= 2')
self.assertEqual(self.flt.filter(rule, self.test_msg1), True)
rule = self.psr.parse('ConnCount ge 1')
self.assertEqual(self.flt.filter(rule, self.test_msg1), True)
rule = self.psr.parse('ConnCount GE 3')
self.assertEqual(self.flt.filter(rule, self.test_msg1), False)
rule = self.psr.parse('ConnCount < 2')
self.assertEqual(self.flt.filter(rule, self.test_msg1), False)
rule = self.psr.parse('ConnCount lt 3')
self.assertEqual(self.flt.filter(rule, self.test_msg1), True)
rule = self.psr.parse('ConnCount <= 2')
self.assertEqual(self.flt.filter(rule, self.test_msg1), True)
rule = self.psr.parse('ConnCount le 3')
self.assertEqual(self.flt.filter(rule, self.test_msg1), True)
rule = self.psr.parse('ConnCount LE 1')
self.assertEqual(self.flt.filter(rule, self.test_msg1), False)
rule = self.psr.parse('ConnCounts LE 1')
self.assertEqual(self.flt.filter(rule, self.test_msg1), None)
def test_04_basic_math(self):
"""
Perform filtering tests with basic math expressions.
"""
self.maxDiff = None
rule = MathBinOpRule('OP_PLUS', VariableRule("ConnCount"), IntegerRule(1))
self.assertEqual(self.flt.filter(rule, self.test_msg1), 3)
rule = MathBinOpRule('OP_MINUS', VariableRule("ConnCount"), IntegerRule(1))
self.assertEqual(self.flt.filter(rule, self.test_msg1), 1)
rule = MathBinOpRule('OP_TIMES', VariableRule("ConnCount"), IntegerRule(5))
self.assertEqual(self.flt.filter(rule, self.test_msg1), 10)
rule = MathBinOpRule('OP_DIVIDE', VariableRule("ConnCount"), IntegerRule(2))
self.assertEqual(self.flt.filter(rule, self.test_msg1), 1)
rule = MathBinOpRule('OP_MODULO', VariableRule("ConnCount"), IntegerRule(2))
self.assertEqual(self.flt.filter(rule, self.test_msg1), 0)
def test_05_parsed_math(self):
"""
Perform filtering tests with parsed math expressions.
"""
self.maxDiff = None
rule = self.psr.parse('ConnCount + 1')
self.assertEqual(self.flt.filter(rule, self.test_msg1), 3)
rule = self.psr.parse('ConnCount - 1')
self.assertEqual(self.flt.filter(rule, self.test_msg1), 1)
rule = self.psr.parse('ConnCount * 5')
self.assertEqual(self.flt.filter(rule, self.test_msg1), 10)
rule = self.psr.parse('ConnCount / 2')
self.assertEqual(self.flt.filter(rule, self.test_msg1), 1)
rule = self.psr.parse('ConnCount % 2')
self.assertEqual(self.flt.filter(rule, self.test_msg1), 0)
def test_06_advanced_filters(self):
"""
Perform advanced filtering tests.
"""
self.maxDiff = None
rule = self.psr.parse('(ConnCount + 10) > 11')
self.assertEqual(repr(rule), "COMPBINOP(MATHBINOP(VARIABLE('ConnCount') OP_PLUS INTEGER(10)) OP_GT INTEGER(11))")
self.assertEqual(self.flt.filter(rule, self.test_msg1), True)
rule = self.psr.parse('((ConnCount + 3) < 5) or ((ConnCount + 10) > 11)')
self.assertEqual(repr(rule), "LOGBINOP(COMPBINOP(MATHBINOP(VARIABLE('ConnCount') OP_PLUS INTEGER(3)) OP_LT INTEGER(5)) OP_OR COMPBINOP(MATHBINOP(VARIABLE('ConnCount') OP_PLUS INTEGER(10)) OP_GT INTEGER(11)))")
self.assertEqual(self.flt.filter(rule, self.test_msg1), True)
rule = self.psr.parse('1')
self.assertEqual(repr(rule), "INTEGER(1)")
self.assertEqual(self.flt.filter(rule, self.test_msg1), True)
def test_07_non_existent_nodes(self):
"""
Perform advanced filtering tests.
"""
self.maxDiff = None
rule = self.psr.parse('(ConnCounts + 10) > 11')
self.assertEqual(self.flt.filter(rule, self.test_msg1), None)
rule = self.psr.parse('ConnCount > ConnCounts')
self.assertEqual(self.flt.filter(rule, self.test_msg1), None)
rule = self.psr.parse('DetectTime < InspectionTime')
self.assertEqual(self.flt.filter(rule, self.test_msg1), None)
def test_08_functions(self):
"""
Perform advanced filtering tests.
"""
self.maxDiff = None
rule = self.psr.parse('(size(Node.Type) > 2)')
self.assertEqual(repr(rule), "COMPBINOP(FUNCTION(size(VARIABLE('Node.Type'),)) OP_GT INTEGER(2))")
self.assertEqual(self.flt.filter(rule, self.test_msg1), True)
rule = self.psr.parse('(size(Source.IP4) > 4)')
self.assertEqual(repr(rule), "COMPBINOP(FUNCTION(size(VARIABLE('Source.IP4'),)) OP_GT INTEGER(4))")
self.assertEqual(self.flt.filter(rule, self.test_msg1), False)
rule = self.psr.parse('(strlen(ID) > 8)')
self.assertEqual(repr(rule), "COMPBINOP(FUNCTION(strlen(VARIABLE('ID'),)) OP_GT INTEGER(8))")
self.assertEqual(self.flt.filter(rule, self.test_msg1), True)
rule = self.psr.parse('(strlen(ID) < 8)')
self.assertEqual(repr(rule), "COMPBINOP(FUNCTION(strlen(VARIABLE('ID'),)) OP_LT INTEGER(8))")
self.assertEqual(self.flt.filter(rule, self.test_msg1), False)
rule = self.psr.parse('(strlen(Description) > 8)')
self.assertEqual(repr(rule), "COMPBINOP(FUNCTION(strlen(VARIABLE('Description'),)) OP_GT INTEGER(8))")
self.assertEqual(self.flt.filter(rule, self.test_msg1), None)
rule = self.psr.parse('(strlen(Description) < 8)')
self.assertEqual(repr(rule), "COMPBINOP(FUNCTION(strlen(VARIABLE('Description'),)) OP_LT INTEGER(8))")
self.assertEqual(self.flt.filter(rule, self.test_msg1), None)
rule = self.psr.parse('(strlen(Node.Name) > 20)')
self.assertEqual(repr(rule), "COMPBINOP(FUNCTION(strlen(VARIABLE('Node.Name'),)) OP_GT INTEGER(20))")
self.assertEqual(self.flt.filter(rule, self.test_msg1), True)
rule = self.psr.parse('(strlen(Node.Name) > 30)')
self.assertEqual(repr(rule), "COMPBINOP(FUNCTION(strlen(VARIABLE('Node.Name'),)) OP_GT INTEGER(30))")
self.assertEqual(self.flt.filter(rule, self.test_msg1), False)
rule = self.psr.parse('(strlen(Node.Name) < 20)')
self.assertEqual(repr(rule), "COMPBINOP(FUNCTION(strlen(VARIABLE('Node.Name'),)) OP_LT INTEGER(20))")
self.assertEqual(self.flt.filter(rule, self.test_msg1), True)
rule = self.psr.parse('(strlen(Node.Name) < 10)')
self.assertEqual(repr(rule), "COMPBINOP(FUNCTION(strlen(VARIABLE('Node.Name'),)) OP_LT INTEGER(10))")
self.assertEqual(self.flt.filter(rule, self.test_msg1), False)
rule = self.psr.parse('(time() > 500.12)')
self.assertEqual(repr(rule), "COMPBINOP(FUNCTION(time()) OP_GT FLOAT(500.12))")
self.assertEqual(self.flt.filter(rule, self.test_msg1), True)
def setUp(self):
self.flt = DataObjectFilter()
self.psr = PynspectFilterParser()
self.psr.build()
self.cpl = IDEAFilterCompiler()
class Config():
addrGroups = dict()
actions = dict()
rules = list()
parser = None
compiler = None
def __init__(self, path, trap=None, warden=None):
"""
:param path Path to YAML configuration file
:param trap Instance of TRAP client used in TrapAction
:param warden Instance of Warden Client used in WardenAction
"""
# Parse given config file
with open(path, 'r') as f:
self.conf = Parser(f)
if not self.conf:
raise Exception(
"Loading YAML file ({0}) failed. Isn't it empty?".format(path))
# Build parser
self.parser = PynspectFilterParser()
self.parser.build()
self.compiler = IDEAFilterCompiler()
self.addrGroups = dict()
# Create all address groups if there are any
if "addressgroups" in self.conf:
for i in self.conf["addressgroups"]:
self.addrGroups[i["id"]] = AddressGroup(i)
self.actions = dict()
# Parse and instantiate all custom actions
if "custom_actions" in self.conf:
for i in self.conf["custom_actions"]:
if "mark" in i:
from .actions.Mark import MarkAction
self.actions[i["id"]] = MarkAction(i)
elif "mongo" in i:
from .actions.Mongo import MongoAction
self.actions[i["id"]] = MongoAction(i)
elif "email" in i:
from .actions.Email import EmailAction
self.actions[i["id"]] = EmailAction(i)
elif "file" in i:
from .actions.File import FileAction
self.actions[i["id"]] = FileAction(i)
elif "warden" in i:
"""
Pass Warden Client instance to the Warden action
"""
from .actions.Warden import WardenAction
self.actions[i["id"]] = WardenAction(i, warden)
elif "trap" in i:
"""
Pass TRAP context instance to the TRAP action
"""
from .actions.Trap import TrapAction
self.actions[i["id"]] = TrapAction(i, trap)
elif "drop" in i:
logger.warning(
"Drop action mustn't be specified in custom_actions!")
continue
else:
raise Exception("Undefined action: " + str(i))
self.actions["drop"] = DropAction()
self.rules = list()
# Parse all rules and match them with actions and address groups
# There must be at least one rule (mandatory field)
if "rules" in self.conf:
if self.conf["rules"]:
for i in self.conf["rules"]:
self.rules.append(
Rule(i,
self.actions,
self.addrGroups,
parser=self.parser,
compiler=self.compiler))
if not self.rules:
raise Exception(
"YAML file should contain at least one `rule` in `rules`.")
else:
raise Exception("YAML file must contain `rules`.")
def match(self, msg):
"""
Check if msg matches rules from config file.
Return a list of bool values representing results of all tested rules.
"""
results = []
actionsDone = []
try:
for rule in self.rules:
self.clearActionLog()
res = rule.filter(msg)
#logger.debug("Filter by rule: %s \n message: %s\n\nresult: %s", rule, msg, res)
#print("Filter by rule: %s \n message: %s\n\nresult: %s" % (rule.rule(), msg, res))
results.append(res)
tmp_msg = copy.deepcopy(msg)
if res:
# condition is True
rule.actions(tmp_msg)
logger.info("action running")
else:
# condition is False
rule.elseactions(tmp_msg)
logger.info("else action running")
actionsDone.append(self.getActionLog())
except DropMsg:
# This exception breaks the processing of rule list.
pass
return (results, actionsDone)
def getActionLog(self):
return Action.actionLog
def clearActionLog(self):
Action.actionLog = []
def loglevel(self):
"""Get logging level
CRITICAL 50
ERROR 40
WARNING 30
INFO 20
DEBUG 10
NOTSET 0
"""
try:
return (self.conf["reporter"]["loglevel"]) * 10
except:
return 30
def __str__(self):
ag = "\n".join([str(self.addrGroups[key]) for key in self.addrGroups])
a = "\n".join(
[key + ":\n\t" + str(self.actions[key]) for key in self.actions])
r = "\n".join([str(val) for val in self.rules])
string = "Address Groups:\n{0}\n----------------\nCustom Actions:\n{1}\n----------------\nRules:\n{2}\n".format(
ag, a, r)
return string
class TestDataObjectFilterIDEA(unittest.TestCase):
"""
Unit test class for testing the :py:mod:`pynspect.filters` module.
"""
test_msg1 = {
"ID" : "e214d2d9-359b-443d-993d-3cc5637107a0",
"WinEndTime" : "2016-06-21 11:25:01Z",
"ConnCount" : 2,
"Source" : [
{
"IP4" : [
"188.14.166.39"
]
}
],
"Format" : "IDEA0",
"WinStartTime" : "2016-06-21 11:20:01Z",
"_CESNET" : {
"StorageTime" : 1466508305
},
"Target" : [
{
"IP4" : [
"195.113.165.128/25"
],
"Port" : [
"22"
],
"Proto" : [
"tcp",
"ssh"
],
"Anonymised" : True
}
],
"Note" : "SSH login attempt",
"DetectTime" : "2016-06-21 13:08:27Z",
"Node" : [
{
"Name" : "cz.cesnet.mentat.warden_filer",
"Type" : [
"Relay"
]
},
{
"AggrWin" : "00:05:00",
"Type" : [
"Connection",
"Honeypot",
"Recon"
],
"SW" : [
"Kippo"
],
"Name" : "cz.uhk.apate.cowrie"
}
],
"Category" : [
"Attempt.Login"
]
}
def setUp(self):
self.flt = DataObjectFilter()
self.psr = PynspectFilterParser()
self.psr.build()
self.cpl = IDEAFilterCompiler()
self.msg_idea = lite.Idea(self.test_msg1)
def build_rule(self, rule_str):
"""
Build and compile rule tree from given rule string.
"""
rule = self.psr.parse(rule_str)
rule = self.cpl.compile(rule)
return rule
def check_rule(self, rule):
"""
Check given rule against internal test message and filter.
"""
return self.flt.filter(rule, self.msg_idea)
def test_01_basic_logical(self):
"""
Perform filtering tests with basic logical expressions.
"""
self.maxDiff = None
rule = LogicalBinOpRule('OP_AND', ConstantRule(True), ConstantRule(True))
self.assertEqual(self.check_rule(rule), True)
rule = LogicalBinOpRule('OP_AND', ConstantRule(True), ConstantRule(False))
self.assertEqual(self.check_rule(rule), False)
rule = LogicalBinOpRule('OP_AND', ConstantRule(False), ConstantRule(True))
self.assertEqual(self.check_rule(rule), False)
rule = LogicalBinOpRule('OP_AND', ConstantRule(False), ConstantRule(False))
self.assertEqual(self.check_rule(rule), False)
rule = LogicalBinOpRule('OP_OR', ConstantRule(True), ConstantRule(True))
self.assertEqual(self.check_rule(rule), True)
rule = LogicalBinOpRule('OP_OR', ConstantRule(True), ConstantRule(False))
self.assertEqual(self.check_rule(rule), True)
rule = LogicalBinOpRule('OP_OR', ConstantRule(False), ConstantRule(True))
self.assertEqual(self.check_rule(rule), True)
rule = LogicalBinOpRule('OP_OR', ConstantRule(False), ConstantRule(False))
self.assertEqual(self.check_rule(rule), False)
rule = LogicalBinOpRule('OP_XOR', ConstantRule(True), ConstantRule(True))
self.assertEqual(self.check_rule(rule), False)
rule = LogicalBinOpRule('OP_XOR', ConstantRule(True), ConstantRule(False))
self.assertEqual(self.check_rule(rule), True)
rule = LogicalBinOpRule('OP_XOR', ConstantRule(False), ConstantRule(True))
self.assertEqual(self.check_rule(rule), True)
rule = LogicalBinOpRule('OP_XOR', ConstantRule(False), ConstantRule(False))
self.assertEqual(self.check_rule(rule), False)
rule = UnaryOperationRule('OP_NOT', ConstantRule(True))
self.assertEqual(self.check_rule(rule), False)
rule = UnaryOperationRule('OP_NOT', ConstantRule(False))
self.assertEqual(self.check_rule(rule), True)
rule = UnaryOperationRule('OP_NOT', VariableRule("Target.Anonymised"))
self.assertEqual(self.check_rule(rule), False)
def test_02_basic_comparison(self):
"""
Perform filtering tests with basic comparison operations.
"""
self.maxDiff = None
rule = ComparisonBinOpRule('OP_EQ', VariableRule("ID"), ConstantRule("e214d2d9-359b-443d-993d-3cc5637107a0"))
self.assertEqual(self.check_rule(rule), True)
rule = ComparisonBinOpRule('OP_EQ', VariableRule("ID"), ConstantRule("e214d2d9-359b-443d-993d-3cc5637107"))
self.assertEqual(self.check_rule(rule), False)
rule = ComparisonBinOpRule('OP_NE', VariableRule("ID"), ConstantRule("e214d2d9-359b-443d-993d-3cc5637107a0"))
self.assertEqual(self.check_rule(rule), False)
rule = ComparisonBinOpRule('OP_NE', VariableRule("ID"), ConstantRule("e214d2d9-359b-443d-993d-3cc5637107"))
self.assertEqual(self.check_rule(rule), True)
rule = ComparisonBinOpRule('OP_LIKE', VariableRule("ID"), ConstantRule("e214d2d9"))
self.assertEqual(self.check_rule(rule), True)
rule = ComparisonBinOpRule('OP_LIKE', VariableRule("ID"), ConstantRule("xxxxxxxx"))
self.assertEqual(self.check_rule(rule), False)
rule = ComparisonBinOpRule('OP_IN', VariableRule("Category"), ListRule(ConstantRule("Phishing"), ListRule(ConstantRule("Attempt.Login"))))
self.assertEqual(self.check_rule(rule), True)
rule = ComparisonBinOpRule('OP_IN', VariableRule("Category"), ListRule(ConstantRule("Phishing"), ListRule(ConstantRule("Spam"))))
self.assertEqual(self.check_rule(rule), False)
rule = ComparisonBinOpRule('OP_IS', VariableRule("Category"), ListRule(ConstantRule("Attempt.Login")))
self.assertEqual(self.check_rule(rule), True)
rule = ComparisonBinOpRule('OP_IS', VariableRule("Category"), ListRule(ConstantRule("Phishing"), ListRule(ConstantRule("Attempt.Login"))))
self.assertEqual(self.check_rule(rule), False)
rule = ComparisonBinOpRule('OP_EQ', VariableRule("ConnCount"), IntegerRule(2))
self.assertEqual(self.check_rule(rule), True)
rule = ComparisonBinOpRule('OP_EQ', VariableRule("ConnCount"), IntegerRule(4))
self.assertEqual(self.check_rule(rule), False)
rule = ComparisonBinOpRule('OP_NE', VariableRule("ConnCount"), IntegerRule(2))
self.assertEqual(self.check_rule(rule), False)
rule = ComparisonBinOpRule('OP_NE', VariableRule("ConnCount"), IntegerRule(4))
self.assertEqual(self.check_rule(rule), True)
rule = ComparisonBinOpRule('OP_GT', VariableRule("ConnCount"), IntegerRule(2))
self.assertEqual(self.check_rule(rule), False)
rule = ComparisonBinOpRule('OP_GT', VariableRule("ConnCount"), IntegerRule(1))
self.assertEqual(self.check_rule(rule), True)
rule = ComparisonBinOpRule('OP_GE', VariableRule("ConnCount"), IntegerRule(2))
self.assertEqual(self.check_rule(rule), True)
rule = ComparisonBinOpRule('OP_GE', VariableRule("ConnCount"), IntegerRule(1))
self.assertEqual(self.check_rule(rule), True)
rule = ComparisonBinOpRule('OP_GE', VariableRule("ConnCount"), IntegerRule(3))
self.assertEqual(self.check_rule(rule), False)
rule = ComparisonBinOpRule('OP_LT', VariableRule("ConnCount"), IntegerRule(2))
self.assertEqual(self.check_rule(rule), False)
rule = ComparisonBinOpRule('OP_LT', VariableRule("ConnCount"), IntegerRule(3))
self.assertEqual(self.check_rule(rule), True)
rule = ComparisonBinOpRule('OP_LE', VariableRule("ConnCount"), IntegerRule(2))
self.assertEqual(self.check_rule(rule), True)
rule = ComparisonBinOpRule('OP_LE', VariableRule("ConnCount"), IntegerRule(3))
self.assertEqual(self.check_rule(rule), True)
rule = ComparisonBinOpRule('OP_LE', VariableRule("ConnCount"), IntegerRule(1))
self.assertEqual(self.check_rule(rule), False)
def test_03_parsed_comparison(self):
"""
Perform filtering tests with basic parsed comparison operations.
"""
self.maxDiff = None
rule = self.build_rule('ID == "e214d2d9-359b-443d-993d-3cc5637107a0"')
self.assertEqual(self.check_rule(rule), True)
rule = self.build_rule('ID eq "e214d2d9-359b-443d-993d-3cc5637107"')
self.assertEqual(self.check_rule(rule), False)
rule = self.build_rule('ID != "e214d2d9-359b-443d-993d-3cc5637107a0"')
self.assertEqual(self.check_rule(rule), False)
rule = self.build_rule('ID ne "e214d2d9-359b-443d-993d-3cc5637107"')
self.assertEqual(self.check_rule(rule), True)
rule = self.build_rule('ID like "e214d2d9"')
self.assertEqual(self.check_rule(rule), True)
rule = self.build_rule('ID LIKE "xxxxxxxx"')
self.assertEqual(self.check_rule(rule), False)
rule = self.build_rule('Category in ["Phishing" , "Attempt.Login"]')
self.assertEqual(self.check_rule(rule), True)
rule = self.build_rule('Category IN ["Phishing" , "Spam"]')
self.assertEqual(self.check_rule(rule), False)
rule = self.build_rule('Category is ["Attempt.Login"]')
self.assertEqual(self.check_rule(rule), True)
rule = self.build_rule('Category IS ["Phishing" , "Attempt.Login"]')
self.assertEqual(self.check_rule(rule), False)
rule = self.build_rule('ConnCount == 2')
self.assertEqual(self.check_rule(rule), True)
rule = self.build_rule('ConnCount eq 4')
self.assertEqual(self.check_rule(rule), False)
rule = self.build_rule('ConnCount != 2')
self.assertEqual(self.check_rule(rule), False)
rule = self.build_rule('ConnCount ne 4')
self.assertEqual(self.check_rule(rule), True)
rule = self.build_rule('ConnCount > 2')
self.assertEqual(self.check_rule(rule), False)
rule = self.build_rule('ConnCount gt 1')
self.assertEqual(self.check_rule(rule), True)
rule = self.build_rule('ConnCount >= 2')
self.assertEqual(self.check_rule(rule), True)
rule = self.build_rule('ConnCount ge 1')
self.assertEqual(self.check_rule(rule), True)
rule = self.build_rule('ConnCount GE 3')
self.assertEqual(self.check_rule(rule), False)
rule = self.build_rule('ConnCount < 2')
self.assertEqual(self.check_rule(rule), False)
rule = self.build_rule('ConnCount lt 3')
self.assertEqual(self.check_rule(rule), True)
rule = self.build_rule('ConnCount <= 2')
self.assertEqual(self.check_rule(rule), True)
rule = self.build_rule('ConnCount le 3')
self.assertEqual(self.check_rule(rule), True)
rule = self.build_rule('ConnCount LE 1')
self.assertEqual(self.check_rule(rule), False)
def test_04_basic_math(self):
"""
Perform filtering tests with basic math operations.
"""
self.maxDiff = None
rule = MathBinOpRule('OP_PLUS', VariableRule("ConnCount"), IntegerRule(1))
self.assertEqual(self.check_rule(rule), 3)
rule = MathBinOpRule('OP_MINUS', VariableRule("ConnCount"), IntegerRule(1))
self.assertEqual(self.check_rule(rule), 1)
rule = MathBinOpRule('OP_TIMES', VariableRule("ConnCount"), IntegerRule(5))
self.assertEqual(self.check_rule(rule), 10)
rule = MathBinOpRule('OP_DIVIDE', VariableRule("ConnCount"), IntegerRule(2))
self.assertEqual(self.check_rule(rule), 1)
rule = MathBinOpRule('OP_MODULO', VariableRule("ConnCount"), IntegerRule(2))
self.assertEqual(self.check_rule(rule), 0)
def test_05_parsed_math(self):
"""
Perform filtering tests with parsed math operations.
"""
self.maxDiff = None
rule = self.build_rule('ConnCount + 1')
self.assertEqual(self.check_rule(rule), 3)
rule = self.build_rule('ConnCount - 1')
self.assertEqual(self.check_rule(rule), 1)
rule = self.build_rule('ConnCount * 5')
self.assertEqual(self.check_rule(rule), 10)
rule = self.build_rule('ConnCount / 2')
self.assertEqual(self.check_rule(rule), 1)
rule = self.build_rule('ConnCount % 2')
self.assertEqual(self.check_rule(rule), 0)
def test_06_advanced_filters(self):
"""
Perform advanced filtering tests.
"""
self.maxDiff = None
rule = self.build_rule('DetectTime + 3600')
self.assertEqual(repr(rule), "MATHBINOP(VARIABLE('DetectTime') OP_PLUS TIMEDELTA(datetime.timedelta(0, 3600)))")
expected_res = (datetime.datetime(2016, 6, 21, 13, 8, 27) + datetime.timedelta(seconds = 3600))
self.assertEqual(self.check_rule(rule), expected_res)
rule = self.build_rule('(ConnCount + 10) > 11')
self.assertEqual(repr(rule), "COMPBINOP(MATHBINOP(VARIABLE('ConnCount') OP_PLUS INTEGER(10)) OP_GT INTEGER(11))")
self.assertEqual(self.check_rule(rule), True)
rule = self.build_rule('(ConnCount + 3) < 5')
self.assertEqual(repr(rule), "COMPBINOP(MATHBINOP(VARIABLE('ConnCount') OP_PLUS INTEGER(3)) OP_LT INTEGER(5))")
self.assertEqual(self.check_rule(rule), False)
rule = self.build_rule('((ConnCount + 3) < 5) or ((ConnCount + 10) > 11)')
self.assertEqual(repr(rule), "LOGBINOP(COMPBINOP(MATHBINOP(VARIABLE('ConnCount') OP_PLUS INTEGER(3)) OP_LT INTEGER(5)) OP_OR COMPBINOP(MATHBINOP(VARIABLE('ConnCount') OP_PLUS INTEGER(10)) OP_GT INTEGER(11)))")
self.assertEqual(self.check_rule(rule), True)
rule = self.build_rule('(DetectTime == 2016-06-21T13:08:27Z)')
self.assertEqual(repr(rule), "COMPBINOP(VARIABLE('DetectTime') OP_EQ DATETIME(datetime.datetime(2016, 6, 21, 13, 8, 27)))")
self.assertEqual(self.check_rule(rule), True)
rule = self.build_rule('(DetectTime != 2016-06-21T13:08:27Z)')
self.assertEqual(repr(rule), "COMPBINOP(VARIABLE('DetectTime') OP_NE DATETIME(datetime.datetime(2016, 6, 21, 13, 8, 27)))")
self.assertEqual(self.check_rule(rule), False)
rule = self.build_rule('(DetectTime >= 2016-06-21T14:08:27Z)')
self.assertEqual(repr(rule), "COMPBINOP(VARIABLE('DetectTime') OP_GE DATETIME(datetime.datetime(2016, 6, 21, 14, 8, 27)))")
self.assertEqual(self.check_rule(rule), False)
rule = self.build_rule('(DetectTime <= 2016-06-21T14:08:27Z)')
self.assertEqual(repr(rule), "COMPBINOP(VARIABLE('DetectTime') OP_LE DATETIME(datetime.datetime(2016, 6, 21, 14, 8, 27)))")
self.assertEqual(self.check_rule(rule), True)
rule = self.build_rule('DetectTime < (utcnow() + 05:00:00)')
self.assertEqual(repr(rule), "COMPBINOP(VARIABLE('DetectTime') OP_LT MATHBINOP(FUNCTION(utcnow()) OP_PLUS TIMEDELTA(datetime.timedelta(0, 18000))))")
self.assertEqual(self.check_rule(rule), True)
rule = self.build_rule('DetectTime > (utcnow() - 05:00:00)')
self.assertEqual(repr(rule), "COMPBINOP(VARIABLE('DetectTime') OP_GT MATHBINOP(FUNCTION(utcnow()) OP_MINUS TIMEDELTA(datetime.timedelta(0, 18000))))")
self.assertEqual(self.check_rule(rule), False)
rule = self.build_rule('(Source.IP4 == 188.14.166.39)')
self.assertEqual(repr(rule), "COMPBINOP(VARIABLE('Source.IP4') OP_EQ IPV4(IP4('188.14.166.39')))")
self.assertEqual(self.check_rule(rule), True)
rule = self.build_rule('(Source.IP4 in ["188.14.166.39","188.14.166.40","188.14.166.41"])')
self.assertEqual(repr(rule), "COMPBINOP(VARIABLE('Source.IP4') OP_IN IPLIST(IPV4(IP4('188.14.166.39')), IPV4(IP4('188.14.166.40')), IPV4(IP4('188.14.166.41'))))")
self.assertEqual(self.check_rule(rule), True)
# list with CIDR addresses
rule = self.build_rule('(Source.IP4 in ["188.14.166.0/24","10.0.0.0/8","189.14.166.41"])')
self.assertEqual(repr(rule), "COMPBINOP(VARIABLE('Source.IP4') OP_IN IPLIST(IPV4(IP4Net('188.14.166.0/24')), IPV4(IP4Net('10.0.0.0/8')), IPV4(IP4('189.14.166.41'))))")
self.assertEqual(self.check_rule(rule), True)
def test_06_shortcuts(self):
"""
Perform tests of shortcut methods.
"""
self.maxDiff = None
# Let the shortcut method initialize everything.
flt = DataObjectFilter(
parser = PynspectFilterParser,
compiler = IDEAFilterCompiler
)
rule = flt.prepare('(Source.IP4 == 188.14.166.39)')
self.assertEqual(repr(rule), "COMPBINOP(VARIABLE('Source.IP4') OP_EQ IPV4(IP4('188.14.166.39')))")
self.assertEqual(self.check_rule(rule), True)
# Create parser and compiler instances by hand, but register them into filter.
cpl = IDEAFilterCompiler()
psr = PynspectFilterParser()
psr.build()
flt = DataObjectFilter(
parser = psr,
compiler = cpl
)
rule = flt.prepare('(Source.IP4 == 188.14.166.39)')
self.assertEqual(repr(rule), "COMPBINOP(VARIABLE('Source.IP4') OP_EQ IPV4(IP4('188.14.166.39')))")
self.assertEqual(self.check_rule(rule), True)
def __init__(self, path, trap=None, warden=None):
"""
:param path Path to YAML configuration file
:param trap Instance of TRAP client used in TrapAction
:param warden Instance of Warden Client used in WardenAction
"""
# Parse given config file
with open(path, 'r') as f:
self.conf = Parser(f)
if not self.conf:
raise Exception(
"Loading YAML file ({0}) failed. Isn't it empty?".format(path))
# Build parser
self.parser = PynspectFilterParser()
self.parser.build()
self.compiler = IDEAFilterCompiler()
self.addrGroups = dict()
# Create all address groups if there are any
if "addressgroups" in self.conf:
for i in self.conf["addressgroups"]:
self.addrGroups[i["id"]] = AddressGroup(i)
self.actions = dict()
# Parse and instantiate all custom actions
if "custom_actions" in self.conf:
for i in self.conf["custom_actions"]:
if "mark" in i:
from .actions.Mark import MarkAction
self.actions[i["id"]] = MarkAction(i)
elif "mongo" in i:
from .actions.Mongo import MongoAction
self.actions[i["id"]] = MongoAction(i)
elif "email" in i:
from .actions.Email import EmailAction
self.actions[i["id"]] = EmailAction(i)
elif "file" in i:
from .actions.File import FileAction
self.actions[i["id"]] = FileAction(i)
elif "warden" in i:
"""
Pass Warden Client instance to the Warden action
"""
from .actions.Warden import WardenAction
self.actions[i["id"]] = WardenAction(i, warden)
elif "trap" in i:
"""
Pass TRAP context instance to the TRAP action
"""
from .actions.Trap import TrapAction
self.actions[i["id"]] = TrapAction(i, trap)
elif "drop" in i:
logger.warning(
"Drop action mustn't be specified in custom_actions!")
continue
else:
raise Exception("Undefined action: " + str(i))
self.actions["drop"] = DropAction()
self.rules = list()
# Parse all rules and match them with actions and address groups
# There must be at least one rule (mandatory field)
if "rules" in self.conf:
if self.conf["rules"]:
for i in self.conf["rules"]:
self.rules.append(
Rule(i,
self.actions,
self.addrGroups,
parser=self.parser,
compiler=self.compiler))
if not self.rules:
raise Exception(
"YAML file should contain at least one `rule` in `rules`.")
else:
raise Exception("YAML file must contain `rules`.")
# Always prefer setuptools over distutils
from setuptools import setup, find_packages
# To use a consistent encoding
from codecs import open
#
# Import local version of pynspect library, so that we can insert correct version
# number into documentation.
#
sys.path.insert(0, os.path.abspath('.'))
import pynspect
# Generate parsetab.py in advance so it can go into package.
from pynspect.gparser import PynspectFilterParser
parser = PynspectFilterParser()
parser.build()
here = os.path.abspath(os.path.dirname(__file__))
#-------------------------------------------------------------------------------
# Get the long description from the README file
with open(os.path.join(here, 'README.rst'), encoding='utf-8') as f:
long_description = f.read()
setup(
name = 'pynspect',
version = pynspect.__version__,
description = 'Python data inspection library',
long_description = long_description,