def runTest(self):
import pytrap
import os
urtempl = "ipaddr IP,uint16 PORT"
c = pytrap.TrapCtx()
c.init(["-i", "f:/tmp/pytrap_test"], 0, 1)
c.setDataFmt(0, pytrap.FMT_UNIREC, urtempl)
t = pytrap.UnirecTemplate("ipaddr IP,uint16 PORT")
t.createMessage()
t.IP = pytrap.UnirecIPAddr("192.168.0.1")
t.PORT = 123
c.send(t.getData())
c.sendFlush()
c.finalize()
c = pytrap.TrapCtx()
c.init(["-i", "f:/tmp/pytrap_test"], 1)
c.setRequiredFmt(0, pytrap.FMT_UNIREC, urtempl)
data = c.recv()
t = pytrap.UnirecTemplate(urtempl)
t.setData(data)
self.assertEqual(t.IP, pytrap.UnirecIPAddr("192.168.0.1"))
self.assertEqual(t.PORT, 123)
c.finalize()
os.unlink("/tmp/pytrap_test")
def main():
# Select input spec for basic and smtp interface
basic_spec = "ipaddr DST_IP,ipaddr SRC_IP,uint64 BYTES,uint64 LINK_BIT_FIELD,time TIME_FIRST,time TIME_LAST,uint32 PACKETS,uint16 DST_PORT,uint16 SRC_PORT,uint8 DIR_BIT_FIELD,uint8 PROTOCOL,uint8 TCP_FLAGS,uint8 TOS,uint8 TTL"
smtp_spec = "ipaddr DST_IP,ipaddr SRC_IP,uint64 BYTES,uint64 LINK_BIT_FIELD,time TIME_FIRST,time TIME_LAST,uint32 PACKETS,uint32 SMTP_2XX_STAT_CODE_COUNT,uint32 SMTP_3XX_STAT_CODE_COUNT,uint32 SMTP_4XX_STAT_CODE_COUNT,uint32 SMTP_5XX_STAT_CODE_COUNT,uint32 SMTP_COMMAND_FLAGS,uint32 SMTP_MAIL_CMD_COUNT,uint32 SMTP_RCPT_CMD_COUNT,uint32 SMTP_STAT_CODE_FLAGS,uint16 DST_PORT,uint16 SRC_PORT,uint8 DIR_BIT_FIELD,uint8 PROTOCOL,uint8 TCP_FLAGS,uint8 TOS,uint8 TTL,string SMTP_DOMAIN,string SMTP_FIRST_RECIPIENT,string SMTP_FIRST_SENDER"
# Create a new trap context
trap = pytrap.TrapCtx()
"""
Trap initialization for two input interfaces, and no output interface
"""
trap.init(sys.argv, 2, 1)
# Set up required format to accept any unirec format.
trap.setRequiredFmt(BASIC_IF, pytrap.FMT_UNIREC,
basic_spec) # Refers to flows without SMTP headers
trap.setRequiredFmt(SMTP_IF, pytrap.FMT_UNIREC,
smtp_spec) # Refers to flows with SMTP headers
basic_rec = pytrap.UnirecTemplate(basic_spec)
smtp_rec = pytrap.UnirecTemplate(smtp_spec)
trap.setDataFmt(0, pytrap.FMT_JSON, "smtp_alert")
log.info("Daemon: Trap initialized.")
detector = SpamDetection(trap)
# Data synchronized queues
flow_queue = Queue() # Synchronize input flows
# Create workers for each receiver
basic_rcv = Thread(name="basic_receiver",
target=fetch_data,
args=(trap, basic_rec, BASIC_IF, flow_queue))
smtp_rcv = Thread(name="smtp_receiver",
target=fetch_data,
args=(trap, smtp_rec, SMTP_IF, flow_queue))
# Handle the received data from receivers
data_handler = Thread(name="data_handler",
target=data_handling,
args=(detector, flow_queue))
# Run multi-receiver
basic_rcv.start()
smtp_rcv.start()
data_handler.start()
log.info("Daemon: Multi-receiver started.")
# Start detector
detector.start()
# Wait until the daemon is requested to stop by releasing the lock (by signal handler)
g.stop_lock.acquire()
signal.signal(signal.SIGINT, signal.SIG_DFL)
signal.signal(signal.SIGTERM, signal.SIG_DFL)
signal.signal(signal.SIGABRT, signal.SIG_DFL)
log.info("Stopping running components ...")
g.is_running = False
detector.stop()
# Join the threads
basic_rcv.join()
smtp_rcv.join()
# Stop data_handler
flow_queue.put(None)
data_handler.join()
detector.join()
# Free allocated memory
trap.finalize()
log.info("***** Finished, main thread exiting. *****")
logging.shutdown()
sys.exit(0)
def runTest(self):
import pytrap
a = pytrap.UnirecTemplate("ipaddr SRC_IP,time TIME_FIRST,uint32 ABC,uint32 BCD,string TEXT,bytes STREAMBYTES")
data = bytearray(b'\x00\x00\x00\x00\x00\x00\x00\x00\x0A\x00\x00\x01\xff\xff\xff\xff\xc8\x76\xbe\xff\xe3\x2b\x6c\x57\x00\x00\x00\x01\x00\x00\x00\x02\x06\x00\x04\x00\x00\x00\x06\x00abcdef\xde\xad\xfe\xed')
self.assertEqual(a.recSize(data), 50)
self.assertEqual(a.recFixlenSize(), 40)
self.assertEqual(a.recVarlenSize(data), 10)
self.assertRaises(TypeError, a.recSize) # recSize can't by called without arguments unless data was set by setData
self.assertRaises(TypeError, a.recVarlenSize)
a.setData(data)
self.assertEqual(a.recSize(), 50) # now it should be OK
self.assertEqual(a.recVarlenSize(), 10)
a = pytrap.UnirecTemplate("uint32 X,ipaddr IP")
data = bytearray(100) # Allocate larger byte array than necessary
a.setData(data)
self.assertEqual(a.recSize(), 20)
self.assertEqual(a.recFixlenSize(), 20)
self.assertEqual(a.recVarlenSize(), 0)
a = pytrap.UnirecTemplate("string STR")
data = bytearray(65535)
data[0:1] = b'\x00\x00\xfb\xff' # Set offset (0) and length (65531 - maximum), let data be set to zeros
a.setData(data)
self.assertEqual(a.recSize(), 65535)
self.assertEqual(a.recFixlenSize(), 4)
self.assertEqual(a.recVarlenSize(), 65531)
def runTest(self):
import pytrap
a = pytrap.UnirecTemplate(
"ipaddr SRC_IP,time TIME_FIRST,uint32 ABC,uint32 BCD,string TEXT,bytes STREAMBYTES"
)
a.createMessage(100)
self.assertEqual(
a.getFieldsDict(), {
'SRC_IP': 0,
'STREAMBYTES': 5,
'TEXT': 4,
'TIME_FIRST': 1,
'ABC': 2,
'BCD': 3
})
a.SRC_IP = pytrap.UnirecIPAddr("1.2.3.4")
a.TIME_FIRST = pytrap.UnirecTime(123456)
a.ABC = 1234
a.BCD = 54321
a.STREAMBYTES = bytearray(b"streamtext")
a.TEXT = "stringmuj text"
tc = a.strRecord()
for i in range(100):
self.assertEqual(tc, a.strRecord())
a = pytrap.UnirecTemplate(
"ipaddr DST_IP,time TIME_FIRST,uint32 BCD,string TEXT2,bytes STREAMBYTES"
)
self.assertEqual(a.getFieldsDict(), {
'BCD': 3,
'DST_IP': 6,
'TIME_FIRST': 1,
'STREAMBYTES': 5,
'TEXT2': 7
})
try:
a.DST_IP = pytrap.UnirecIPAddr("4.4.4.4")
except:
pass
else:
self.fail("Data was not set, this should raise exception.")
a.createMessage(100)
try:
a.SRC_IP = pytrap.UnirecIPAddr("4.4.4.4")
except:
pass
else:
self.fail("This template has no SRC_IP.")
a.DST_IP = pytrap.UnirecIPAddr("4.4.4.4")
a.STREAMBYTES = bytearray(b"hello")
valdict = {}
for name, value in a:
valdict[name] = value
self.assertEqual(
valdict, {
'STREAMBYTES': bytearray(b'hello'),
'DST_IP': pytrap.UnirecIPAddr('4.4.4.4'),
'TIME_FIRST': pytrap.UnirecTime(0),
'TEXT2': '',
'BCD': 0
})
def interface_loop(self, index, interface):
"""Consuming loop for a single interface.
Args:
index (int): Unique interface index.
interface ([type]): Interface specification in BOTA config format.
"""
_, format_spec = self.trap.getDataFmt(index)
ur = pytrap.UnirecTemplate(format_spec)
while True:
try:
data = self.trap.recv(index)
except pytrap.FormatChanged as e:
_, format_spec = self.trap.getDataFmt(index)
ur = pytrap.UnirecTemplate(format_spec)
data = e.data
# EOF
if len(data) <= 1:
message = {"type": "eof", "data": {}}
with self.lock:
self.callback(message)
break
ur.setData(data)
data = {}
for k in ur.getFieldsDict():
v = getattr(ur, k)
# mapping
if isinstance(v, (pytrap.UnirecIPAddr, pytrap.UnirecMACAddr)):
v = str(v)
if isinstance(v, pytrap.UnirecTime):
v = v.format("%Y-%m-%dT%H:%M:%S.%f")
if isinstance(v, bytearray):
v = v.hex()
if k == "PPI_PKT_TIMES":
v = [x.format("%Y-%m-%dT%H:%M:%S.%f") for x in v]
if isinstance(v, list) and not isinstance(v, str):
v = "[" + "|".join(map(str, v)) + "]"
data[k.lower()] = v
message = {"type": interface["type"], "data": data}
with self.lock:
self.callback(message)
def runTest(self):
import pytrap
a = pytrap.UnirecTemplate(
"ipaddr SRC_IP,time TIME_FIRST,uint32 ABC,uint32 BCD,string TEXT,bytes STREAMBYTES"
)
a.createMessage(100)
self.assertEqual(
set(a.getFieldsDict().keys()),
set(['SRC_IP', 'STREAMBYTES', 'TEXT', 'TIME_FIRST', 'ABC', 'BCD']))
a.SRC_IP = pytrap.UnirecIPAddr("1.2.3.4")
a.TIME_FIRST = pytrap.UnirecTime(123456)
a.ABC = 1234
a.BCD = 54321
a.STREAMBYTES = bytearray(b"streamtext")
a.TEXT = "stringmuj text"
tc = a.strRecord()
for i in range(100):
self.assertEqual(tc, a.strRecord())
a = pytrap.UnirecTemplate(
"ipaddr DST_IP,time TIME_FIRST,uint32 BCD,string TEXT2,bytes STREAMBYTES"
)
self.assertEqual(
set(a.getFieldsDict().keys()),
set(['DST_IP', 'TIME_FIRST', 'BCD', 'STREAMBYTES', 'TEXT2']))
# Data was not set, this should raise exception.
with self.assertRaises(pytrap.TrapError):
a.DST_IP = pytrap.UnirecIPAddr("4.4.4.4")
a.createMessage(100)
# This template has no SRC_IP.
with self.assertRaises(AttributeError):
a.SRC_IP = pytrap.UnirecIPAddr("4.4.4.4")
a.DST_IP = pytrap.UnirecIPAddr("4.4.4.4")
a.STREAMBYTES = bytearray(b"hello")
valdict = {}
for name, value in a:
valdict[name] = value
self.assertEqual(
valdict, {
'STREAMBYTES': bytearray(b'hello'),
'DST_IP': pytrap.UnirecIPAddr('4.4.4.4'),
'TIME_FIRST': pytrap.UnirecTime(0),
'TEXT2': '',
'BCD': 0
})
def _receive_data(detector_class):
"""
Fetches data from trap context and aggregates them
Arguments:
trap pytrap.trapCtx
iface_num int IP/URL/DNS
queue Queue
"""
my_iface_num = detector_class.iface_num
my_queue = detector_class.queue
while not stop:
try:
data = trap.recv(my_iface_num)
except pytrap.FormatMismatch:
print(
"Error: output and input interfaces data format or data specifier mismatch"
)
break
except pytrap.FormatChanged as e:
fmttype, inputspec = trap.getDataFmt(my_iface_num)
detector_class.ur_input = pytrap.UnirecTemplate(inputspec)
data = e.data
except pytrap.Terminated:
print("Terminated TRAP.")
break
except pytrap.TrapError:
print("TrapError.")
break
if len(data) <= 1:
break
my_queue.put(data)
def runTest(self):
import pytrap
a = pytrap.UnirecTemplate(
"ipaddr SRC_IP,time TIME_FIRST,uint32 ABC,uint32 BCD,string TEXT,bytes STREAMBYTES"
)
with self.assertRaises(pytrap.TrapError):
a.createMessage(100000)
def runTest(self):
import pytrap
a = pytrap.UnirecTemplate("ipaddr SRC_IP,time TIME_FIRST,uint32 ABC,uint32 BCD,string TEXT,bytes STREAMBYTES")
data = bytearray(b'\x00\x00\x00\x00\x00\x00\x00\x00\x0A\x00\x00\x01\xff\xff\xff\xff\xc8\x76\xbe\xff\xe3\x2b\x6c\x57\x00\x00\x00\x01\x00\x00\x00\x02\x06\x00\x04\x00\x00\x00\x06\x00abcdef\xde\xad\xfe\xed')
a.setData(data)
tc = a.strRecord()
for i in range(100):
self.assertEqual(tc, a.strRecord())
def runTest(self):
import pytrap
a = pytrap.UnirecTemplate(
"ipaddr SRC_IP,time TIME_FIRST,uint32 ABC,uint32 BCD,string TEXT,bytes STREAMBYTES"
)
data = bytearray(
b'\x00\x00\x00\x00\x00\x00\x00\x00\x0A\x00\x00\x01\xff\xff\xff\xff\xc8\x76\xbe\xff\xe3\x2b\x6c\x57\x00\x00\x00\x01\x00\x00\x00\x02\x06\x00\x04\x00\x00\x00\x06\x00abcdef\xde\xad\xfe\xed'
)
a.setData(data)
self.assertEqual(len(a), 6, "Number of fields differs, 6 expected.")
self.assertEqual(
str(a),
"(ipaddr SRC_IP,time TIME_FIRST,uint32 ABC,uint32 BCD,bytes STREAMBYTES,string TEXT)"
)
d = a.getFieldsDict()
self.assertEqual(type(d), dict)
self.assertEqual(
d, {
'TIME_FIRST': 1,
'ABC': 2,
'BCD': 3,
'TEXT': 4,
'STREAMBYTES': 5,
'SRC_IP': 0
})
self.assertEqual(a.get(data, "SRC_IP"),
pytrap.UnirecIPAddr("10.0.0.1"))
self.assertEqual(a.get(data, "SRC_IP"), a.getByID(data, 0))
self.assertEqual(a.get(data, "SRC_IP"), a.SRC_IP)
self.assertEqual(a.get(data, "TIME_FIRST"), a.getByID(data, 1))
self.assertEqual(a.get(data, "TIME_FIRST"),
pytrap.UnirecTime(1466706915, 999))
self.assertEqual(a.get(data, "TIME_FIRST"), a.TIME_FIRST)
self.assertEqual(a.get(data, "ABC"), 16777216)
self.assertEqual(a.get(data, "ABC"), a.getByID(data, 2))
self.assertEqual(a.get(data, "ABC"), a.ABC)
self.assertEqual(a.get(data, "BCD"), 33554432)
self.assertEqual(a.get(data, "BCD"), a.getByID(data, 3))
self.assertEqual(a.get(data, "BCD"), a.BCD)
self.assertEqual(a.get(data, "TEXT"), "abcdef")
self.assertEqual(a.get(data, "TEXT"), a.getByID(data, 4))
self.assertEqual(a.get(data, "TEXT"), a.TEXT)
self.assertEqual(type(a.get(data, "STREAMBYTES")), bytearray)
self.assertEqual(a.get(data, "STREAMBYTES"),
bytearray(b'\xde\xad\xfe\xed'))
self.assertEqual(a.get(data, "STREAMBYTES"), a.getByID(data, 5))
self.assertEqual(a.get(data, "STREAMBYTES"), a.STREAMBYTES)
stream = a.get(data, "STREAMBYTES")
self.assertEqual(" ".join([hex(i) for i in stream]),
"0xde 0xad 0xfe 0xed")
def runTest(self):
import pytrap
a = pytrap.UnirecTemplate(
"ipaddr SRC_IP,time TIME_FIRST,uint32 ABC,uint32 BCD,string TEXT,bytes STREAMBYTES"
)
data = a.createMessage(100)
for i in range(100):
getdata = a.getData()
# size of created (allocated) message and retrieved message differs (due to non-filled variable fields)
self.assertEqual(data[:len(getdata)], getdata)
a.ABC = 666
self.assertEqual(a.ABC, 666)
a.SRC_IP = pytrap.UnirecIPAddr("147.32.1.1")
self.assertEqual(a.SRC_IP, pytrap.UnirecIPAddr("147.32.1.1"))
a.setByID(data, 0, pytrap.UnirecIPAddr("fd7c:e770:9b8a::465"))
self.assertEqual(a.SRC_IP, pytrap.UnirecIPAddr("fd7c:e770:9b8a::465"))
a.set(data, "SRC_IP", pytrap.UnirecIPAddr("10.0.0.1"))
self.assertEqual(a.SRC_IP, pytrap.UnirecIPAddr("10.0.0.1"))
a.TIME_FIRST = pytrap.UnirecTime(666, 0)
self.assertEqual(a.TIME_FIRST, pytrap.UnirecTime(666, 0))
a.setByID(data, 1, pytrap.UnirecTime(1234, 666))
self.assertEqual(a.TIME_FIRST, pytrap.UnirecTime(1234, 666))
a.set(data, "TIME_FIRST", pytrap.UnirecTime(1468962758, 166))
self.assertEqual(a.TIME_FIRST, pytrap.UnirecTime(1468962758, 166))
a.TEXT = "different text"
self.assertEqual(a.TEXT, "different text")
a.setByID(data, 4, "my long text")
self.assertEqual(a.TEXT, "my long text")
a.set(data, "TEXT", "long text")
self.assertEqual(a.TEXT, "long text")
a.STREAMBYTES = bytearray(b"he\x01\x01")
self.assertEqual(a.STREAMBYTES, bytearray(b"he\x01\x01"))
a.STREAMBYTES = bytes(b"\xca\xfe")
self.assertEqual(a.STREAMBYTES, bytes(b"\xca\xfe"))
a.setByID(data, 5, bytes(b"\xde\xad\xbe\xef"))
self.assertEqual(a.STREAMBYTES, bytes(b"\xde\xad\xbe\xef"))
data = a.createMessage(100)
a.setByID(data, 2, int(1234))
a.ABC = int(1)
self.assertEqual(a.ABC, int(1))
a.set(data, "ABC", int(666))
self.assertEqual(a.ABC, int(666))
a.ABC = int(222)
self.assertEqual(a.ABC, int(222))
# overflow
a.ABC = int(4294967296)
self.assertEqual(a.ABC, int(0))
def mainLoop():
global u
for i in range(9000000):
try:
a = ctx.recv(0)
except pytrap.FormatChanged as e:
fmt = ctx.getDataFmt(0)
u = pytrap.UnirecTemplate(fmt[1])
numfields = len(fmt[1].split(","))
a = e.data
del (e)
for i in range(numfields):
v = u.get(i, a)
def runTest(self):
import pytrap
a = pytrap.UnirecTemplate(
"ipaddr SRC_IP,time TIME_FIRST,uint32 ABC,uint32 BCD,string TEXT,bytes STREAMBYTES"
)
astr = str(a)
b = a.copy()
bstr = str(b)
self.assertEqual(astr, bstr)
self.assertEqual(
astr,
'(ipaddr SRC_IP,time TIME_FIRST,uint32 ABC,uint32 BCD,bytes STREAMBYTES,string TEXT)'
)
def mainLoop():
num = 0
numport = 0
while True:
try:
a = ctx.recv(0)
except pytrap.FormatChanged as e:
UR_Flow = pytrap.UnirecTemplate(ctx.getDataFmt(0)[1])
rec = URWrapper(UR_Flow)
a = e.data
del(e)
rec.setData(a)
if len(a) <= 1:
break
num = num + 1
print(("SSH src port count: {}".format(numport)))
print(("Total flows processed: {}".format(num)))
def _fetch_data(self, input_class):
"""
Fetches data from trap context and puts them to
queue as a IP/URL/DNS flow based on interface input (detector)
Arguments:
trap pytrap.trapCtx
interface int IP/URL/DNS
queue Queue
"""
# Set local variables for faster access
my_iface_num = input_class.iface_num
if not isinstance(input_class, IP_URL_Interface):
my_ur_input = input_class.ur_input
while not stop:
try:
data = trap.recv(my_iface_num)
except pytrap.FormatMismatch:
print(
"Error: output and input interfaces data format or data specifier mismatch"
)
break
except pytrap.FormatChanged as e:
fmttype, inputspec = trap.getDataFmt(my_iface_num)
my_ur_input = pytrap.UnirecTemplate(inputspec)
data = e.data
except pytrap.Terminated:
print("Terminated TRAP.")
break
except pytrap.TrapError:
break
if len(data) <= 1:
break
if isinstance(input_class, IP_URL_Interface):
# IP and URL events are sent in JSON (from aggregator)
rec = json.loads(data.decode())
else:
# DNS has Unirec format
# There has to be a copy, otherwise only reference is stored in the queue and rec is rewritten
rec = my_ur_input.copy()
rec.setData(data)
# No locking needed, the queue object does it internally
self.queue.put((my_iface_num, rec))
def mainLoop():
num = 0
numport = 0
while True:
try:
a = ctx.recv()
except pytrap.FormatChanged as e:
recTmpl = pytrap.UnirecTemplate(ctx.getDataFmt(0)[1])
a = e.data
del(e)
recTmpl.setData(a)
if recTmpl.SRC_PORT == 22 or recTmpl.DST_PORT == 22:
numport += 1
if len(a) <= 1:
break
num = num + 1
print("SSH src port count: {}".format(numport))
print("Total flows processed: {}".format(num))
def load_pytrap():
"""Init nemea libraries and set format of IP flows.
Returns
--------
rec : pytrap
Templete of IP flows.
trap : pytrap
Init pytrap NEMEA library, for capture IP flows from IFC interface setted in paramater -i.
"""
trap = pytrap.TrapCtx()
trap.init(sys.argv)
# Set the list of required fields in received messages.
# This list is an output of e.g. flow_meter - basic flow.
input_spec = "ipaddr DST_IP,ipaddr SRC_IP,uint64 BYTES,uint64 LINK_BIT_FIELD,time TIME_FIRST,time TIME_LAST,uint32 PACKETS,uint16 DST_PORT,uint16 SRC_PORT,uint8 DIR_BIT_FIELD,uint8 PROTOCOL,uint8 TCP_FLAGS"
trap.setRequiredFmt(0, pytrap.FMT_UNIREC, input_spec)
rec = pytrap.UnirecTemplate(input_spec)
return rec, trap
def runTest(self):
import pytrap
a = pytrap.UnirecTemplate(
"int64* ARRF1,int32* ARRF2,int16* ARRF3,int8* ARRF4,uint64* ARRF5,uint32* ARRF6,uint16* ARRF7,uint8* ARRF8"
)
a.createMessage(10000)
a.ARRF1 = [-1, -1]
a.ARRF2 = [-1, -1]
a.ARRF3 = [-1, -1]
a.ARRF4 = [-1, -1]
a.ARRF5 = [-1, -1]
a.ARRF6 = [-1, -1]
a.ARRF7 = [-1, -1]
a.ARRF8 = [-1, -1]
self.assertEqual(a.ARRF1, [-1, -1])
self.assertEqual(a.ARRF2, [-1, -1])
self.assertEqual(a.ARRF3, [-1, -1])
self.assertEqual(a.ARRF4, [-1, -1])
self.assertEqual(a.ARRF5, [18446744073709551615, 18446744073709551615])
self.assertEqual(a.ARRF6, [4294967295, 4294967295])
self.assertEqual(a.ARRF7, [65535, 65535])
self.assertEqual(a.ARRF8, [255, 255])
def fetch_data(trap, rec, interface, queue):
"""
Fetches data from trap context and puts them to
queue as a Flow or SMTP_Flow based on interface input
Arguments:
trap pytrap.trapCtx
interface int BASIC_IF/SMTP_IF
queue Queue
"""
while (g.is_running):
try:
data = trap.recv(interface)
except pytrap.FormatChanged as e:
fmttype, inputspec = trap.getDataFmt(interface)
rec = pytrap.UnirecTemplate(inputspec)
data = e.data
if len(data) <= 1:
break
rec.setData(data)
if interface is BASIC_IF:
if rec.DST_PORT in list(g.email_protocols.values()):
flow = Flow(rec)
queue.put(flow)
else:
try: #TODO Flow are discarded at the moment
flow = SMTP_Flow(rec)
if (flow == None):
pass
else:
queue.put(flow)
except FlowConversionException as e:
log.error("{0} Flow discarded.".format(e))
log.info("Receiver finished! Closing on interface {0}".format(interface))
return True
def __init__(self):
self.ur_input = pytrap.UnirecTemplate(self.template_in)
# coding: utf-8
import pytrap
a = pytrap.UnirecTemplate(
"ipaddr SRC_IP,time TIME_FIRST,uint32 ABC,uint32 BCD,string TEXT,bytes STREAMBYTES"
)
data = b'\x00\x00\x00\x00\x00\x00\x00\x00\x0A\x00\x00\x01\xff\xff\xff\xff\x01\x00\x00\x00\xe3\x2b\x6c\x57\x00\x00\x00\x01\x00\x00\x00\x02\x06\x00\x04\x00\x00\x00\x06\x00abcdef\xde\xad\xfe\xed'
a.setData(data)
print(len(a))
print(a)
print(a.getFieldsDict())
print("GET 0")
print(a.get(0, data))
print("GET 1")
print(a.get(1, data))
print("GET 2")
print(a.get(2, data))
print("GET 3")
print(a.get(3, data))
print("GET 4")
print(a.get(4, data))
print(a.TEXT)
print("GET 5")
# coding: utf-8
import sys
import pdb
import pytrap
ctx = pytrap.TrapCtx()
ctx.init(sys.argv)
ctx.setRequiredFmt(0)
print("\nReceiving one UniRec message")
try:
a = ctx.recv(0)
except pytrap.FormatChanged as e:
fmt = ctx.getDataFmt(0)
rec = pytrap.UnirecTemplate(fmt[1])
a = e.data
del(e)
print(rec)
rec.setData(a)
print("\nDirect access using index")
for i in range(len(rec)):
print((rec.get(i, a)))
print("\nAttribute access")
print((rec.SRC_IP))
for i in ["SRC_IP", "DST_IP", "SRC_PORT", "DST_PORT"]:
v = getattr(rec, i)
print(v)
print("\nIteration over all fields")
for i in rec:
#!/usr/bin/env python
import pytrap
import sys
trap = pytrap.TrapCtx()
trap.init(sys.argv, 1, 1)
trap.setRequiredFmt(0)
while True:
try:
data = trap.recv()
except pytrap.FormatChanged as e:
fmttype, fmtspec = trap.getDataFmt(0)
trap.setDataFmt(0, fmttype, fmtspec)
rec = pytrap.UnirecTemplate(fmtspec)
data = e.data
if len(data) <= 1:
break
rec.setData(data)
print((rec.strRecord()))
trap.send(data)
trap.finalize()
def amendflows(args, resolver):
"""Read flow data, add resolved fields and send amended flow data."""
trap = pytrap.TrapCtx()
trap.init(sys.argv, 1, 1)
urinput = pytrap.UnirecTemplate(args.urformat)
# this module accepts all Unirec fieds -> set required format:
trap.setRequiredFmt(0, pytrap.FMT_UNIREC, args.urformat)
trap.ifcctl(0, False, pytrap.CTL_BUFFERSWITCH, 0)
uroutformat = urspecaddresolvedfields(args.urformat, args.outfields)
uroutput = pytrap.UnirecTemplate(uroutformat)
trap.setDataFmt(0, pytrap.FMT_UNIREC, uroutformat)
if args.verbose:
print("Set output format to '{}'."
"\nWaiting for events.".format(uroutformat))
while True:
# Read data from input interface
try:
indata = trap.recv()
if args.verbose:
sys.stdout.write('.')
except pytrap.FormatMismatch:
print("Error: input and output interfaces data format"
" or data specifier mismatch")
break
except pytrap.FormatChanged as err:
# Get data format from negotiation, amend it, and set it
# for output iface
fmttype, fmtspec = trap.getDataFmt(0) # pylint:disable=unused-variable
# Update Unirec templates
urinput = pytrap.UnirecTemplate(fmtspec)
uroutformat = urspecaddresolvedfields(fmtspec, args.outfields)
uroutput = pytrap.UnirecTemplate(uroutformat)
trap.setDataFmt(0, pytrap.FMT_UNIREC, uroutformat)
if args.verbose:
print("Reset output format to '{}'.".format(uroutformat))
# Store data from the exception
indata = err.data # pylint:disable=no-member
except pytrap.Terminated:
print("Terminated trap.")
break
except pytrap.TrapError:
break
# Check for "end-of-stream" record
if len(indata) <= 1:
break
# Set indata for access using attributes
urinput.setData(indata)
inputvarlensize = urinput.recVarlenSize()
# Copy flow info from indata to outdata
resolvedvarlenmaxsize = sum(ftype[1] for fname, ftype in args.outfields
if len(ftype) > 1)
outdata = uroutput.createMessage(inputvarlensize +
resolvedvarlenmaxsize)
uroutput.setData(outdata)
for attr, value in urinput:
setattr(uroutput, attr, value)
for fieldstoresolve, resolution, outfield in args.validatedresolvspecs:
setattr(uroutput, outfield,
getattr(resolver, resolution)(urinput, *fieldstoresolve))
try:
trap.send(outdata)
except pytrap.Terminated:
print("Terminated trap.")
break
if args.verbose:
sys.stdout.write(',')
trap.sendFlush()
self._timer.cancel()
self.is_running = False
# Parse remaining command-line arguments
(options, args) = parser.parse_args()
# Initialize module
trap = pytrap.TrapCtx()
trap.init(sys.argv, 1, 1)
# Specifier of UniRec records will be received during libtrap negotiation
alertURFormat = "ipaddr SRC_IP,uint32 ADDR_CNT,time TIME_FIRST," + \
"time TIME_LAST,uint8 EVENT_TYPE,uint8 PROTOCOL"
UR_Input = pytrap.UnirecTemplate(alertURFormat)
# this module accepts all UniRec fieds -> set required format:
trap.setRequiredFmt(0, pytrap.FMT_UNIREC, alertURFormat)
trap.ifcctl(0, False, pytrap.CTL_BUFFERSWITCH, 0)
trap.setDataFmt(0, pytrap.FMT_UNIREC, alertURFormat)
UR_Output = pytrap.UnirecTemplate(alertURFormat)
URtmp = UR_Input.copy()
# Send aggregated alerts by RepeatedTimer
def sendEvents():
global eventList
if not eventList:
trap = pytrap.TrapCtx()
# Export
fmt = "uint64 ID,double TIME,double VALUE"
# Notification
# fmt = "double TIME,uint64 ID,double homeID,double nodeID,double GENRE,double CMDCLASS,double INSTANCE,double INDEX,double TYPE,double BYTE"
# NodeStats
# fmt = "uint64 ID,double TIME,double averageRequestRTT,double averageResponseRTT,double lastRequestRTT,double lastResponseRTT,double quality,double receiveDuplications,double receiveUnsolicited,double receivedCount,double sentCount,double sentFailed"
# HciStats
# fmt = "uint64 ID,double TIME,double aclPackets,double address,double rxAcls,double rxBytes,double rxErrors,double rxEvents,double rxScos,double scoMtu,double scoPackets,double txAcls,double txBytes,double txCmds,double txErrors,double txScos"
# Dispatch
# fmt = "uint64 ID,double TIME,string cmd"
# DriverStats
# fmt = "double ACKCount,double ACKWaiting,double CANCount,uint64 ID,double NAKCount,double OOFCount,double SOFCount,double TIME,double badChecksum,double badroutes,double broadcastReadCount,double broadcastWriteCount,double callbacks,double dropped,double netBusy,double noACK,double nonDelivery,double notIdle,double readAborts,double readCount,double retries,double routedBusy,double writeCount"
tmplt = pytrap.UnirecTemplate(fmt)
tmplt.createMessage()
trap.init(["-i", "u:coliot-socket"], 0, 1)
trap.setDataFmt(0, pytrap.FMT_UNIREC, fmt)
signal.signal(signal.SIGINT, signal_h)
i = 0
while True:
i += 1
# Export
tmplt.ID = i
tmplt.TIME = time.time()
if isinstance(o, pytrap.UnirecIPAddr):
return str(o)
elif isinstance(o, pytrap.UnirecTime):
return float(o)
else:
return repr(o)
# Main loop
stop = False
while not stop:
try:
data = trap.recv()
except pytrap.FormatChanged as e:
fmttype, inputspec = trap.getDataFmt(0)
rec = pytrap.UnirecTemplate(inputspec)
data = e.data
if len(data) <= 1:
# Send end-of-stream message and exit
if not options.no_eos:
trap.send(bytearray(b"0"))
trap.sendFlush(0)
break
rec.setData(data)
j = json.dumps({k: v for k, v in rec}, default=default)
if options.verbose:
print(j)
trap.send(bytearray(j, "utf-8"))
# Free allocated TRAP IFCs
trap.finalize()
def Run(module_name, module_desc, req_type, req_format, conv_func, arg_parser = None):
"""Run the main loop of the reporter module called `module_name` with `module_desc` (used in help).
The module requires data format of `req_type` type and `req_format` specifier - these must be given by author of the module.
`conv_func(rec, args)` is a callback function that must translate given incoming alert `rec` (typically in UniRec according to `req_type`) into IDEA message. `args` contains CLI arguments parsed by ArgumentParser. `conv_func` must return dict().
"""
global trap
# *** Parse command-line arguments ***
if arg_parser is None:
arg_parser = argparse.ArgumentParser()
arg_parser.formatter_class = argparse.RawDescriptionHelpFormatter
# Set description
arg_parser.description = str.format(desc_template,
name=module_name,
type={pytrap.FMT_RAW:'raw', pytrap.FMT_UNIREC:'UniRec', pytrap.FMT_JSON:'JSON'}.get(req_type,'???'),
fmt=req_format,
original_desc = module_desc+"\n\n " if module_desc else "",
)
# Add arguments defining outputs
# TRAP output
arg_parser.add_argument('-T', '--trap', action='store_true',
help='Enable output via TRAP interface (JSON type with format id "IDEA"). Parameters are set using "-i" option as usual.')
# Config file
arg_parser.add_argument('-c', '--config', metavar="FILE", default="./config.yaml", type=str,
help='Specify YAML config file path which to load.')
arg_parser.add_argument('-d', '--dry', action='store_true',
help="""Do not run, just print loaded config.""")
# Warden3 output
arg_parser.add_argument('-W', '--warden', metavar="CONFIG_FILE",
help='Send IDEA messages to Warden server. Load configuration of Warden client from CONFIG_FILE.')
# Other options
arg_parser.add_argument('-n', '--name', metavar='NODE_NAME',
help='Name of the node, filled into "Node.Name" element of the IDEA message. Required argument.')
arg_parser.add_argument('-v', '--verbose', metavar='VERBOSE_LEVEL', default=3, type=int,
help="""Enable verbose mode (may be used by some modules, common part doesn't print anything).\nLevel 1 logs everything, level 5 only critical errors. Level 0 doesn't log.""")
# TRAP parameters
trap_args = arg_parser.add_argument_group('Common TRAP parameters')
trap_args.add_argument('-i', metavar="IFC_SPEC", required=True,
help='See http://nemea.liberouter.org/trap-ifcspec/ for more information.')
# Parse arguments
args = arg_parser.parse_args()
# Set log level
logging.basicConfig(level=(args.verbose*10), format=FORMAT)
# Check if node name is set if Warden output is enabled
if args.name is None:
#if args.warden:
# sys.stderr.write(module_name+": Error: Node name must be specified if Warden output is used (set param --name).\n")
# exit(1)
logger.warning("Node name is not specified.")
# *** Initialize TRAP ***
logger.info("Trap arguments: %s", args.i)
trap.init(["-i", args.i], 1, 1 if args.trap else 0)
#trap.setVerboseLevel(3)
signal.signal(signal.SIGINT, signal_h)
# Set required input format
trap.setRequiredFmt(0, req_type, req_format)
if args.trap:
trap.setDataFmt(0, pytrap.FMT_JSON, "IDEA")
# *** Create output handles/clients/etc ***
wardenclient = None
if args.warden:
try:
import warden_client
except:
logger.error("There is no available warden_client python module. Install it or remove '--warden' from the module's arguments.")
sys.exit(1)
config = warden_client.read_cfg(args.warden)
config['name'] = args.name
wardenclient = warden_client.Client(**config)
# Initialize configuration
config = Config.Config(args.config, trap = trap, warden = wardenclient)
if not args.dry:
# *** Main loop ***
URInputTmplt = None
if req_type == pytrap.FMT_UNIREC and req_format != "":
URInputTmplt = pytrap.UnirecTemplate(req_format) # TRAP expects us to have predefined template for required set of fields
rec = URInputTmplt
stop = False
while not stop:
logger.info("Starting receiving")
# *** Read data from input interface ***
try:
data = trap.recv()
except pytrap.FormatMismatch:
logger.error("Input data format mismatch in receiving from TRAP interface")
break
except pytrap.FormatChanged as e:
# Get negotiated input data format
(fmttype, fmtspec) = trap.getDataFmt(0)
# If data type is UniRec, create UniRec template
if fmttype == pytrap.FMT_UNIREC:
URInputTmplt = pytrap.UnirecTemplate(fmtspec)
else:
URInputTmplt = None
rec = URInputTmplt
data = e.data
except pytrap.Terminated:
break
# Check for "end-of-stream" record
if len(data) <= 1:
if args.trap:
# If we have output, send "end-of-stream" record and exit
trap.send(0, b"0")
break
# Assert that if UniRec input is required, input template is set
assert(req_type != pytrap.FMT_UNIREC or URInputTmplt is not None)
# Convert raw input data to UniRec object (if UniRec input is expected)
if req_type == pytrap.FMT_UNIREC:
rec.setData(data)
elif req_type == pytrap.FMT_JSON:
rec = json.loads(str(data))
else: # TRAP_FMT_RAW
rec = data
# *** Convert input record to IDEA ***
# Pass the input record to conversion function to create IDEA message
idea = conv_func(rec, args)
if idea is None:
# Record can't be converted - skip it
continue
if args.name is not None:
idea['Node'][0]['Name'] = args.name
# *** Send IDEA to outputs ***
# Perform rule matching and action running on the idea message
try:
config.match(idea)
except pytrap.Terminated:
logger.error("PyTrap was terminated")
break
except DropMsg:
logger.info("Message was dropped by Drop action.")
continue
except Exception as e:
logger.error(str(e))
break
else:
# DRY argument given, just print config and exit
print(config)
if wardenclient:
wardenclient.close()
trap.finalize()
def runTest(self):
import pytrap
a = pytrap.UnirecTemplate("ipaddr IP,time TIME,uint64 U64,uint32 U32,uint16 U16,uint8 U8,int64 I64,int32 I32,int16 I16,int8 I8,float FL,double DB,char CHR,string TEXT,bytes STREAMBYTES")
a.createMessage(100)
a.IP = pytrap.UnirecIPAddr("1.2.3.4")
a.TIME = pytrap.UnirecTime(123456)
a.U64 = 0x100000000
a.U32 = 0x10000
a.U16 = 0x100
a.U8 = 0x1
a.I64 = -1
a.I32 = -1
a.I16 = -1
a.I8 = -1
a.FL = 1.234
a.DB = 1.234
#a.CHR = "a"
a.TEXT = "text"
a.STREAMBYTES = b"streambytes"
self.assertTrue(a.IP == pytrap.UnirecIPAddr("1.2.3.4"))
self.assertTrue(a.TIME == pytrap.UnirecTime(123456))
self.assertTrue(a.U64 == 0x100000000)
self.assertTrue(a.U32 == 0x10000)
self.assertTrue(a.U16 == 0x100)
self.assertTrue(a.U8 == 0x1)
self.assertTrue(a.I64 == -1)
self.assertTrue(a.I32 == -1)
self.assertTrue(a.I16 == -1)
self.assertTrue(a.I8 == b'\xff')
self.assertTrue(1.234 - a.FL < 1e-7)
self.assertTrue(a.DB == 1.234)
#self.assertTrue(a.CHR == "a")
self.assertTrue(a.TEXT == "text")
self.assertTrue(a.STREAMBYTES == b"streambytes")
# Check types
self.assertEqual(type(a.IP), pytrap.UnirecIPAddr)
self.assertEqual(type(a.TIME), pytrap.UnirecTime)
self.assertEqual(type(a.U64), long)
self.assertEqual(type(a.U32), int)
self.assertEqual(type(a.U16), int)
self.assertEqual(type(a.U8), int)
self.assertEqual(type(a.I64), long)
self.assertEqual(type(a.I32), int)
self.assertEqual(type(a.I16), int)
self.assertEqual(type(a.I8), bytes)
self.assertEqual(type(a.CHR), int)
self.assertEqual(type(a.FL), float)
self.assertEqual(type(a.DB), float)
self.assertEqual(type(a.TEXT), str)
self.assertEqual(type(a.STREAMBYTES), bytearray)
self.assertEqual(a.getFieldType("IP"), pytrap.UnirecIPAddr)
self.assertEqual(a.getFieldType("TIME"), pytrap.UnirecTime)
self.assertEqual(a.getFieldType("U64"), long)
self.assertEqual(a.getFieldType("U32"), long)
self.assertEqual(a.getFieldType("U16"), long)
self.assertEqual(a.getFieldType("U8"), long)
self.assertEqual(a.getFieldType("I64"), long)
self.assertEqual(a.getFieldType("I32"), long)
self.assertEqual(a.getFieldType("I16"), long)
self.assertEqual(a.getFieldType("I8"), long)
self.assertEqual(a.getFieldType("CHR"), long)
self.assertEqual(a.getFieldType("FL"), float)
self.assertEqual(a.getFieldType("DB"), float)
self.assertEqual(a.getFieldType("TEXT"), str)
self.assertEqual(a.getFieldType("STREAMBYTES"), bytearray)
help="target output directory.")
parser.add_argument(
"--tmp-dir",
default='./',
help=
"directory for storing temporary file before moving them to output destination."
)
args = parser.parse_args()
context = pytrap.TrapCtx()
context.init(['-i', args.ifcspec])
context.setRequiredFmt(0, pytrap.FMT_UNIREC, "")
unirec = pytrap.UnirecTemplate("ipaddr DST_IP,ipaddr SRC_IP,uint64 BYTES,uint64 LINK_BIT_FIELD,time TIME_FIRST, "\
"time TIME_LAST,uint32 DNS_RR_TTL,uint32 PACKETS,uint16 DNS_ANSWERS,uint16 DNS_CLASS,"\
"uint16 DNS_ID,uint16 DNS_PSIZE,uint16 DNS_QTYPE,uint16 DNS_RLENGTH,uint16 DST_PORT,"\
"uint16 SRC_PORT,uint8 DIR_BIT_FIELD,uint8 DNS_DO,uint8 DNS_RCODE,"\
"uint8 PROTOCOL,uint8 TCP_FLAGS,uint8 TOS,uint8 TTL,string DNS_NAME,bytes DNS_RDATA")
tmp_dir = args.tmp_dir
if not os.path.exists(tmp_dir):
sys.stderr.write("Path given by --tmp-dir not found.\n")
exit(1)
output_path = args.destination
if not os.path.exists(output_path):
sys.stderr.write("Path given by --destination not found.\n")
exit(1)
records = list()
try: