def run(self):
'''
Runs the collector
'''
# Templates we build out from the NetFlow v9 or IPFIX source
templates = {"netflow": {} , "ipfix": {}}
to_retry = []
# Process packets forever
while True:
# Read from the input queue
try:
payload = self.input.get(block=True, timeout=0.5)
except queue.Empty:
continue
try:
export = netflow.parse_packet(payload.data, templates)
except netflow.utils.UnknownExportVersion as e:
logger.error(f"Unknown version: {e}")
continue
except (netflow.v9.V9TemplateNotRecognized, netflow.ipfix.IPFIXTemplateNotRecognized) as e:
if (time.time() - payload.ts) > self.TIMEOUT:
logger.error("Dropping timed out packet.")
else:
to_retry.append(payload)
logger.warning("Adding to queue to retry later (as templates update).")
continue
# Process the flows
self._process_export(export)
# Look for templates to re-process flows with
if export.header.version in [9, 10] and export.contains_new_templates and to_retry:
logger.info(f"New templates recieved! Reprocessing {len(to_retry)} frames.")
for retry_payload in to_retry:
retry_export = netflow.parse_packet(retry_payload.data, templates)
self._process_export(retry_export)
to_retry.clear()
def run(self):
# Process packets from the queue
try:
templates = {}
to_retry = []
while not self._shutdown.is_set():
try:
# 0.5s delay to limit CPU usage while waiting for new packets
pkt = self.input.get(block=True, timeout=0.5)
except queue.Empty:
continue
try:
export = parse_packet(pkt.data, templates)
except UnknownNetFlowVersion as e:
__log__.error("%s, ignoring the packet", e)
continue
except TemplateNotRecognized:
if time.time() - pkt.ts > PACKET_TIMEOUT:
__log__.warning(
"Dropping an old and undecodable v9 ExportPacket")
else:
to_retry.append(pkt)
__log__.debug(
"Failed to decode a v9 ExportPacket - will "
"re-attempt when a new template is discovered")
continue
__log__.debug("Processed a v%d ExportPacket with %d flows.",
export.header.version, export.header.count)
# If any new templates were discovered, dump the unprocessable
# data back into the queue and try to decode them again
if (export.header.version == 9
and export.contains_new_templates and to_retry):
__log__.debug("Received new template(s)")
__log__.debug(
"Will re-attempt to decode %d old v9 ExportPackets",
len(to_retry))
for p in to_retry:
self.input.put(p)
to_retry.clear()
self.output.put((pkt.ts, export))
finally:
self.server.shutdown()
self.server.server_close()
return H
def return_IP(raw):
return socket.inet_ntoa(struct.pack('!L', raw))
start = time.time()
stop = time.time() + 1200
flag = []
while True: # (time.time() < stop) :
payload, client = sock.recvfrom(
1464) # 4096 is experimental, tested with 1464
p = netflow.parse_packet(
payload) # Test result: <ExportPacket v5 with 30 records>
flag.append(p.header.timestamp)
if flag[-1] - flag[0] < interval:
for flow in p.flows:
### Possible fields
#flow.SRC_AS
#flow.DST_AS
#flow.SRC_PORT
#flow.DST_PORT
#flow.IN_OCTETS
#flow.IN_PACKETS
#flow.IPV4_SRC_ADDR
#flow.IPV4_DST_ADDR
#definisco una variabile che mi contenga il sysUpTime del mio router
sysUpTime = 0
#definizione del socket in attesa su porta 2055
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
sock.bind(("0.0.0.0", 2055))
try:
#inizializzo la REST API
api.run_rest_api()
#ciclo infinito, raccolgo i dati provenienti dal router
while True:
payload, client = sock.recvfrom(4096)
try:
# il template viene passato per riferimento al parser,
# viene aggiornato da quest'ultimo quando il pacchetto lo definisce
p = netflow.parse_packet(payload, templates)
if (p.header.version != 9):
print("Attivare la versione 9 di traffic flow su router")
sys.exit()
if (flow_sequence_exp == 0):
flow_sequence_exp = p.header.sequence
if (sysUpTime == 0):
sysUpTime = p.header.uptime
#Se il numero di sequenza del pacchetto è minore di quello atteso
# è possibile sia un resend del pacchetto e lo scarto.
#Tuttavia si deve controllare il sysUpTime perchè nel caso il mio
# router si sia riavviato io non devo scartare alcun pacchetto.
#Nel caso invece che il sysUpTime non sia minore dell'ultima
