def obsloss_report_stream_biflow(ipfix_stream, timeslice=0):
df = qof.dataframe_from_ipfix_stream(
ipfix_stream,
("flowStartMilliseconds", "flowEndMilliseconds", "packetDeltaCount",
"reversePacketDeltaCount", "transportPacketDeltaCount",
"reverseTransportPacketDeltaCount", "octetDeltaCount",
"reverseOctetDeltaCount", "transportOctetDeltaCount",
"reverseTransportOctetDeltaCount", "tcpSequenceCount",
"reverseTcpSequenceCount", "tcpSequenceLossCount",
"reverseTcpSequenceLossCount"))
allcount = len(df)
df["lossy"] = (df["tcpSequenceLossCount"] >
0) | (df["reverseTcpSequenceLossCount"] > 0)
lossycount = len(df[df["lossy"]])
print("Total flows: %u " % (allcount))
print(" of which lossy: %u (%.2f%%)" %
(lossycount, lossycount * 100 / allcount))
if timeslice:
lossy_flows = np.where(df["lossy"], 1, 0)
lossy_flows.index = df["flowEndMilliseconds"]
total_flows = pd.Series(1, index=lossy_flows.index)
lossy_flows = lossy_flows.resample(str(timeslice) + "S", how='sum')
total_flows = total_flows.resample(str(timeslice) + "S", how='sum')
lossy_flow_rate = lossy_flows / total_flows
dfout = pd.DataFrame({
'total': total_flows,
'lossy': lossy_flows,
'rate': lossy_flow_rate
})
dfout.to_csv(stdout)
def obsloss_report_stream_biflow(ipfix_stream, timeslice = 0):
df = qof.dataframe_from_ipfix_stream(ipfix_stream, (
"flowStartMilliseconds", "flowEndMilliseconds",
"packetDeltaCount", "reversePacketDeltaCount",
"transportPacketDeltaCount", "reverseTransportPacketDeltaCount",
"octetDeltaCount", "reverseOctetDeltaCount",
"transportOctetDeltaCount", "reverseTransportOctetDeltaCount",
"tcpSequenceCount", "reverseTcpSequenceCount",
"tcpSequenceLossCount", "reverseTcpSequenceLossCount"))
allcount = len(df)
df["lossy"] = (df["tcpSequenceLossCount"] > 0) | (df["reverseTcpSequenceLossCount"] > 0)
lossycount = len(df[df["lossy"]])
print ("Total flows: %u " % (allcount))
print (" of which lossy: %u (%.2f%%)" % (lossycount, lossycount * 100 / allcount))
if timeslice:
lossy_flows = np.where(df["lossy"], 1, 0)
lossy_flows.index = df["flowEndMilliseconds"]
total_flows = pd.Series(1, index=lossy_flows.index)
lossy_flows = lossy_flows.resample(str(timeslice)+"S", how='sum')
total_flows = total_flows.resample(str(timeslice)+"S", how='sum')
lossy_flow_rate = lossy_flows / total_flows;
dfout = pd.DataFrame({'total': total_flows,
'lossy': lossy_flows,
'rate': lossy_flow_rate})
dfout.to_csv(stdout)
def opts_report_stream(ipfix_stream):
df = qof.dataframe_from_ipfix_stream(ipfix_stream,
("initialTCPFlags","reverseInitialTCPFlags",
"unionTCPFlags","reverseUnionTCPFlags",
"qofTcpCharacteristics", "reverseQofTcpCharacteristics",
"tcpSequenceLossCount", "reverseTcpSequenceLossCount",
"packetDeltaCount", "reversePacketDeltaCount",
"sourceIPv4Address", "destinationIPv4Address"))
print ("Total flows: "+str(len(df)))
df = qof.drop_lossy(df)
print (" of which lossless: "+str(len(df)))
df = qof.drop_incomplete(df)
print (" of which complete: "+str(len(df)))
ecn_nego_s = ecn_negotiated(df)
ecn_ect0_s = characteristic_present(df, qof.QOF_ECT0)
ecn_ect1_s = characteristic_present(df, qof.QOF_ECT1)
ecn_ce_s = characteristic_present(df, qof.QOF_CE)
print_proportion("ECN nego", ecn_nego_s)
print_proportion("ECT0", ecn_ect0_s)
print_proportion("ECT1", ecn_ect1_s)
print_proportion("nego->ECT0", ecn_nego_s & ecn_ect0_s)
print_proportion("nego->ECT1", ecn_nego_s & ecn_ect1_s)
print_proportion("CE", ecn_ce_s)
print()
print_proportion("ECT0+ECT1", ecn_ect0_s & ecn_ect1_s)
print_proportion("ECT0+CE", ecn_ce_s & ecn_ect0_s)
print_proportion("ECT1+CE", ecn_ce_s & ecn_ect1_s)
print_proportion("any ECx", ecn_ce_s | ecn_ect0_s | ecn_ect1_s)
print_proportion("all ECx", ecn_ce_s & ecn_ect0_s & ecn_ect1_s)
print()
tcp_ws_s = characteristic_present(df, qof.QOF_WS)
tcp_ts_s = characteristic_present(df, qof.QOF_TS)
tcp_sack_s = characteristic_present(df, qof.QOF_SACK)
print_proportion("WS", tcp_ws_s)
print_proportion("TS", tcp_ts_s)
print_proportion("SACK", tcp_sack_s)
print()
all_sources = ip4_sources(df)
ws_sources = ip4_sources_characteristic(df, qof.QOF_WS)
print("WS observed from %8u sources (%8.5f%%)" %
(len(ws_sources), len(ws_sources) * 100 / len(all_sources)))
ts_sources = ip4_sources_characteristic(df, qof.QOF_TS)
print("TS observed from %8u sources (%8.5f%%)" %
(len(ts_sources), len(ts_sources) * 100 / len(all_sources)))
sack_sources = ip4_sources_characteristic(df, qof.QOF_SACK)
print("SACK observed from %8u sources (%8.5f%%)" %
(len(sack_sources), len(sack_sources) * 100 / len(all_sources)))
nego_sources = ip4_sources_given(df, ecn_nego_s, ecn_nego_s)
print("ECN nego involved %8u sources (%8.5f%%)" %
(len(nego_sources), len(nego_sources) * 100 / len(all_sources)))
def opts_report_stream(ipfix_stream):
df = qof.dataframe_from_ipfix_stream(
ipfix_stream, ("initialTCPFlags", "reverseInitialTCPFlags",
"unionTCPFlags", "reverseUnionTCPFlags",
"qofTcpCharacteristics", "reverseQofTcpCharacteristics",
"tcpSequenceLossCount", "reverseTcpSequenceLossCount",
"packetDeltaCount", "reversePacketDeltaCount",
"sourceIPv4Address", "destinationIPv4Address"))
print("Total flows: " + str(len(df)))
df = qof.drop_lossy(df)
print(" of which lossless: " + str(len(df)))
df = qof.drop_incomplete(df)
print(" of which complete: " + str(len(df)))
ecn_nego_s = ecn_negotiated(df)
ecn_ect0_s = characteristic_present(df, qof.QOF_ECT0)
ecn_ect1_s = characteristic_present(df, qof.QOF_ECT1)
ecn_ce_s = characteristic_present(df, qof.QOF_CE)
print_proportion("ECN nego", ecn_nego_s)
print_proportion("ECT0", ecn_ect0_s)
print_proportion("ECT1", ecn_ect1_s)
print_proportion("nego->ECT0", ecn_nego_s & ecn_ect0_s)
print_proportion("nego->ECT1", ecn_nego_s & ecn_ect1_s)
print_proportion("CE", ecn_ce_s)
print()
print_proportion("ECT0+ECT1", ecn_ect0_s & ecn_ect1_s)
print_proportion("ECT0+CE", ecn_ce_s & ecn_ect0_s)
print_proportion("ECT1+CE", ecn_ce_s & ecn_ect1_s)
print_proportion("any ECx", ecn_ce_s | ecn_ect0_s | ecn_ect1_s)
print_proportion("all ECx", ecn_ce_s & ecn_ect0_s & ecn_ect1_s)
print()
tcp_ws_s = characteristic_present(df, qof.QOF_WS)
tcp_ts_s = characteristic_present(df, qof.QOF_TS)
tcp_sack_s = characteristic_present(df, qof.QOF_SACK)
print_proportion("WS", tcp_ws_s)
print_proportion("TS", tcp_ts_s)
print_proportion("SACK", tcp_sack_s)
print()
all_sources = ip4_sources(df)
ws_sources = ip4_sources_characteristic(df, qof.QOF_WS)
print("WS observed from %8u sources (%8.5f%%)" %
(len(ws_sources), len(ws_sources) * 100 / len(all_sources)))
ts_sources = ip4_sources_characteristic(df, qof.QOF_TS)
print("TS observed from %8u sources (%8.5f%%)" %
(len(ts_sources), len(ts_sources) * 100 / len(all_sources)))
sack_sources = ip4_sources_characteristic(df, qof.QOF_SACK)
print("SACK observed from %8u sources (%8.5f%%)" %
(len(sack_sources), len(sack_sources) * 100 / len(all_sources)))
nego_sources = ip4_sources_given(df, ecn_nego_s, ecn_nego_s)
print("ECN nego involved %8u sources (%8.5f%%)" %
(len(nego_sources), len(nego_sources) * 100 / len(all_sources)))
def qofalize(ipfix_stream, ienames, binsize=300):
df = qof.dataframe_from_ipfix_stream(ipfix_stream, ienames)
# basic data rate
bps = df["octetDeltaCount"]
if "reverseOctetDeltaCount" in ienames:
bps += df["reverseOctetDeltaCount"]
bps *= 8 / binsize
bps.index = df["flowEndMilliseconds"]
dfo = pd.DataFrame({'bps': bps})
# packet rate
pps = df["packetDeltaCount"]
if "reversePacketDeltaCount" in ienames:
pps += df["reversePacketDeltaCount"]
pps /= binsize
pps.index = df["flowEndMilliseconds"]
dfo['pps'] = pps
# resample
dfo = dfo.resample(str(binsize)+"S", how=sum)
# and print
dfo.to_csv(stdout)
def stream_to_hdf5(ipfix_stream, ienames, hdf5_store, hdf5_table):
store = pd.HDFStore(hdf5_store)
df = qof.dataframe_from_ipfix_stream(ipfix_stream, ienames)
qof.coerce_timestamps(df)
store[hdf5_table] = df
store.close()
def stream_to_hdf5(ipfix_stream, ienames, hdf5_store, hdf5_table):
store = pd.HDFStore(hdf5_store)
df = qof.dataframe_from_ipfix_stream(ipfix_stream, ienames)
qof.coerce_timestamps(df)
store[hdf5_table] = df
store.close()
