def process_single_day(day):
SES_CLIENT = 0
SES_AP = 1
SES_START = 2
SES_END = 3
FLOWS_COUNT = 0
seslen_qry = "select client,ap,start,end from seslen where day = " + str(
day)
rses = RunSQL("syslog_final.db")
rflows = RunSQL("unc.db")
sesres = rses.sqlq(seslen_qry)
fcount = list()
for single_ses in sesres:
flows_qry = "select count(*) from flows where client = " + str(
single_ses[SES_CLIENT]) + " and ap = " + str(
single_ses[SES_AP]) + " and ts >= " + str(
single_ses[SES_START]) + " and ts <= " + str(
single_ses[SES_END]) + " and day = " + str(
day) + " group by client, ap"
flowres = rflows.sqlq(flows_qry)
if (flowres != None and len(flowres) > 0):
fcount.append(flowres[0])
return fcount
def process_single_day(day):
SES_CLIENT = 0
SES_AP = 1
SES_START = 2
SES_END = 3
SNMP_INB = 0
SNMP_OUTB = 1
seslen_qry = "select client,ap,start,end from seslen where day = " + str(
day)
rses = RunSQL("syslog_final.db")
rsnmp = RunSQL("unc.db")
sesres = rses.sqlq(seslen_qry)
inb = list()
outb = list()
for single_ses in sesres:
snmp_qry = "select sum(bytr), sum(byts) from snmp where client = " + str(
single_ses[SES_CLIENT]) + " and ap = " + str(
single_ses[SES_AP]) + " and ts >= " + str(
single_ses[SES_START]) + " and ts <= " + str(
single_ses[SES_END]) + " and day = " + str(
day) + " group by client, ap"
snmpres = rsnmp.sqlq(snmp_qry)
if (snmpres != None and len(snmpres) > 0 and snmpres[0][SNMP_INB] > 0
and snmpres[0][SNMP_OUTB] > 0):
inb.append(snmpres[0][SNMP_INB])
outb.append(snmpres[0][SNMP_OUTB])
return (inb, outb)
def create_seslen():
dl = get_day_list()
r = RunSQL("unc-proc.db")
slen = list()
inb = list()
outb = list()
for d in dl:
q = "select seslen, bin, bout from sessions where day = " + str(
d) + " and seslen > 0 and bin > 0 and bout > 0"
print "Getting data for day - ", d
slist = r.sqlq(q)
n = len(slist)
for i in xrange(n):
slen.append(slist[i][0])
inb.append(slist[i][1])
outb.append(slist[i][2])
fname = "day_" + str(d)
slenf = get_output_dir("seslen") + "/" + fname
inbf = get_output_dir("inb") + "/" + fname
outbf = get_output_dir("outb") + "/" + fname
print "Writing slen - " + slenf
write_data(slenf, slen)
print "Writing inb - " + inbf
write_data(inbf, inb)
print "Writing outb - " + outbf
write_data(outbf, outb)
def create_seslen():
dl = get_day_list()
r = RunSQL("unc-proc.db")
slen = list()
inb = list()
outb = list()
for d in dl:
q = "select seslen, bin, bout from sessions where day = " + str(d) + " and seslen > 0 and bin > 0 and bout > 0"
print "Getting data for day - ", d
slist = r.sqlq(q)
n = len(slist)
for i in xrange(n):
slen.append(slist[i][0])
inb.append(slist[i][1])
outb.append(slist[i][2])
fname = "day_" + str(d)
slenf = get_output_dir("seslen") + "/" + fname
inbf = get_output_dir("inb") + "/" + fname
outbf = get_output_dir("outb") + "/" + fname
print "Writing slen - " + slenf
write_data(slenf, slen)
print "Writing inb - " + inbf
write_data(inbf, inb)
print "Writing outb - " + outbf
write_data(outbf, outb)
def get_dataset_info(tag):
q = "select filename from datasets where unique_id = '" + tag + "'"
r = RunSQL("files_and_analysis.db")
rs = r.sqlq(q)
filename = rs[0].replace("/home/gautam", os.getenv("HOME"))
return {
"dir": os.path.dirname(filename),
"file": os.path.basename(filename)
}
def create_ses_count_inter():
dl = get_day_list()
r = RunSQL("unc.db")
r1 = RunSQL("syslog_final.db")
for d in dl:
print "Processing day - ", d
q1 = "select client, ap, start, end from seslen where day = " + str(d)
l1 = r1.sqlq(q1)
fcount = list()
sesinter = list()
for ses in l1:
start_ts = ses[2]
end_ts = ses[3]
client = ses[0]
ap = ses[1]
q = "select ts from flows" + \
" where day = " + str(d) + \
" and ts >= " + str(start_ts) + \
" and ts <= " + str(end_ts) + \
" and client = " + str(client) + \
" and ap = " + str(ap) + \
" order by ts"
l = r.sqlq(q)
if ( l != None and len(l) > 0 ):
fcount.append(len(l))
n = len(l)
npl = np.array(l)
inpl = (npl[1:n] - npl[0:n-1])
inpl1 = inpl[np.where(inpl>0)]
if ( len(inpl1) > 0 ):
sesinter.extend(inpl1.tolist())
ifname = get_output_dir("intrases") + "/day_" + str(d)
cfname = get_output_dir("fcount") + "/day_" + str(d)
print "Writing file - ", ifname
write_data(ifname, sesinter)
print "Writing file - ", cfname
write_data(cfname, fcount)
def process_single_day(day):
SES_CLIENT=0
SES_AP=1
SES_START=2
SES_END=3
FLOWS_COUNT=0
seslen_qry = "select client,ap,start,end from seslen where day = " + str(day)
rses = RunSQL("syslog_final.db")
rflows = RunSQL("unc.db")
sesres = rses.sqlq(seslen_qry)
fcount = list()
for single_ses in sesres:
flows_qry = "select count(*) from flows where client = " + str(single_ses[SES_CLIENT]) + " and ap = " + str(single_ses[SES_AP]) + " and ts >= " + str(single_ses[SES_START]) + " and ts <= " + str(single_ses[SES_END]) + " and day = " + str(day) + " group by client, ap"
flowres = rflows.sqlq(flows_qry)
if ( flowres != None and len(flowres) > 0 ):
fcount.append(flowres[0])
return fcount
def main(args):
r1 = RunSQL("files_and_analysis.db")
r2 = RunSQL("logn_fits.db")
unique_id = args[0].upper()
distribution = args[1].upper()
r1q = "select type, ks_fit, q_fit from fits where unique_id='" + unique_id + "' and distribution='" + distribution + "'"
r2q = "select type, ks_fit, q_fit from logn_fits where unique_id='" + unique_id + "'"
if "LOGN" == distribution:
# Do something
rs = r2.sqlq(r2q)
for rss in rs:
print distribution + "-" + rss[0] + "-" + "KS: " + kstex(rss[1])
print distribution + "-" + rss[0] + "-" + "FIT:" + qprint(rss[2]) + " " + qtex(rss[2])
else:
# Do something
rs = r1.sqlq(r1q)
rss = rs[0]
print distribution + "-" + rss[0] + "-" + "KS: " + kstex(rss[1])
print distribution + "-" + rss[0] + "-" + "FIT:" + qprint(rss[2]) + " " + qtex(rss[2])
def create_ses_inter():
dl = get_day_list()
r = RunSQL("unc.db")
for d in dl:
q = "select ts from flows where day = " + str(d) + " and term > 1 order by ts"
print "Running query for day - ", str(d)
ts = np.array(r.sqlq(q))
n = len(ts)
its = ts[1:n] - ts[0:n-1]
nits = its[np.where(its > 0)]
fname = get_output_dir("interses") + "/day_" + str(d)
print "Writing to file - ", fname
write_data(fname, nits)
def process_single_day(day):
SES_CLIENT=0
SES_AP=1
SES_START=2
SES_END=3
SNMP_INB=0
SNMP_OUTB=1
seslen_qry = "select client,ap,start,end from seslen where day = " + str(day)
rses = RunSQL("syslog_final.db")
rsnmp = RunSQL("unc.db")
sesres = rses.sqlq(seslen_qry)
inb = list()
outb = list()
for single_ses in sesres:
snmp_qry = "select sum(bytr), sum(byts) from snmp where client = " + str(single_ses[SES_CLIENT]) + " and ap = " + str(single_ses[SES_AP]) + " and ts >= " + str(single_ses[SES_START]) + " and ts <= " + str(single_ses[SES_END]) + " and day = " + str(day) + " group by client, ap"
snmpres = rsnmp.sqlq(snmp_qry)
if ( snmpres != None and len(snmpres) > 0 and snmpres[0][SNMP_INB] > 0 and snmpres[0][SNMP_OUTB] > 0 ):
inb.append(snmpres[0][SNMP_INB])
outb.append(snmpres[0][SNMP_OUTB])
return (inb, outb)
def main(args):
r1 = RunSQL("files_and_analysis.db")
r2 = RunSQL("logn_fits.db")
unique_id = args[0].upper()
distribution = args[1].upper()
r1q = "select type, ks_fit, q_fit from fits where unique_id='" + unique_id + "' and distribution='" + distribution + "'"
r2q = "select type, ks_fit, q_fit from logn_fits where unique_id='" + unique_id + "'"
if "LOGN" == distribution:
# Do something
rs = r2.sqlq(r2q)
for rss in rs:
print distribution + "-" + rss[0] + "-" + "KS: " + kstex(rss[1])
print distribution + "-" + rss[0] + "-" + "FIT:" + qprint(
rss[2]) + " " + qtex(rss[2])
else:
# Do something
rs = r1.sqlq(r1q)
rss = rs[0]
print distribution + "-" + rss[0] + "-" + "KS: " + kstex(rss[1])
print distribution + "-" + rss[0] + "-" + "FIT:" + qprint(
rss[2]) + " " + qtex(rss[2])
def processClientVisits(dayNum, clientNum):
TS=0
AP=1
EVENT=2
REASON=3
qry = "select ts, ap, event, reason from syslog where client = " + str(clientNum) + " and day = " + str(dayNum) + " order by ts,ap"
r = RunSQL("syslog_tmp.db")
l = r.sqlq(qry)
current_ap = None
event_map = dict()
for l1 in l:
event_ts = l1[TS]
event_ap = l1[AP]
event = l1[EVENT]
event_reason = l1[REASON]
emap_key = str(clientNum) + "-" + str(event_ap)
if emap_key in event_map:
# Exists
if ( "deauthenticated" == event.lower() or "disassociated" == event.lower() ):
if ( event_reason.lower().find("is leaving") > 0 or event_reason.lower().find("success") > 0 ):
event_entry = event_map[emap_key]
event_entry["ets"] = event_ts
event_entry["valid"] = True
event_list.append(event_entry)
event_map.pop(emap_key)
else:
#First entry
if ( "associated" == event.lower() or "reassociated" == event.lower() ):
event_entry = dict()
event_entry["day"] = dayNum
event_entry["sts"] = event_ts
event_entry["ets"] = None
event_entry["valid"] = False
event_entry["ap"] = event_ap
event_entry["client"] = clientNum
event_entry["event"] = event
event_map[emap_key] = event_entry
else:
## Do nothing - invalid
pass
return event_list
def main(args):
r2 = RunSQL("logn_fits.db")
unique_id = args[0].upper()
r2q = "select type, mu, sigma from logn_fits where unique_id='" + unique_id + "'"
rs = r2.sqlq(r2q)
txstr = "$%MLE-MU%$ & $%MLE-SIG%$ & $%MME-MU%$ & $%MME-SIG%$ & $%FITMIN-MU%$ & $%FITMIN-SIG%$ \\tabularnewline"
for rss in rs:
print "LOGN-" + rss[0]
print "\t mu: " + vtex(rss[1])
print "\t sigma: " + vtex(rss[2])
replace_str_mu = "%"+rss[0]+"-MU%"
replace_str_sig = "%"+rss[0]+"-SIG%"
txstr = txstr.replace(replace_str_mu, vtex(rss[1]))
txstr = txstr.replace(replace_str_sig, vtex(rss[2]))
print txstr
def main(args):
r2 = RunSQL("logn_fits.db")
unique_id = args[0].upper()
r2q = "select type, mu, sigma from logn_fits where unique_id='" + unique_id + "'"
rs = r2.sqlq(r2q)
txstr = "$%MLE-MU%$ & $%MLE-SIG%$ & $%MME-MU%$ & $%MME-SIG%$ & $%FITMIN-MU%$ & $%FITMIN-SIG%$ \\tabularnewline"
for rss in rs:
print "LOGN-" + rss[0]
print "\t mu: " + vtex(rss[1])
print "\t sigma: " + vtex(rss[2])
replace_str_mu = "%" + rss[0] + "-MU%"
replace_str_sig = "%" + rss[0] + "-SIG%"
txstr = txstr.replace(replace_str_mu, vtex(rss[1]))
txstr = txstr.replace(replace_str_sig, vtex(rss[2]))
print txstr
def getClientList(dayNum):
qry = "select distinct client from syslog where client > 0 and day = " + str(dayNum)
r = RunSQL("syslog_tmp.db")
clientList = r.sqlq(qry)
return clientList
def getDayList():
r = RunSQL("syslog_tmp.db")
dayList = r.sqlq("select distinct day from syslog")
return dayList
def get_dataset_info(tag):
q = "select filename from datasets where unique_id = '" + tag + "'"
r = RunSQL("files_and_analysis.db")
rs = r.sqlq(q)
filename = rs[0].replace("/home/gautam", os.getenv("HOME"))
return {"dir": os.path.dirname(filename), "file": os.path.basename(filename)}