def diurnalPDF(ts,val,bw,mode='median'): # ts is time in seconds # val are the values of time series # bw in bin size # mode is median or mean ts,val=order(ts,val) dl=86400 nbc=lambda x: (x[0]/x[1] if (x[0]%x[1]==0) else x[0]/x[1]+1) nb=nbc((dl,bw)) a=min(ts) b=max(ts) ts=[(xx-a)%dl for xx in ts] i=0 d=num.array([-1.0]*nb) tm=num.array([-1.0]*nb) his=0 for i in range(nb): b1=i*bw b2=(i+1)*bw for j in range(his,len(ts)): if ts[j]>=b2: break if j-his>0: if mode=='median': d[i]=num.median(val[his:j]) elif mode=='mean': d[i]=num.mean(temp) else: return None tm[i]=b1 i=i+1 his=j ind=d>-1 rv=d[ind] sp=sum(rv) return (tm[ind],rv/sp)
def tsA(fName):
dF='CSV/'+fName+'.db'
if not os.path.exists(dF):
print 'Error: Database does not exist'
return
D=sq.connect(dF)
cur=D.cursor()
qq="""select cP,Community,SPD,lt,download_rate,cAS
from meta
where
SPD not null and Community not null and SPD!=-1 and cP not null"""
cur.execute(qq)
A=cur.fetchall()
D={}
SP={}
for w in A:
try:
D[(w[0],w[1],w[2],w[-1])]=D[(w[0],w[1],w[2],w[-1])]+[(w[3],w[4])]
except KeyError:
D[(w[0],w[1],w[2],w[-1])]=[(w[3],w[4])]
for w in D.keys():
temp=D[w]
if len(temp)<700:
del D[w]
#~ pref,com,sp=zip(*D.keys())
#~ d={}
#~ for k,p in enumerate(pref):
#~ try:
#~ d[p].append((com[k],sp[k]))
#~ except KeyError:
#~ d[p]=[(com[k],sp[k])]
#~ for ww in d.items():
#~ print ww
#~ raw_input('=================>')
p=[]
l=str(len(D.keys()))
for j,w in enumerate(D.keys()):
print str(j)+' '+l
temp=D[w]
cP='"'+w[0]+'"'
C=str(w[1])
qq="select lt,download_rate\n\
from meta\n\
where\n \
SPD=-1 and Community="+C+" and cP="+cP+";"
cur.execute(qq)
A=cur.fetchall()
lt2,r2=zip(*A)
lt,r=zip(*temp)
ts=lt+lt2
at=min(ts)
y=r+r2
tsd,zd=diurnalPDF(ts,y,2*60*60)
ew=KL(zd)
p.append(ew)
if ew<.3:
continue
#~ y,x=CDF(p)
#~ pl.plot(x,y)
#~ pl.show()
print tsd
print zd
pl.plot(tsd,zd)
pl.show()
raw_input('============>')
ts,y=order(ts,y)
tsp=[float(xx-at)/(3600) for xx in ts]
out=eventDetection(y,sigma=num.var(r),l=30,p=0.01)
#~ if len(out)>0:
if True:
f,ax=pl.subplots(3,sharex=True)
ax[0].plot(tsp,y)
ax[0].set_ylabel('Download(Mbps)')
for xx in out:
if xx[1]>0:
ax[1].plot([tsp[xx[0]],tsp[xx[0]]],[0,xx[1]],'b-',lw=3)
if xx[1]<0:
ax[1].plot([tsp[xx[0]],tsp[xx[0]]],[0,xx[1]],'r-',lw=3)
ax[1].axhline(0,lw=2,color='black')
xax,Z=bina(ts,y,4*60*60)
ax[2].plot(xax,Z,'c--')
ax[2].set_xlabel('Time in hour')
pl.suptitle('AS'+str(w[-1]),fontsize=20)
pl.show()
recPlot(Z,2)
pl.show()
pn=0
down=[]
for i in ii:
ad='CSV/D-'+fName+'/uos'+i
mb=[]
t=[]
with open(ad,'r') as f:
k=0
for line in f:
if k==0:
k=1
continue
mb.append(float(line.split(',')[3]))
t.append(float(line.split(',')[1]))
down=down+mb
t,a1=order(t,mb)
xmin=min(min(t),xmin)
xmax=max(max(t),xmax)
tt.append(t)
out.append(eventDetection(a1,num.std(a1),l,p))
pl.subplot(2,2,pn)
pn=pn+1
pl.plot(t,a1)
pl.xlabel('Time',fontsize=20)
pl.suptitle('Download Throughput for different uos in comcast',fontsize=20)
pl.show()
#~ f, (ax1, ax2) = pl.subplots(2, sharex=True, sharey=False)
#~ ax1.plot(range(len(a1)),a1)
#~ pl.ylabel('Sequence Magnitude',fontsize=20)
#~ pl.xlabel('Sample Number',fontsize=20)
for j in [0,1,2,3]:
# This function is to see the analyze the statistics of
# MLAB NDT test within a specific month
import os
from ip2net import ip2net
from myBasic import list2dic,order
import pickle as pk
if __name__=='__main__':
th=4000
fName='ndt201401'
bgpFile='01jan14'
ipl=[]
with open('CSV/'+fName) as f:
for line in f:
ip=line.split(',')[0].strip()
ipl.append(ip)
del ipl[0]
A=ip2net(bgpFile,ipl)
print 'Obtaining Stats...'
asn=[xx[0] for xx in A if xx]
pref=[xx[1] for xx in A if xx]
o=list2dic(asn)
asp1=[(o[w],w) for w in o.keys() if o[w]>4000]
unused,asp=order(*zip(*asp1),mode=1)
f=open('Prolific/asn','w')
pk.dump(asp,f)
f.close()
