def satsignal():
maincount = 3
while maincount > 0:
d('getting signal strength')
msg = 'AT+CSQ\r'
satuart.write(msg)
# this will discard our echo, blank line etc
# now we hope for reply
# at some read we will get CSQ:3\r\n
count = 60
while count > 0:
ret = str(satuart.readline())
#d(ret)
csqpos = ret.find("CSQ:")
if (csqpos >= 0) and (len(ret) > 5):
strength = int(ret[4 + csqpos])
if (strength > 0):
return strength
else:
d('STRENGTH IS ZERO - trying again.')
break
count = count - 1
sleep_ms(100)
sleep(2)
maincount = maincount - 1
return (0)
def gen_pathlen_graph(buckets,outfile,ipv6):
""" Generate graph from buckets of each day. Graph pathlengths (one point per day). """
common.d('gen_pathlen_graph genering', outfile)
graph.gen_lineplot([((i+1),avg_pathlen(b)) for i,b in enumerate(buckets)],
outfile, xlabel='Prefix length', ylabel='Avg path len',
title='\# of hops')
def gpsloop():
global finalgps
fixcount = 0
#turn GPS on
GPSpower.value(1)
t = 0
gps10 = []
# wait for typ warm-up time
d('waiting for warmup period')
sleep(11)
d('starting gps loop')
while t < positiontimeout:
start = pyb.millis()
nmea = gpsuart.readline()
if (nmea == None) or (len(nmea) < 32):
continue
thetype, data = processGPS(nmea)
if (thetype == 't'):
#We got a timestamp
d('got timestamp')
(gpsYY, gpsMM, gpsDD, gpshh, gpsmm, gpsss) = data
# if gpstime is > 10s different from extrtc - set extrtc
#BROKEN RTC if(abs(gpsYY-YY)>0 or abs(gpsMM-MM)>0 or abs(gpsDD-DD)>0 or abs(gpshh-hh)>0 or abs(gpsmm-mm)>0 or abs(gpsss-ss)>10):
#update external RTC. We're >10s out
#d('Setting ext RTC')
#extrtc.set_time(gpsYY, gpsMM, gpsDD, gpshh, gpsmm, gpsss)
elif (thetype == 'p'):
#We got some positional data, may need to say q=4 ?
d('got position')
(lat, lon, alt, qual, hdop, sats, nmeafix) = data
if (int(qual) > QUALTHRESH):
#count happy GPS fix.
fixcount += 1
else:
#Theres been some kind of problem. Timeout?
#d('No data received - Timeout?')
pass
#once we have seen 15 fixes we store and exit QUICKHACK
if (fixcount >= 15):
# store it in file
with open('data.txt', 'a') as file:
# assume we saw timedate for now
tostore = gpsYY[
2:] + gpsMM + gpsDD + gpshh + gpsmm + "," + nmeafix
file.write(tostore)
break
#Set the time for our positiontimeout
t = pyb.millis() - start
# turn GPS off
d('GPS off')
GPSpower.value(0)
def satloop():
data = []
payload = ""
with open('data.txt','r') as file:
i = 0
for line in file:
#Check we are only sending a few readings.
i = i + 1
if(i>8):
break
#Load readings into file
line_Str=file.readline()
line_Str=line_Str.rstrip('\n')
line_Str=line_Str.rstrip('\r')
data.append(line)
for item in data:
payload = payload + data
# turn SAT on
SATpower.value(1)
#wait for sat to boot
waitforsat(satuart)
strength = satsignal(satuart)
if strength != None:
d('Sat strength: ' + strength)
else:
pass
def gen_pathlen_textfile(buckets,outfile,ipv6):
""" Gen textfile from buckets of one day (=list 1..32 or 128 of lists of pathlenghts).
"""
common.d('gen_pathlen_textfile genering', outfile)
with open(outfile,'w') as of:
for l in format_buckets(buckets):
of.write(l+"\n")
def run():
s = 0
e = 5
for e in xrange(5,10000):
dd = d(e)
if d(dd)==e and dd!=e:
s += e
return s
def gen_pathlen_timegraphs(bucket_matrix, filenamepfx, ipv6=True):
""" Generate graphs pathlen4-<number> when number is the length of the examined
prefix. Graph contains average length of paths with the prefix length set by number.
It also creates 3d graph pathlen4-3d with all prefix lenght in one dimension.
And finally, it creates pathlen4-avg which graphs average for all prefix lengths.
:param bucket_matrix: Matrix representing the buckets
:param str filenamepfx: File name prefix to create files and write them
:param bool ipv6: IPv6 flag
"""
rng=32
if ipv6:
rng=128
pfxlen = []
for i in range(0,rng+1):
pfxlen.append([])
avg = []
d3d = []
if ipv6:
filenamepfx=filenamepfx+'/pathlen6-'
else:
filenamepfx=filenamepfx+'/pathlen4-'
times=sorted(bucket_matrix.keys())
for t in times:
ts=str(t)
avgt=0
nonzerocnt=0
for i in range(0,rng+1):
a=avg_pathlen(bucket_matrix[t][i])
if not a==0:
d3d.append((ts,i,a))
pfxlen[i].append((ts,a))
avgt+=a
if a>0:
nonzerocnt+=1
if nonzerocnt > 0:
avg.append((ts,avgt/float(nonzerocnt)))
if avg:
common.d("bgp.gen_pathlen_timegraph creating", filenamepfx+'avg')
graph.gen_lineplot(avg,filenamepfx+'avg', ylabel='Total avg path len', title='\# of hops')
for i in range(0,rng+1):
if pfxlen[i]:
common.d("bgp.gen_pathlen_timegraph creating", filenamepfx+str(i))
graph.gen_lineplot(pfxlen[i],filenamepfx+str(i), ylabel='Avg path len', title='\# of hops')
if d3d:
graph.gen_3dplot(d3d,filenamepfx+'3d', ylabel='Prefix length', zlabel=r'Avg path \nlength', title='\# of hops')
def waitforOK():
count = 20
while count > 0:
ret = satuart.read()
if ret == None:
count = count - 1
sleep_ms(50)
else:
d('got OK')
return (True)
return (False)
def gen_pathlen_graph(buckets,outfile,ipv6):
""" Generate graph from buckets of each day. Graph pathlengths (one point per day).
:param buckets: List representing the buckets
:param str outfile: File name to write
:param bool ipv6: IPv6 flag
"""
common.d('gen_pathlen_graph genering', outfile)
graph.gen_lineplot([((i+1),avg_pathlen(b)) for i,b in enumerate(buckets)],
outfile, xlabel='Prefix length', ylabel='Avg path len',
title='\# of hops')
def gen_pathlen_textfile(buckets,outfile,ipv6):
""" Gen textfile from buckets of one day (=list 1..32 or 128 of lists of pathlenghts).
:param buckets: List representing the buckets
:param str outfile: File name to write
:param bool ipv6: IPv6 flag
"""
common.d('gen_pathlen_textfile genering', outfile)
with open(outfile,'w') as of:
for l in format_buckets(buckets):
of.write(l+"\n")
def waitforREADY():
count = 20
while count > 0:
ret = satuart.read()
if ret == None:
count = count - 1
sleep_ms(50)
else:
ret = str(ret)
if ret.find("READY") > 0:
d('got READY')
return (True)
return (False)
def module_process(ianadir, host, days, ipv6=False, bestonly=False):
"""
Match BGP prefixes in IANA's directory and generate text
outputs and stats that determine average active prefix counts
and average de-aggregation for each RIR.
:param IanaDirectory ianadir: IanaDirectory instance to match agains
:param str host: Host to take BGP feeds from
:param days: List of days to analyze
:param bool ipv6: IPv6 flag
:param bool bestonly: Take only best BGP paths into account
"""
timeline=[]
timelineavg=[]
for t in days:
rirpfxlens={}
ifn = bgp.bgpdump_pickle(t, host, ipv6)
if not ifn:
continue
bgpdump=common.load_pickle(ifn)
common.d("ianaspace.module_run: matching prefixes in a tree (%d)"%len(bgpdump))
for pv in bgpdump:
if bestonly and not (pv[0] and '>' in pv[0]):
continue
net = ipaddr.IPNetwork(pv[1])
r=ianadir.resolve_network(net)
if not r:
common.w("No IANA assignment for", str(pv[1]))
continue
name=r[2]
if r[1] == 'LEGACY' and not name in RIRS:
name='LEGACY'
if not name in rirpfxlens:
rirpfxlens[name]=[]
rirpfxlens[name].append(net.prefixlen)
timeline.append([str(t)]+[len(rirpfxlens[n]) for n in RIRS])
timelineavg.append([str(t)]+[(reduce(lambda x, y: x + y, rirpfxlens[n])/
float(len(rirpfxlens[n]))) for n in RIRS])
outtxt = '%s/rirstats%d-%s.txt'%(common.resultdir(t), (6 if ipv6 else 4), host)
common.d("Generating output RIR stats text "+outtxt)
with open(outtxt,'w') as f:
for i,k in enumerate(RIRS):
f.write('%s: %d (avg pfxlen: %.2f)\n'%(str(k), timeline[-1][1+i],
round(timelineavg[-1][1+i], 2)))
if timeline:
outgraph = '%s/rirpfxcount%d-%s'%(common.resultdir(), (6 if ipv6 else 4), host)
common.d("Generating output RIR pfxcount graph with prefix "+outgraph)
graph.gen_multilineplot(timeline, outgraph, legend=RIRS, ylabel='Pfx count')
if timelineavg:
outgraph = '%s/rirpfxlen%d-%s'%(common.resultdir(), (6 if ipv6 else 4), host)
common.d("Generating output RIR pfxlen graph with prefix "+outgraph)
graph.gen_multilineplot(timelineavg, outgraph, legend=RIRS, ylabel='Avg pfx len')
def waitforsentOK():
count = 200
while count > 0:
ret = str(satuart.readline())
d('Count is ' + str(count))
#d(ret)
pos = ret.find("+SBDIX:")
if (pos >= 0) and (len(ret) > 5):
fields = ret.split(',')
rcode = fields[0].split(' ')[1]
d('returncode' + str(rcode))
#d(rcode)
if rcode == '0':
return (True)
count = count - 1
sleep_ms(200)
return (False)
def create_path_matrix(host, days, ipv6=False):
""" Generate matrix: [t:buckets,...] where buckets (r) contains
r[16]=[x,y,z,...] ; x,y,z are ints. It means that there was
prefixes with netmask /16. One with AS-path length x, another y, ...
"""
bucket_matrix={}
for t in days:
bgpfile=bgpdump_pickle(t, host, ipv6)
if not bgpfile:
common.d("bgp.create_path_matrix skipping time "+str(t)+"...")
continue
common.d("bgp.create_path_matrix processing time "+str(t)+"...")
bgpdump=common.load_pickle(bgpfile)
bucket_matrix[t]=gen_buckets(bgpdump, ipv6, bestonly=True)
return bucket_matrix
def gpsloop():
#turn GPS on
GPSpower.value(1)
t = 0
gps10 = []
# wait for typ warm-up time
sleep(10)
while t<positiontimeout:
start = millis()
nmea = gpsuart.readline()
thetype, data = processGPS(nmea)
if(thetype=='t'):
#We got a timestamp
d('got timestamp')
(gpsYY, gpsMM, gpsDD, gpshh, gpsmm, gpsss, gpswday gpsn1, gpsn2) = data
# if gpstime is > 10s different from extrtc - set extrtc
if(abs(gpsYY-YY)>0 or abs(gpsMM-MM)>0 or abs(gpsDD-DD)>0 or abs(gpshh-hh)>0 or abs(gpsmm-mm)>0 or abs(gpsss-ss)>10):
#update external RTC. We're >10s out
d('Setting ext RTC')
extrtc.set_time(gpsYY, gpsMM, gpsDD, gpshh, gpsmm, gpsss)
elif(thetype=='p'):
#We got some positional data
d('got position')
(lat,lon,alt,qual,hdop,sats) = data
if(qual>=4):
#We are happy with the quality of the GPS fix.
finalgps = data
gps10.append([lat,lon])
else:
#Theres been some kind of problem. Timeout?
d('No data received - Timeout?')
pass
#Only store last 10 gps readings in array
if(gps10.length>10):
gps10.pop(0)
#Set the time for our positiontimeout
t = millis()-start
# turn GPS off
GPSpower.value(0)
def module_preprocess(bgp_hosts, bgp_data, ipv6=False):
"""
Runs Cisco parser and parse files from data like
data/marge/bgp-ipv4-2014-04-01-01-17-01.txt.bz2
and creates
results/2014-04-01/bgp4-marge.pickle
Returns list of Time objects.
bgp_hosts: list of hostnames
bgp_data: hash bgp_host -> source directory
"""
out_days = []
for host in bgp_hosts:
for t,fn in module_listdays(bgp_hosts, bgp_data, ipv6):
common.d('BGP in:', fn, 'time:', t)
outdir = common.resultdir(t)
outfile = bgpdump_pickle(t, host, ipv6, False)
if os.path.isfile(outfile):
common.d('BGP out:', outfile, 'exists. Skip.')
else:
common.d('BGP out:', outfile)
cisco.gen_bgpdump_pickle(fn, outfile, ipv6)
def create_path_matrix(host, days, ipv6=False):
""" Generate matrix: [t:buckets,...] where buckets (r) contains
r[16]=[x,y,z,...] ; x,y,z are ints. It means that there was
prefixes with netmask /16. One with AS-path length x, another y, ...
:param str host: Host name to analyze
:param days: List of Day obj. to analyze
:param bool ipv6: IPv6 flag
:returns: Bucket matrix
"""
bucket_matrix={}
for t in days:
bgpfile=bgpdump_pickle(t, host, ipv6)
if not bgpfile:
common.d("bgp.create_path_matrix skipping time "+str(t)+"...")
continue
common.d("bgp.create_path_matrix processing time "+str(t)+"...")
bgpdump=common.load_pickle(bgpfile)
bucket_matrix[t]=gen_buckets(bgpdump, ipv6, bestonly=True)
return bucket_matrix
def run():
abn=[]
for a in xrange(12, MX_N):
if d(a) > a:
abn.append(a)
n = [1] * MX_N
n[0] = 0
for a in xrange(len(abn)):
for b in xrange(a+1):
c = abn[a] + abn[b]
if c < len(n):
n[c] = 0
return sum(a if n[a] else 0 for a in xrange(len(n)))
def gen_prefixcount_timegraphs(bucket_matrix, filenamepfx, ipv6=False):
""" Generate graphs pfxcount4-<number> that shows how many prefixes
of the length <number> was in DFZ at the defined time. It also generates
graph pfxcount-sum that shows all the prefixes regardless of prefix length.
:param bucket_matrix: Matrix representing the buckets
:param str filenamepfx: File name prefix to create files and write them
:param bool ipv6: IPv6 flag
"""
rng=32
if ipv6:
rng=128
if ipv6:
filenamepfx=filenamepfx+'/pfxcount6-'
else:
filenamepfx=filenamepfx+'/pfxcount4-'
sumall=[]
avgpfxlen=[]
counts=[]
for i in range(0,rng+1):
counts.append([])
times=sorted(bucket_matrix.keys())
for t in times:
s=0
apl=0
ts=str(t)
for i in range(0,rng+1):
cnt=len(bucket_matrix[t][i])
s+=cnt
apl+=cnt*i
counts[i].append((ts,cnt))
sumall.append((ts,s))
avgpfxlen.append((ts,(float(apl)/s)))
if sumall:
common.d("bgp.gen_prefixcount_timegraph creating", filenamepfx+'sum')
graph.gen_lineplot(sumall,filenamepfx+'sum', ylabel='Prefix count', title='\# of pfxes')
if avgpfxlen:
common.d("bgp.gen_prefixcount_timegraph creating", filenamepfx+'avgpfxlen')
graph.gen_lineplot(avgpfxlen,filenamepfx+'avgpfxlen', ylabel='avg pfx length', title='BGP average')
for i in range(0,rng+1):
if counts[i]:
common.d("bgp.gen_prefixcount_timegraph creating", filenamepfx+str(i))
graph.gen_lineplot(counts[i],filenamepfx+str(i), ylabel='/%d pfx count'%i, title='\# of pfxes')
def waitforsat():
count = 10
while count > 0:
sleep(1)
d('sending AT')
satuart.write('AT\r')
#discard our echo
satuart.readline()
#utime.sleep_ms(200)
ret = satuart.readline()
if ret != None:
d(ret)
count = 0
return (True)
else:
count = count - 1
d('nothing yet')
return (False)
def sendmsg(msg):
d('sending message')
#txt = 'AT+SBDWT=' + msg + '\r'
#satuart.write(txt)
#waitforOK()
satuart.write('AT+SBDWT\r')
waitforREADY()
txt = msg + '\r'
satuart.write(txt)
sleep(1)
# was it ok?
satuart.write('AT+SBDIX\r')
#discard our echo
satuart.readline()
# probably need sleep 1 or 2
sleep(1)
count = 200
if waitforsentOK():
d('message sent')
return (True)
else:
d('message failed')
return (False)
def module_preprocess(bgp_hosts, bgp_data, ipv6=False):
""" Runs Cisco parser and parse files from data like
data/marge/bgp-ipv4-2014-04-01-01-17-01.txt.bz2
and creates
results/2014-04-01/bgp4-marge.pickle
Returns list of Time objects.
:param bgp_hosts: list of hostnames
:param bgp_data: hash bgp_host -> source directory
"""
out_days = []
for host in bgp_hosts:
for t,fn in module_listdays(bgp_hosts, bgp_data, ipv6):
common.d('BGP in:', fn, 'time:', t)
outdir = common.resultdir(t)
outfile = bgpdump_pickle(t, host, ipv6, False)
if os.path.isfile(outfile):
common.d('BGP out:', outfile, 'exists. Skip.')
else:
common.d('BGP out:', outfile)
cisco.gen_bgpdump_pickle(fn, outfile, ipv6)
payload = payload + data
# turn SAT on
SATpower.value(1)
#wait for sat to boot
waitforsat(satuart)
strength = satsignal(satuart)
if strength != None:
d('Sat strength: ' + strength)
else:
pass
# send fix via sat
d('Sending fix via satellite')
sendmsg(satuart, ''.join(finalgps))
d('Done interfacing with sat.')
#turn SAT off
SATpower.value(0)
#we're done.
return
def gpsloop():
#turn GPS on
GPSpower.value(1)
