def eventCreator():
aLotOfData=[]
tries=0
while(True):
d=q.get()
m=json.loads(d)
data = {
'_type': 'throughput'
}
# print(m)
source=m['meta']['source']
destination=m['meta']['destination']
data['MA']=m['meta']['measurement_agent']
data['src']=source
data['dest']=destination
so=siteMapping.getPS(source)
de=siteMapping.getPS(destination)
if so!= None:
data['srcSite']=so[0]
data['srcVO']=so[1]
if de!= None:
data['destSite']=de[0]
data['destVO']=de[1]
data['srcProduction']=siteMapping.isProductionThroughput(source)
data['destProduction']=siteMapping.isProductionThroughput(destination)
if not 'datapoints'in m:
print(threading.current_thread().name, 'no datapoints in this message!')
q.task_done()
continue
su=m['datapoints']
for ts, th in su.iteritems():
dati=datetime.utcfromtimestamp(float(ts))
data['_index']="network_weather_2-"+str(dati.year)+"."+str(dati.month)+"."+str(dati.day)
data['timestamp']=int(float(ts)*1000)
data['throughput']=th
#print(data)
aLotOfData.append(copy.copy(data))
q.task_done()
if len(aLotOfData)>100:
reconnect=True
try:
res = helpers.bulk(es, aLotOfData, raise_on_exception=False,request_timeout=60)
print (threading.current_thread().name, "\t inserted:",res[0], '\tErrors:',res[1])
aLotOfData=[]
reconnect=False
except es_exceptions.ConnectionError as e:
print('ConnectionError ', e)
except es_exceptions.TransportError as e:
print('TransportError ', e)
except helpers.BulkIndexError as e:
print(e[0])
print(e[1][0])
#for i in e[1]:
# print(i)
except:
print('Something seriously wrong happened. ')
if reconnect: es = GetESConnection()
def eventCreator():
aLotOfData = []
es_conn = tools.get_es_connection()
while True:
d = q.get()
m = json.loads(d)
data = {'_type': 'doc'}
# print(m)
source = m['meta']['source']
destination = m['meta']['destination']
data['MA'] = m['meta']['measurement_agent']
data['src'] = source
data['dest'] = destination
data['src_host'] = m['meta']['input_source']
data['dest_host'] = m['meta']['input_destination']
data['ipv6'] = False
if ':' in source or ':' in destination:
data['ipv6'] = True
so = siteMapping.getPS(source)
de = siteMapping.getPS(destination)
if so != None:
data['src_site'] = so[0]
data['src_VO'] = so[1]
if de != None:
data['dest_site'] = de[0]
data['dest_VO'] = de[1]
data['src_production'] = siteMapping.isProductionThroughput(source)
data['dest_production'] = siteMapping.isProductionThroughput(
destination)
if not 'datapoints' in m:
print(threading.current_thread().name,
'no datapoints in this message!')
q.task_done()
continue
su = m['datapoints']
for ts, th in su.items():
dati = datetime.utcfromtimestamp(float(ts))
data['_index'] = INDEX_PREFIX + str(dati.year) + "." + str(
dati.month) # + "." + str(dati.day)
data['timestamp'] = int(float(ts) * 1000)
data['throughput'] = th
# print(data)
aLotOfData.append(copy.copy(data))
q.task_done()
if len(aLotOfData) > 100:
succ = tools.bulk_index(
aLotOfData,
es_conn=es_conn,
thread_name=threading.current_thread().name)
if succ is True:
aLotOfData = []
def eventCreator():
aLotOfData = []
es_conn = tools.get_es_connection()
while True:
d = q.get()
m = json.loads(d)
data = {
'_type': 'retransmits'
}
# print(m)
source = m['meta']['source']
destination = m['meta']['destination']
data['MA'] = m['meta']['measurement_agent']
data['src'] = source
data['dest'] = destination
data['ipv6'] = False
if ':' in source or ':' in destination:
data['ipv6'] = True
so = siteMapping.getPS(source)
de = siteMapping.getPS(destination)
if so != None:
data['srcSite'] = so[0]
data['srcVO'] = so[1]
if de != None:
data['destSite'] = de[0]
data['destVO'] = de[1]
data['srcProduction'] = siteMapping.isProductionThroughput(source)
data['destProduction'] = siteMapping.isProductionThroughput(
destination)
if not 'datapoints'in m:
print(threading.current_thread().name,
'no datapoints in this message!')
q.task_done()
continue
su = m['datapoints']
for ts, th in su.iteritems():
dati = datetime.utcfromtimestamp(float(ts))
data['_index'] = "network_weather-" + \
str(dati.year) + "." + str(dati.month) + "." + str(dati.day)
data['timestamp'] = int(float(ts) * 1000)
data['retransmits'] = th
# print(data)
aLotOfData.append(copy.copy(data))
q.task_done()
if len(aLotOfData) > 100:
succ = tools.bulk_index(aLotOfData, es_conn=es_conn, thread_name=threading.current_thread().name)
if succ is True:
aLotOfData = []
def eventCreator(self, message):
"""
"""
m = json.loads(message)
data = {}
# print(m)
source = m['meta']['source']
destination = m['meta']['destination']
data['MA'] = m['meta']['measurement_agent']
data['src'] = source
data['dest'] = destination
data['src_host'] = m['meta']['input_source']
data['dest_host'] = m['meta']['input_destination']
data['ipv6'] = False
if ':' in source or ':' in destination:
data['ipv6'] = True
so = siteMapping.getPS(source)
de = siteMapping.getPS(destination)
if so != None:
data['src_site'] = so[0]
data['src_VO'] = so[1]
if de != None:
data['dest_site'] = de[0]
data['dest_VO'] = de[1]
data['src_production'] = siteMapping.isProductionThroughput(source)
data['dest_production'] = siteMapping.isProductionThroughput(
destination)
if not 'datapoints' in m:
print(threading.current_thread().name,
'no datapoints in this message!')
return
su = m['datapoints']
for ts, th in su.items():
data['_index'] = self.INDEX
data['timestamp'] = int(float(ts) * 1000)
sha1_hash = hashlib.sha1()
sha1_hash.update(m['meta']['org_metadata_key'].encode())
sha1_hash.update(str(data['timestamp']).encode())
data['_id'] = sha1_hash.hexdigest()
data['throughput'] = th
# print(data)
self.aLotOfData.append(copy.copy(data))
def eventCreator(self, message):
"""
"""
m = json.loads(message)
data = {'_type': 'doc'}
# print(m)
source = m['meta']['source']
destination = m['meta']['destination']
data['MA'] = m['meta']['measurement_agent']
data['src'] = source
data['dest'] = destination
data['src_host'] = m['meta']['input_source']
data['dest_host'] = m['meta']['input_destination']
data['ipv6'] = False
if ':' in source or ':' in destination:
data['ipv6'] = True
so = siteMapping.getPS(source)
de = siteMapping.getPS(destination)
if so != None:
data['src_site'] = so[0]
data['src_VO'] = so[1]
if de != None:
data['dest_site'] = de[0]
data['dest_VO'] = de[1]
data['src_production'] = siteMapping.isProductionThroughput(source)
data['dest_production'] = siteMapping.isProductionThroughput(
destination)
if not 'datapoints' in m:
print(threading.current_thread().name,
'no datapoints in this message!')
return
su = m['datapoints']
for ts, th in su.items():
dati = datetime.utcfromtimestamp(float(ts))
data['_index'] = self.es_index_prefix + self.INDEX_PREFIX + str(
dati.year) + "." + str(dati.month) # + "." + str(dati.day)
data['timestamp'] = int(float(ts) * 1000)
data['_id'] = hash(
(m['meta']['org_metadata_key'], data['timestamp']))
data['throughput'] = th
# print(data)
self.aLotOfData.append(copy.copy(data))
def eventCreator(self, message):
m = json.loads(message)
data = {'_type': 'doc'}
# print(m)
source = m['meta']['source']
destination = m['meta']['destination']
data['MA'] = m['meta']['measurement_agent']
data['src'] = source
data['dest'] = destination
data['src_host'] = m['meta']['input_source']
data['dest_host'] = m['meta']['input_destination']
data['ipv6'] = False
if ':' in source or ':' in destination:
data['ipv6'] = True
so = siteMapping.getPS(source)
de = siteMapping.getPS(destination)
if so != None:
data['src_site'] = so[0]
data['src_VO'] = so[1]
if de != None:
data['dest_site'] = de[0]
data['dest_VO'] = de[1]
data['src_production'] = siteMapping.isProductionThroughput(source)
data['dest_production'] = siteMapping.isProductionThroughput(
destination)
if not 'datapoints' in m:
print(threading.current_thread().name,
"no datapoints found in the message")
return
dp = m['datapoints']
# print(su)
for ts in dp:
dati = datetime.utcfromtimestamp(float(ts))
data['_index'] = self.es_index_prefix + self.INDEX_PREFIX + str(
dati.year) + "." + str(dati.month) + "." + str(dati.day)
data['timestamp'] = int(float(ts) * 1000)
data['_id'] = hash(
(m['meta']['org_metadata_key'], data['timestamp']))
data['hops'] = []
data['rtts'] = []
data['ttls'] = []
hops = dp[ts]
for hop in hops:
if 'ttl' not in hop or 'ip' not in hop or 'query' not in hop:
continue
nq = int(hop['query'])
if nq != 1:
continue
data['hops'].append(hop['ip'])
data['ttls'].append(int(hop['ttl']))
if 'rtt' in hop and hop['rtt'] != None:
data['rtts'].append(float(hop['rtt']))
else:
data['rtts'].append(0.0)
# print(data)
hs = ''
for h in data['hops']:
if h == None:
hs += "None"
else:
hs += h
data['n_hops'] = len(data['hops'])
if len(data['rtts']):
data['max_rtt'] = max(data['rtts'])
data['hash'] = hash(hs)
self.aLotOfData.append(copy.copy(data))
def eventCreator():
aLotOfData = []
es_conn = tools.get_es_connection()
while True:
d = q.get()
m = json.loads(d)
data = {
'_type': 'traceroute'
}
# print(m)
source = m['meta']['source']
destination = m['meta']['destination']
data['MA'] = m['meta']['measurement_agent']
data['src'] = source
data['dest'] = destination
data['src_host'] = m['meta']['input_source']
data['dest_host'] = m['meta']['input_destination']
data['ipv6'] = False
if ':' in source or ':' in destination:
data['ipv6'] = True
so = siteMapping.getPS(source)
de = siteMapping.getPS(destination)
if so != None:
data['srcSite'] = so[0]
data['srcVO'] = so[1]
if de != None:
data['destSite'] = de[0]
data['destVO'] = de[1]
data['srcProduction'] = siteMapping.isProductionThroughput(source)
data['destProduction'] = siteMapping.isProductionThroughput(
destination)
if not 'datapoints' in m:
q.task_done()
print(threading.current_thread().name,
"no datapoints found in the message")
continue
dp = m['datapoints']
# print(su)
for ts in dp:
dati = datetime.utcfromtimestamp(float(ts))
data['_index'] = "network_weather-" + \
str(dati.year) + "." + str(dati.month) + "." + str(dati.day)
data['timestamp'] = int(float(ts) * 1000)
data['hops'] = []
data['rtts'] = []
data['ttls'] = []
hops = dp[ts]
for hop in hops:
if 'ttl' not in hop or 'ip' not in hop or 'query' not in hop:
continue
nq = int(hop['query'])
if nq != 1:
continue
data['hops'].append(hop['ip'])
data['ttls'].append(int(hop['ttl']))
if 'rtt' in hop and hop['rtt'] != None:
data['rtts'].append(float(hop['rtt']))
else:
data['rtts'].append(0.0)
# print(data)
hs = ''
for h in data['hops']:
if h == None:
hs += "None"
else:
hs += h
data['n_hops'] = len(data['hops'])
if len(data['rtts']):
data['max_rtt'] = max(data['rtts'])
data['hash'] = hash(hs)
aLotOfData.append(copy.copy(data))
q.task_done()
if len(aLotOfData) > 100:
succ = tools.bulk_index(aLotOfData, es_conn=es_conn, thread_name=threading.current_thread().name)
if succ is True:
aLotOfData = []
def eventCreator():
tries=0
aLotOfData=[]
while(True):
d=q.get()
m=json.loads(d)
data = {
'_type': 'traceroute'
}
# print(m)
source=m['meta']['source']
destination=m['meta']['destination']
data['MA']=m['meta']['measurement_agent']
data['src']=source
data['dest']=destination
so=siteMapping.getPS(source)
de=siteMapping.getPS(destination)
if so!= None:
data['srcSite']=so[0]
data['srcVO']=so[1]
if de!= None:
data['destSite']=de[0]
data['destVO']=de[1]
data['srcProduction']=siteMapping.isProductionThroughput(source)
data['destProduction']=siteMapping.isProductionThroughput(destination)
if not 'datapoints' in m:
q.task_done()
print(threading.current_thread().name, "no datapoints found in the message")
continue
dp=m['datapoints']
# print(su)
for ts in dp:
dati=datetime.utcfromtimestamp(float(ts))
data['_index']="network_weather_2-"+str(dati.year)+"."+str(dati.month)+"."+str(dati.day)
data['timestamp']=int(float(ts)*1000)
data['hops']=[]
data['rtts']=[]
data['ttls']=[]
hops = dp[ts]
for hop in hops:
if 'ttl' not in hop or 'ip' not in hop or 'query' not in hop : continue
nq=int(hop['query'])
if nq!=1: continue
data['hops'].append(hop['ip'])
data['ttls'].append(int(hop['ttl']))
if 'rtt' in hop and hop['rtt']!=None:
data['rtts'].append(float(hop['rtt']))
else:
data['rtts'].append(0.0)
# print(data)
hs=''
for h in data['hops']:
if h==None:
hs+="None"
else:
hs+=h
data['hash']=hash(hs)
aLotOfData.append(copy.copy(data))
q.task_done()
if len(aLotOfData)>100:
reconnect=True
try:
res = helpers.bulk(es, aLotOfData, raise_on_exception=False,request_timeout=60)
print(threading.current_thread().name, "\t inserted:",res[0], '\tErrors:',res[1])
aLotOfData=[]
reconnect=False
except es_exceptions.ConnectionError as e:
print('ConnectionError ', e)
except es_exceptions.TransportError as e:
print('TransportError ', e)
except helpers.BulkIndexError as e:
print(e[0])
print(e[1][0])
# for i in e[1]:
# print(i)
except:
print('Something seriously wrong happened.')
if reconnect: es = GetESConnection()
def eventCreator():
aLotOfData = []
es_conn = tools.get_es_connection()
while True:
d = q.get()
m = json.loads(d)
data = {
'_type': 'traceroute'
}
# print(m)
source = m['meta']['source']
destination = m['meta']['destination']
data['MA'] = m['meta']['measurement_agent']
data['src'] = source
data['dest'] = destination
data['ipv6'] = False
if ':' in source or ':' in destination:
data['ipv6'] = True
so = siteMapping.getPS(source)
de = siteMapping.getPS(destination)
if so != None:
data['srcSite'] = so[0]
data['srcVO'] = so[1]
if de != None:
data['destSite'] = de[0]
data['destVO'] = de[1]
data['srcProduction'] = siteMapping.isProductionThroughput(source)
data['destProduction'] = siteMapping.isProductionThroughput(
destination)
if not 'datapoints' in m:
q.task_done()
print(threading.current_thread().name,
"no datapoints found in the message")
continue
dp = m['datapoints']
# print(su)
for ts in dp:
dati = datetime.utcfromtimestamp(float(ts))
data['_index'] = "network_weather-" + \
str(dati.year) + "." + str(dati.month) + "." + str(dati.day)
data['timestamp'] = int(float(ts) * 1000)
data['hops'] = []
data['rtts'] = []
data['ttls'] = []
hops = dp[ts]
for hop in hops:
if 'ttl' not in hop or 'ip' not in hop or 'query' not in hop:
continue
nq = int(hop['query'])
if nq != 1:
continue
data['hops'].append(hop['ip'])
data['ttls'].append(int(hop['ttl']))
if 'rtt' in hop and hop['rtt'] != None:
data['rtts'].append(float(hop['rtt']))
else:
data['rtts'].append(0.0)
# print(data)
hs = ''
for h in data['hops']:
if h == None:
hs += "None"
else:
hs += h
data['n_hops'] = len(data['hops'])
data['max_rtt'] = max(data['rtts'])
data['hash'] = hash(hs)
aLotOfData.append(copy.copy(data))
q.task_done()
if len(aLotOfData) > 100:
succ = tools.bulk_index(aLotOfData, es_conn=es_conn, thread_name=threading.current_thread().name)
if succ is True:
aLotOfData = []
def eventCreator(self, message):
m = json.loads(message)
data = {}
# print(m)
source = m['meta']['source']
destination = m['meta']['destination']
data['MA'] = m['meta']['measurement_agent']
data['src'] = source
data['dest'] = destination
data['src_host'] = m['meta']['input_source']
data['dest_host'] = m['meta']['input_destination']
data['ipv6'] = False
if ':' in source or ':' in destination:
data['ipv6'] = True
so = siteMapping.getPS(source)
de = siteMapping.getPS(destination)
if so != None:
data['src_site'] = so[0]
data['src_VO'] = so[1]
if de != None:
data['dest_site'] = de[0]
data['dest_VO'] = de[1]
data['src_production'] = siteMapping.isProductionThroughput(source)
data['dest_production'] = siteMapping.isProductionThroughput(
destination)
if not 'datapoints' in m:
print(threading.current_thread().name,
"no datapoints found in the message")
return
dp = m['datapoints']
# print(su)
for ts in dp:
data['_index'] = self.INDEX
data['timestamp'] = int(float(ts) * 1000)
sha1_hash = hashlib.sha1()
sha1_hash.update(m['meta']['org_metadata_key'].encode())
sha1_hash.update(str(data['timestamp']).encode())
data['_id'] = sha1_hash.hexdigest()
data['hops'] = []
data['asns'] = []
data['rtts'] = []
data['ttls'] = []
hops = dp[ts]
for hop in hops:
if 'ttl' not in hop or 'ip' not in hop or 'query' not in hop:
continue
nq = int(hop['query'])
if nq != 1:
continue
data['hops'].append(hop['ip'])
data['ttls'].append(int(hop['ttl']))
if 'rtt' in hop and hop['rtt'] != None:
data['rtts'].append(float(hop['rtt']))
else:
data['rtts'].append(0.0)
if 'as' in hop:
data['asns'].append(hop['as']['number'])
else:
data['asns'].append(0)
# print(data)
data['n_hops'] = len(data['hops'])
if len(data['rtts']):
data['max_rtt'] = max(data['rtts'])
if len(data['hops']) == 0:
print('ERROR: we should have no data without any hops.')
self.aLotOfData.append(copy.copy(data))
continue
data['destination_reached'] = False
core_path = copy.copy(data['hops'])
if core_path[-1] == data['dest']:
core_path.remove(data['dest'])
# destination has been reached if the last hop is == destination
data['destination_reached'] = True
route_hash = hashlib.sha1()
route_hash.update(";".join(core_path).encode())
data['route-sha1'] = route_hash.hexdigest()
# path incomplete means len(ttls) is not equal to the last ttl
data['path_complete'] = False
if len(data['ttls']) == data['ttls'][-1]:
data['path_complete'] = True
# looping path contains at least one non-unique IP. it includes src and dest.
core_path.append(data['src'])
core_path.append(data['dest'])
data['looping'] = len(set(core_path)) != len(core_path)
self.aLotOfData.append(copy.copy(data))