def eventCreator():
aLotOfData = []
es_conn = tools.get_es_connection()
while True:
d = q.get()
m = json.loads(d)
data = {'_type': 'doc'}
source = m['meta']['source']
destination = m['meta']['destination']
data['MA'] = m['meta']['measurement_agent']
data['src'] = source
data['dest'] = destination
data['src_host'] = m['meta']['input_source']
data['dest_host'] = m['meta']['input_destination']
data['ipv6'] = False
if ':' in source or ':' in destination:
data['ipv6'] = True
so = siteMapping.getPS(source)
de = siteMapping.getPS(destination)
if so is not None:
data['src_site'] = so[0]
data['src_VO'] = so[1]
if de is not None:
data['dest_site'] = de[0]
data['dest_VO'] = de[1]
data['src_production'] = siteMapping.isProductionLatency(source)
data['dest_production'] = siteMapping.isProductionLatency(destination)
if 'datapoints' not in m:
q.task_done()
print(threading.current_thread().name,
"no datapoints found in the message")
continue
su = m['datapoints']
# print(su)
for ts, th in su.items():
dati = datetime.utcfromtimestamp(float(ts))
data['_index'] = INDEX_PREFIX + str(dati.year) + "." + str(
dati.month) + "." + str(dati.day)
data['timestamp'] = int(float(ts) * 1000)
data['packet_loss'] = th
# print(data)
aLotOfData.append(copy.copy(data))
q.task_done()
if len(aLotOfData) > 500:
succ = tools.bulk_index(
aLotOfData,
es_conn=es_conn,
thread_name=threading.current_thread().name)
if succ is True:
aLotOfData = []
def eventCreator():
aLotOfData = []
es_conn = tools.get_es_connection()
while True:
d = q.get()
m = json.loads(d)
data = {'_type': 'doc'}
source = m['meta']['source']
destination = m['meta']['destination']
data['MA'] = m['meta']['measurement_agent']
data['src'] = source
data['dest'] = destination
so = siteMapping.getPS(source)
de = siteMapping.getPS(destination)
if so is not None:
data['src_site'] = so[0]
data['src_VO'] = so[1]
if de is not None:
data['dest_site'] = de[0]
data['dest_VO'] = de[1]
data['src_production'] = siteMapping.isProductionLatency(source)
data['dest_production'] = siteMapping.isProductionLatency(destination)
if 'summaries' not in m:
q.task_done()
print(threading.current_thread().name,
"no summaries found in the message")
continue
su = m['summaries']
for s in su:
if s['summary_window'] == '60' and s[
'summary_type'] == 'statistics':
results = s['summary_data']
# print(results)
for r in results:
dati = datetime.utcfromtimestamp(float(r[0]))
data['_index'] = INDEX_PREFIX + str(dati.year) + "." + str(
dati.month) + "." + str(dati.day)
data['timestamp'] = r[0] * 1000
data['sim_util'] = r[1]['ml']
# print(data)
aLotOfData.append(copy.copy(data))
q.task_done()
if len(aLotOfData) > 500:
succ = tools.bulk_index(
aLotOfData,
es_conn=es_conn,
thread_name=threading.current_thread().name)
if succ is True:
aLotOfData = []
def eventCreator():
aLotOfData = []
es_conn = tools.get_es_connection()
while True:
d = q.get()
m = json.loads(d)
data = {
'_type': 'packet_loss_rate'
}
source = m['meta']['source']
destination = m['meta']['destination']
data['MA'] = m['meta']['measurement_agent']
data['src'] = source
data['dest'] = destination
data['ipv6'] = False
if ':' in source or ':' in destination:
data['ipv6'] = True
so = siteMapping.getPS(source)
de = siteMapping.getPS(destination)
if so is not None:
data['srcSite'] = so[0]
data['srcVO'] = so[1]
if de is not None:
data['destSite'] = de[0]
data['destVO'] = de[1]
data['srcProduction'] = siteMapping.isProductionLatency(source)
data['destProduction'] = siteMapping.isProductionLatency(destination)
if 'datapoints' not in m:
q.task_done()
print(threading.current_thread().name, "no datapoints found in the message")
continue
su = m['datapoints']
# print(su)
for ts, th in su.iteritems():
dati = datetime.utcfromtimestamp(float(ts))
data['_index'] = "network_weather-" + str(dati.year) + "." + str(dati.month) + "." + str(dati.day)
data['timestamp'] = int(float(ts) * 1000)
data['packet_loss'] = th
# print(data)
aLotOfData.append(copy.copy(data))
q.task_done()
if len(aLotOfData) > 500:
succ = tools.bulk_index(aLotOfData, es_conn=es_conn, thread_name=threading.current_thread().name)
if succ is True:
aLotOfData = []
def eventCreator():
aLotOfData = []
es_conn = tools.get_es_connection()
while True:
d = q.get()
m = json.loads(d)
data = {
'_type': 'link_utilization'
}
source = m['meta']['source']
destination = m['meta']['destination']
data['MA'] = m['meta']['measurement_agent']
data['src'] = source
data['dest'] = destination
so = siteMapping.getPS(source)
de = siteMapping.getPS(destination)
if so is not None:
data['srcSite'] = so[0]
data['srcVO'] = so[1]
if de is not None:
data['destSite'] = de[0]
data['destVO'] = de[1]
data['srcProduction'] = siteMapping.isProductionLatency(source)
data['destProduction'] = siteMapping.isProductionLatency(destination)
if 'summaries' not in m:
q.task_done()
print(threading.current_thread().name, "no summaries found in the message")
continue
su = m['summaries']
for s in su:
if s['summary_window'] == '60' and s['summary_type'] == 'statistics':
results = s['summary_data']
# print(results)
for r in results:
dati = datetime.utcfromtimestamp(float(r[0]))
data['_index'] = "network_weather-" + str(dati.year) + "." + str(dati.month) + "." + str(dati.day)
data['timestamp'] = r[0] * 1000
data['sim_util'] = r[1]['ml']
#print(data)
aLotOfData.append(copy.copy(data))
q.task_done()
if len(aLotOfData) > 500:
succ = tools.bulk_index(aLotOfData, es_conn=es_conn, thread_name=threading.current_thread().name)
if succ is True:
aLotOfData = []
def eventCreator(self, message):
m = json.loads(message)
data = {}
source = m['meta']['source']
destination = m['meta']['destination']
data['MA'] = m['meta']['measurement_agent']
data['src'] = source
data['dest'] = destination
data['src_host'] = m['meta']['input_source']
data['dest_host'] = m['meta']['input_destination']
data['ipv6'] = False
if ':' in source or ':' in destination:
data['ipv6'] = True
so = siteMapping.getPS(source)
de = siteMapping.getPS(destination)
if so is not None:
data['src_site'] = so[0]
data['src_VO'] = so[1]
if de is not None:
data['dest_site'] = de[0]
data['dest_VO'] = de[1]
data['src_production'] = siteMapping.isProductionLatency(source)
data['dest_production'] = siteMapping.isProductionLatency(destination)
if 'datapoints' not in m:
print(threading.current_thread().name,
"no datapoints found in the message")
return
su = m['datapoints']
# print(su)
for ts, th in su.items():
data['_index'] = self.INDEX
data['timestamp'] = int(float(ts) * 1000)
sha1_hash = hashlib.sha1()
sha1_hash.update(m['meta']['org_metadata_key'].encode())
sha1_hash.update(str(data['timestamp']).encode())
data['_id'] = sha1_hash.hexdigest()
data['packet_loss'] = th
# print(data)
self.aLotOfData.append(copy.copy(data))
def eventCreator(self, message):
m = json.loads(message)
data = {'_type': 'doc'}
source = m['meta']['source']
destination = m['meta']['destination']
data['MA'] = m['meta']['measurement_agent']
data['src'] = source
data['dest'] = destination
data['src_host'] = m['meta']['input_source']
data['dest_host'] = m['meta']['input_destination']
data['ipv6'] = False
if ':' in source or ':' in destination:
data['ipv6'] = True
so = siteMapping.getPS(source)
de = siteMapping.getPS(destination)
if so is not None:
data['src_site'] = so[0]
data['src_VO'] = so[1]
if de is not None:
data['dest_site'] = de[0]
data['dest_VO'] = de[1]
data['src_production'] = siteMapping.isProductionLatency(source)
data['dest_production'] = siteMapping.isProductionLatency(destination)
if 'datapoints' not in m:
print(threading.current_thread().name,
"no datapoints found in the message")
return
su = m['datapoints']
# print(su)
for ts, th in su.items():
dati = datetime.utcfromtimestamp(float(ts))
data['_index'] = self.es_index_prefix + self.INDEX_PREFIX + str(
dati.year) + "." + str(dati.month) + "." + str(dati.day)
data['timestamp'] = int(float(ts) * 1000)
data['_id'] = hash(
(m['meta']['org_metadata_key'], data['timestamp']))
data['packet_loss'] = th
# print(data)
self.aLotOfData.append(copy.copy(data))
def eventCreator(self, message):
m = json.loads(message)
data = {'_type': 'doc'}
source = m['meta']['source']
destination = m['meta']['destination']
data['MA'] = m['meta']['measurement_agent']
data['src'] = source
data['dest'] = destination
so = siteMapping.getPS(source)
de = siteMapping.getPS(destination)
if so is not None:
data['src_site'] = so[0]
data['src_VO'] = so[1]
if de is not None:
data['dest_site'] = de[0]
data['dest_VO'] = de[1]
data['src_production'] = siteMapping.isProductionLatency(source)
data['dest_production'] = siteMapping.isProductionLatency(destination)
if 'summaries' not in m:
print(threading.current_thread().name,
"no summaries found in the message")
return
su = m['summaries']
for s in su:
if s['summary_window'] == '60' and s[
'summary_type'] == 'statistics':
results = s['summary_data']
# print(results)
for r in results:
dati = datetime.utcfromtimestamp(float(r[0]))
data['_index'] = self.es_index_prefix + INDEX_PREFIX + str(
dati.year) + "." + str(dati.month) + "." + str(
dati.day)
data['timestamp'] = r[0] * 1000
data['_id'] = hash(
(m['meta']['org_metadata_key'], data['timestamp']))
data['sim_util'] = r[1]['ml']
# print(data)
self.aLotOfData.append(copy.copy(data))
def eventCreator(self, message):
m = json.loads(message)
data = {}
source = m['meta']['source']
destination = m['meta']['destination']
data['MA'] = m['meta']['measurement_agent']
data['src'] = source
data['dest'] = destination
so = siteMapping.getPS(source)
de = siteMapping.getPS(destination)
if so is not None:
data['src_site'] = so[0]
data['src_VO'] = so[1]
if de is not None:
data['dest_site'] = de[0]
data['dest_VO'] = de[1]
data['src_production'] = siteMapping.isProductionLatency(source)
data['dest_production'] = siteMapping.isProductionLatency(destination)
if 'summaries' not in m:
print(threading.current_thread().name,
"no summaries found in the message")
return
su = m['summaries']
for s in su:
if s['summary_window'] == '60' and s[
'summary_type'] == 'statistics':
results = s['summary_data']
# print(results)
for r in results:
data['_index'] = self.INDEX
data['timestamp'] = r[0] * 1000
sha1_hash = hashlib.sha1()
sha1_hash.update(m['meta']['org_metadata_key'].encode())
sha1_hash.update(str(data['timestamp']).encode())
data['_id'] = sha1_hash.hexdigest()
data['sim_util'] = r[1]['ml']
# print(data)
self.aLotOfData.append(copy.copy(data))
def eventCreator():
aLotOfData = []
es_conn = tools.get_es_connection()
while True:
d = q.get()
m = json.loads(d)
data = {'_type': 'latency'}
source = m['meta']['source']
destination = m['meta']['destination']
data['MA'] = m['meta']['measurement_agent']
data['src'] = source
data['dest'] = destination
data['src_host'] = m['meta']['input_source']
data['dest_host'] = m['meta']['input_destination']
data['ipv6'] = False
if ':' in source or ':' in destination:
data['ipv6'] = True
so = siteMapping.getPS(source)
de = siteMapping.getPS(destination)
if so is not None:
data['srcSite'] = so[0]
data['srcVO'] = so[1]
if de is not None:
data['destSite'] = de[0]
data['destVO'] = de[1]
data['srcProduction'] = siteMapping.isProductionLatency(source)
data['destProduction'] = siteMapping.isProductionLatency(destination)
su = m['datapoints']
for ts, th in su.iteritems():
dati = datetime.utcfromtimestamp(float(ts))
data['_index'] = "network_weather-" + str(dati.year) + "." + str(
dati.month) + "." + str(dati.day)
data['timestamp'] = int(float(ts) * 1000)
th_fl = dict((float(k), v) for (k, v) in th.items())
# mean
samples = sum([v for k, v in th_fl.items()])
th_mean = sum(k * v for k, v in th_fl.items()) / samples
data['delay_mean'] = th_mean
# std dev
data['delay_sd'] = math.sqrt(
sum((k - th_mean)**2 * v for k, v in th_fl.items()) / samples)
# median
csum = 0
ordered_th = [(k, v) for k, v in sorted(th_fl.items())]
midpoint = samples // 2
if samples % 2 == 0: # even number of samples
for index, entry in enumerate(ordered_th):
csum += entry[1]
if csum > midpoint + 1:
data['delay_median'] = entry[0]
break
elif csum == midpoint:
data['delay_median'] = entry[0] + ordered_th[index +
1][0] / 2
break
elif csum == midpoint + 1 and index == 0:
data['delay_median'] = entry[0]
break
elif csum == midpoint + 1 and index > 0:
data['delay_median'] = entry[0] + ordered_th[index -
1][0] / 2
break
else: # odd number of samples
for index, entry in enumerate(ordered_th):
csum += entry[1]
if csum >= midpoint + 1:
data['delay_median'] = entry[0]
break
aLotOfData.append(copy.copy(data))
q.task_done()
if len(aLotOfData) > 500:
succ = tools.bulk_index(
aLotOfData,
es_conn=es_conn,
thread_name=threading.current_thread().name)
if succ is True:
aLotOfData = []
def eventCreator(self, message):
m = json.loads(message)
data = {'_type': 'doc'}
source = m['meta']['source']
destination = m['meta']['destination']
data['MA'] = m['meta']['measurement_agent']
data['src'] = source
data['dest'] = destination
data['src_host'] = m['meta']['input_source']
data['dest_host'] = m['meta']['input_destination']
data['ipv6'] = False
if ':' in source or ':' in destination:
data['ipv6'] = True
so = siteMapping.getPS(source)
de = siteMapping.getPS(destination)
if so is not None:
data['src_site'] = so[0]
data['src_VO'] = so[1]
if de is not None:
data['dest_site'] = de[0]
data['dest_VO'] = de[1]
data['src_production'] = siteMapping.isProductionLatency(source)
data['dest_production'] = siteMapping.isProductionLatency(destination)
su = m['datapoints']
for ts, th in su.items():
dati = datetime.utcfromtimestamp(float(ts))
data['_index'] = self.es_index_prefix + self.INDEX_PREFIX + str(
dati.year) + "." + str(dati.month) + "." + str(dati.day)
data['timestamp'] = int(float(ts) * 1000)
data['_id'] = hash((m['meta']['org_metadata_key'], ts))
th_fl = dict((float(k), v) for (k, v) in th.items())
# mean
samples = sum([v for k, v in th_fl.items()])
th_mean = sum(k * v for k, v in th_fl.items()) / samples
data['delay_mean'] = th_mean
# std dev
data['delay_sd'] = math.sqrt(
sum((k - th_mean)**2 * v for k, v in th_fl.items()) / samples)
# median
csum = 0
ordered_th = [(k, v) for k, v in sorted(th_fl.items())]
midpoint = samples // 2
if samples % 2 == 0: # even number of samples
for index, entry in enumerate(ordered_th):
csum += entry[1]
if csum > midpoint + 1:
data['delay_median'] = entry[0]
break
elif csum == midpoint:
data['delay_median'] = entry[0] + ordered_th[index +
1][0] / 2
break
elif csum == midpoint + 1 and index == 0:
data['delay_median'] = entry[0]
break
elif csum == midpoint + 1 and index > 0:
data['delay_median'] = entry[0] + ordered_th[index -
1][0] / 2
break
else: # odd number of samples
for index, entry in enumerate(ordered_th):
csum += entry[1]
if csum >= midpoint + 1:
data['delay_median'] = entry[0]
break
self.aLotOfData.append(copy.copy(data))
def eventCreator():
aLotOfData=[]
tries=0
while(True):
d=q.get()
m=json.loads(d)
data = {
'_type': 'latency'
}
source=m['meta']['source']
destination=m['meta']['destination']
data['MA']=m['meta']['measurement_agent']
data['src']=source
data['dest']=destination
so=siteMapping.getPS(source)
de=siteMapping.getPS(destination)
if so!= None:
data['srcSite']=so[0]
data['srcVO']=so[1]
if de!= None:
data['destSite']=de[0]
data['destVO']=de[1]
data['srcProduction']=siteMapping.isProductionLatency(source)
data['destProduction']=siteMapping.isProductionLatency(destination)
if not 'summaries'in m:
q.task_done()
print(threading.current_thread().name, "no summaries found in the message")
continue
su=m['summaries']
for s in su:
if s['summary_window']=='300' and s['summary_type']=='statistics':
results=s['summary_data']
#print(results)
for r in results:
dati=datetime.utcfromtimestamp(float(r[0]))
data['_index']="network_weather_2-"+str(dati.year)+"."+str(dati.month)+"."+str(dati.day)
data['timestamp']=r[0]*1000
data['delay_mean']=r[1]['mean']
data['delay_median']=r[1]['median']
data['delay_sd']=r[1]['standard-deviation']
#print(data)
aLotOfData.append(copy.copy(data))
q.task_done()
if len(aLotOfData)>500:
reconnect=True
# print('writing out data...')
try:
res = helpers.bulk(es, aLotOfData, raise_on_exception=True,request_timeout=60)
print(threading.current_thread().name, "\t inserted:",res[0], '\tErrors:',res[1])
aLotOfData=[]
reconnect=False
except es_exceptions.ConnectionError as e:
print('ConnectionError ', e)
except es_exceptions.TransportError as e:
print('TransportError ', e)
except helpers.BulkIndexError as e:
print(e[0])
# for i in e[1]:
# print(i)
except:
print('Something seriously wrong happened.')
if reconnect: es = GetESConnection()