def __call__(cls, dateFrom=None, dateTo=None, *args):
print('* OBJECT DICT ', len(cls._dict), cls._dict)
if (dateFrom is None) or (dateTo is None):
defaultDT = hp.defaultTimeRange()
dateFrom = defaultDT[0]
dateTo = defaultDT[1]
if (dateFrom, dateTo) in cls._dict:
print('** OBJECT EXISTS', cls, dateFrom, dateTo)
instance = cls._dict[(dateFrom, dateTo)]
else:
print('** OBJECT DOES NOT EXIST', cls, dateFrom, dateTo)
if (len(cls._dict) > 0) and ([dateFrom, dateTo] !=
cls._registered):
print('*** provide the latest and start thread', cls, dateFrom,
dateTo)
instance = cls._dict[list(cls._dict.keys())[-1]]
refresh = threading.Thread(target=cls.nextPeriodData,
args=(dateFrom, dateTo, *args))
refresh.start()
elif ([dateFrom, dateTo] == cls._registered):
print('*** provide the latest', cls, dateFrom, dateTo)
instance = cls._dict[list(cls._dict.keys())[-1]]
elif (len(cls._dict) == 0):
print('*** no data yet, refresh and wait', cls, dateFrom,
dateTo)
cls.nextPeriodData(dateFrom, dateTo, *args)
instance = cls._dict[(dateFrom, dateTo)]
# keep only a few objects in memory
if len(cls._dict) >= 2:
cls._dict.pop(list(cls._dict.keys())[0])
return instance
def queryNodesGeoLocation():
include=["geolocation","external_address.ipv4_address", "external_address.ipv6_address", "config.site_name", "host"]
period = hp.GetTimeRanges(*hp.defaultTimeRange(days=30))
query = {
"query": {
"bool": {
"filter": [
{
"range": {
"timestamp": {
"gte": period[0],
"lte": period[1]
}
}
}
]
}
}
}
data = scan(client=hp.es, index='ps_meta', query=query,
_source=include, filter_path=['_scroll_id', '_shards', 'hits.hits._source'])
count = 0
neatdata = []
ddict = {}
for res in data:
if not count%100000: print(count)
data = res['_source']
if 'config' in data:
site = data['config']['site_name']
else: site = None
if 'ipv4_address' in data['external_address']:
ip = data['external_address']['ipv4_address']
else: ip = data['external_address']['ipv6_address']
geoip = [None, None]
if 'geolocation' in data:
geoip = data['geolocation'].split(",")
# if 'speed' in data['external_address']:
# speed = data['external_address']['speed']
if (ip in ddict) and (site is not None):
ddict[ip]['site'] = site
else: ddict[ip] = {'lat': geoip[0], 'lon': geoip[1], 'site': site, 'host':data['host']}
count=count+1
return ddict
def UpdateAllData(self):
print()
print(
f'{datetime.now()} New data is on its way at {datetime.utcnow()}')
print('Active threads:', threading.active_count())
# query period must be the same for all data loaders
defaultDT = hp.defaultTimeRange()
GeneralDataLoader(defaultDT[0], defaultDT[1])
SiteDataLoader(defaultDT[0], defaultDT[1])
PrtoblematicPairsDataLoader(defaultDT[0], defaultDT[1])
SitesRanksDataLoader(defaultDT[0], defaultDT[1])
self.lastUpdated = hp.roundTime(datetime.utcnow())
self.StartThread()
# Sends the alarms
@timer
def sendAlarms(data):
ALARM = alarms('Networking', 'RENs', 'path changed')
for issue in data:
# print(issue)
ALARM.addAlarm(
body="Path changed",
tags=issue['sites'],
source=issue
)
# query the past 12 hours and split the period into 8 time ranges
dateFrom, dateTo = hp.defaultTimeRange(12)
# dateFrom, dateTo = ['2022-05-25 09:40', '2022-05-25 21:40']
runInParallel(dateFrom, dateTo)
df = pd.DataFrame(list(data))
df['pair'] = df['src']+'-'+df['dest']
df = fixMissingMetadata(df)
asn2Hop, hop2ASN, max_ttl = mapHopsAndASNs(df)
cricDict = getCricASNInfo()
altASNsDict = getAltASNs(asn2Hop, hop2ASN)
altsOfAlts = getAltsOfAlts(altASNsDict)
mapASNsManualy(513, 20969)
mapASNsManualy(20969, 513)
mapASNsManualy(291, 293)
rawDf['dest_site'] = rawDf['dest_site'].fillna(rawDf.pop('site_dest'))
rawDf['src_host'] = rawDf['src_host'].fillna(rawDf.pop('host_src'))
rawDf['dest_host'] = rawDf['dest_host'].fillna(rawDf.pop('host_dest'))
return rawDf
"""
'src', 'src_host', 'src_site': info about the source
'dest', 'dest_host', 'dest_site': info about the destination
'avg_value': the average value for the pair
'tests_done': % of tests done for the whole period. The calculation is based on the assumption
that there should be 1 measure per minute
"""
dateFrom, dateTo = hp.defaultTimeRange(hours=24)
dateFromF, dateToF = dateFrom.replace(' ', 'T'), dateTo.replace(' ', 'T')
plsDf = markPairs(dateFrom, dateTo)
plsDf = plsDf[plsDf['tests_done'] != '0%']
plsDf['from'] = dateFromF
plsDf['to'] = dateToF
cols = [
'from', 'to', 'src', 'dest', 'src_host', 'dest_host', 'src_site',
'dest_site', 'avg_value', 'tests_done'
]
##### High packet loss #####
sign_ploss = plsDf[(plsDf['flag'] == 1)][cols]
sign_ploss['avg_value%'] = round(sign_ploss['avg_value'] * 100, 1)