def pingTargets(targets):
fpingBinary = ''
try:
fpingBinary = config.getSharedConfigValue('check.fpingBinary')
except KeyError as err:
fpingBinary = ''
if (fpingBinary == '' or not os.path.isfile(fpingBinary)
or not os.access(fpingBinary, os.X_OK)):
fpingBinary = searchFpingBinary()
hosts = []
addressTranslationTable = {}
for target in targets:
if ('address' in target):
hosts.append(target['address'])
addressTranslationTable[target['address']] = target['host']
else:
hosts.append(target['host'])
# Ping the targets
result = subprocess.run([
fpingBinary, '-t',
str(config.getSharedConfigValue('check.initialTimeout')), '-p',
str(config.getSharedConfigValue('check.packetInterval')), '-c',
str(config.getSharedConfigValue('check.numberOfPackets')), '-q'
] + hosts,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
# Parse the raw results
pingResults = parseRawResult(result.stdout.decode('utf-8'),
addressTranslationTable)
return pingResults
def getStatusForLossValue(lossAverage):
status = 'online'
if (lossAverage > float(config.getSharedConfigValue('analysis.thresholdDown'))):
status = 'offline'
elif (lossAverage > float(config.getSharedConfigValue('analysis.thresholdUnstable'))):
status = 'unstable'
return status
def received_message(self, message):
parsedMessage = json.loads(str(message))
if (parsedMessage['command'] == 'authenticate'):
if (parsedMessage['key'] == config.getSharedConfigValue(
'secretKey')):
self.isAuthorized = True
websocket.manager.add(self)
logging.info(
'Observer {} authenticated with secret key.'.format(
self.peer_address[0]))
self.send(
json.dumps({
'response': 'welcome',
'leader': collector.leader
}))
else:
self.isAuthorized = False
logging.warning(
'Authorization with invalid secret key from {}.'.format(
self.peer_address[0]))
self.send(json.dumps({'response': 'secret_key_invalid'}))
self.close(1000, 'Secret key invalid.')
else:
executeServerRequest(self, parsedMessage)
def startCollectorThread():
time.sleep(2)
analysisTimeInterval = config.getSharedConfigValue('analysis.timeInterval')
observerName = config.getLocalConfigValue('observerName')
while (not distping.exitApplication):
try:
checkConnections()
if (collector.leader == observerName and
(lastAnalysisTime + analysisTimeInterval) < time.time()):
logging.debug('Start analysis...')
analyzeTargets()
if (collector.leader != observerName and
(lastAnalysisTime + (2 * analysisTimeInterval)) < time.time()):
logging.debug(
'Leader did not collect the information. Taking control...'
)
switchLeader(time.time() - analysisTimeInterval)
if (collector.leader == False):
collector.leader = observerName
time.sleep(15)
except KeyboardInterrupt as err:
return
def getTargets():
tree = config.getSharedConfigValue('targets')
targets = []
for group in tree:
targets = targets + group['targets']
return targets
def status(self):
statusNumbers = countTargetsByStatus()
return template.renderTemplate(
'pages/index.html',
statusNumbers=statusNumbers,
observerConnections=collector.getConnectionStatistics(),
targets=config.getSharedConfigValue('targets'),
latestValues=monitor.getLatestValues())
def executeActions(eventName, data):
actions = config.getSharedConfigValue('actions')
data['sender'] = config.getLocalConfigValue('observerName')
data['sendTime'] = time.time()
for action in actions:
if (action['event'] == eventName):
logging.debug('Execute action {} for event {}.'.format(
action['name'], eventName))
executeAction(action, data)
def startMonitorThread():
while (not distping.exitApplication):
try:
if (monitor.nextCheck < time.time()):
monitor.lastCheck = int(time.time())
monitor.nextCheck = monitor.lastCheck + config.getSharedConfigValue('check.interval')
logging.debug('Perform a check...')
executeCheck()
time.sleep(1)
except KeyboardInterrupt as err:
return
def analyzeTargets():
collector.lastData = {}
observers = config.getSharedConfigValue('observers')
startTime = time.time()
for name, observerData in observers.items():
if (name == config.getLocalConfigValue('observerName')):
collector.lastData[name] = monitor.getLatestValues()
elif (name in collector.connections):
connection = collector.connections[name]
connection.send(json.dumps({'command': 'get_latest_values'}))
waitForData = True
complete = False
counter = 0
while (waitForData):
if (len(collector.lastData) == len(observers)):
logging.debug('Got all data responses from observers.')
complete = True
waitForData = False
time.sleep(0.2)
counter = counter + 1
if (counter > 50):
logging.error(
'Not all observers sent their latest usage data. Process will continue without the data.'
)
waitForData = False
logging.debug('Group the data to start the analysis.')
dataByTarget = groupDataByTarget()
logging.debug('Analyze the grouped data.')
analyzeData(dataByTarget)
switchLeader(startTime)
def checkConnections():
for name, observerData in config.getSharedConfigValue('observers').items():
if (name == config.getLocalConfigValue('observerName')):
continue
if (name in collector.connections):
connection = collector.connections[name]
if (not connection.terminated):
continue
else:
connection.close()
del connections[name]
try:
connection = websocket.DistPingClient(observerData['url'])
connection.connect()
collector.connections[name] = connection
except OSError as err:
logging.error(
'Cannot connect to "{}" ({}). Error message: {}'.format(
name, observerData['url'], str(err)))
def opened(self):
self.send(
json.dumps({
'command': 'authenticate',
'key': config.getSharedConfigValue('secretKey')
}))
def getConnectionStatistics():
return {
'total': len(config.getSharedConfigValue('observers')),
'connected': len(collector.connections) +
1 # Add 1 for this process because we are also an observer connection
}