def execute(self, context):
"""This method monitors the environment
An [in]finite loop monitoring the cluster nodes for unhealthy ones
Args:
context (dict): context for the command to use
Returns:
bool: Command Success
"""
if context.get('foreground'):
self.ioc.getLogger().foreground = True
monitor = NodeMonitoring(self.ioc)
if context.get('loop'):
i = 0
while i < int(context.get('loop', 0)):
if not monitor.monitor():
self.ioc.getLogger().error("Monitoring Process Failed", notify=False)
else:
monitor.scanComplete()
i += 1
time.sleep(5)
else:
while True:
if not monitor.monitor():
self.ioc.getLogger().error("Monitoring Process Failed", notify=False)
else:
monitor.scanComplete()
# sleep for a random interval to ensure not all nodes poll at the same time
time.sleep(int(str(random.choice(range(1, 5))) + str(random.choice(range(0, 9)))))
if context.get('foreground'):
self.ioc.getLogger().foreground = False
return True
def test_scanComplete(self):
n = NodeMonitoring()
originalServer = n.ioc.getCollection('JobServer').find_one(
{'_id': ObjectId(n.ioc.getConfig().NodeIdentity)})
self.assertTrue(originalServer)
originalServer = dict(originalServer)
originalSourcing = n.ioc.getCollection('SourceData').find({
'grease_data.execution.server':
ObjectId(originalServer.get('_id'))
}).count()
n.scanComplete()
newSourcing = n.ioc.getCollection('SourceData').find({
'grease_data.execution.server':
ObjectId(originalServer.get('_id'))
}).count()
newServer = n.ioc.getCollection('JobServer').find_one(
{'_id': ObjectId(n.ioc.getConfig().NodeIdentity)})
self.assertTrue(newServer)
newServer = dict(newServer)
self.assertGreaterEqual(newSourcing, originalSourcing)
self.assertGreaterEqual(newServer.get('jobs'),
originalServer.get('jobs'))
n.ioc.getCollection('SourceData').drop()
n.ioc.getCollection('JobServer').update_one(
{'_id': ObjectId(originalServer.get('_id'))},
{'$set': {
'jobs': originalServer.get('jobs', 0)
}})