def wake_up_one(self):
self.log.info('%s: No candidates found,' % (self.action_id,) +
'trying to wake up machine...')
sleepers = FSM.zoowrap.get_sleeping()
self.log.info('Found %d sleeping host' % (len(sleepers)))
if len(sleepers) > 0:
self.log.info(' ==== >NOW WAKEUP IS NEEDED')
url = "http://%s:%s/wake" % (sleepers[0]+"-man", "42501") # TODO port
requests.post(url)
ipc.execute('avoid')
else:
ipc.execute('avoid')
def wake_up_one(self):
self.log.info('%s: No candidates found,' % (self.action_id, ) +
'trying to wake up machine...')
sleepers = FSM.zoowrap.get_sleeping()
self.log.info('Found %d sleeping host' % (len(sleepers)))
if len(sleepers) > 0:
self.log.info(' ==== >NOW WAKEUP IS NEEDED')
url = "http://%s:%s/wake" % (sleepers[0] + "-man", "42501"
) # TODO port
requests.post(url)
ipc.execute('avoid')
else:
ipc.execute('avoid')
def quit(self):
self.quitting = True
for t in self.states_aux:
self.notify_if_up(t)
ipc.execute('quit')
time.sleep(0.1)
if self.statser.isAlive():
self.statser.stop = True
self.statser.cond.acquire()
self.statser.cond.notify()
self.statser.cond.release()
time.sleep(0.1)
if self.executor.isAlive():
self.log.warning("Thread Executor is alive, should i kill it?")
time.sleep(0.1)
def quit(self):
self.quitting = True
for t in self.states_aux:
self.notify_if_up(t)
ipc.execute('quit')
time.sleep(0.1)
if self.statser.isAlive():
self.statser.stop = True
self.statser.cond.acquire()
self.statser.cond.notify()
self.statser.cond.release()
time.sleep(0.1)
if self.executor.isAlive():
self.log.warning("Thread Executor is alive, should i kill it?")
time.sleep(0.1)
def run(self):
self.log.debug('Connecting to zookeeper')
self.log.debug('Delete sleeping state')
FSM.zoowrap.wake()
self.log.info('Wake up confirmed')
self.sleep()
if FSM.quitting:
self.log.warning('Quitting...')
elif FSM.state == self.name:
# self.log.info('woke up, checking...')
# if not self.check_all():
# self.log.info('Nothing detected, chilling out')
# ipc.execute('chillout')
self.log.debug('woke up, switching normal')
ipc.execute('chillout')
else:
self.log.warning(self.name+' is not the state anymore')
def run(self):
while True:
self.sleep()
if FSM.quitting:
self.log.warning('Quitting...')
return
elif FSM.state == 'confirmed':
FSM.receiving_vms[:] = [mm for mm in FSM.receiving_vms
if self.check_state_vm(mm)]
if len(FSM.receiving_vms) == 0:
self.log.info('No more receiving vms')
self.log.info('State confirmed: chilling out')
ipc.execute('chillout')
return
else:
self.log.warning(self.name+' is not the state anymore')
return
def check_state(self, loadtype, st, callback):
FSM.mona.get_alarms()
if loadtype == 'overload':
alarm = FSM.mona.get_my_alarm_over()
if loadtype == 'underload':
alarm = FSM.mona.get_my_alarm_under()
if alarm:
if alarm['state'] == st:
self.log.info('Alarm is %s!!' % st)
ipc.execute(callback)
return True
else:
self.log.debug('Alarm is not %s: %s' % (st, alarm['state']))
else:
self.log.critical('alarm %s not found!!' % loadtype)
return False
def check_state(self, loadtype, st, callback):
FSM.mona.get_alarms()
if loadtype == 'overload':
alarm = FSM.mona.get_my_alarm_over()
if loadtype == 'underload':
alarm = FSM.mona.get_my_alarm_under()
if alarm:
if alarm['state'] == st:
self.log.info('Alarm is %s!!' % st)
ipc.execute(callback)
return True
else:
self.log.debug('Alarm is not %s: %s' % (st, alarm['state']))
else:
self.log.critical('alarm %s not found!!' % loadtype)
return False
def run(self):
self.log.debug('Connecting to zookeeper')
self.log.debug('Delete sleeping state')
FSM.zoowrap.wake()
self.log.info('Wake up confirmed')
self.sleep()
if FSM.quitting:
self.log.warning('Quitting...')
elif FSM.state == self.name:
# self.log.info('woke up, checking...')
# if not self.check_all():
# self.log.info('Nothing detected, chilling out')
# ipc.execute('chillout')
self.log.debug('woke up, switching normal')
ipc.execute('chillout')
else:
self.log.warning(self.name + ' is not the state anymore')
def check_avoided(self):
FSM.mona.get_alarms()
alarm = None
if self.type == 'overload':
alarm = FSM.mona.get_my_alarm_over()
elif self.type == 'underload':
alarm = FSM.mona.get_my_alarm_under()
if alarm:
self.log.debug('Recheck state: '+str(alarm['state']))
if alarm['state'] == 'ALARM':
ipc.execute('retry')
return True
elif alarm['state'] == 'OK' or alarm['state'] == 'UNDEFINED':
ipc.execute('chillout')
return True
else:
self.log.critical('alarm '+self.type+'not found!!')
return False
def run(self):
while True:
self.sleep()
if FSM.quitting:
self.log.warning('Quitting...')
return
elif FSM.state == 'confirmed':
FSM.receiving_vms[:] = [
mm for mm in FSM.receiving_vms if self.check_state_vm(mm)
]
if len(FSM.receiving_vms) == 0:
self.log.info('No more receiving vms')
self.log.info('State confirmed: chilling out')
ipc.execute('chillout')
return
else:
self.log.warning(self.name + ' is not the state anymore')
return
def check_avoided(self):
FSM.mona.get_alarms()
alarm = None
if self.type == 'overload':
alarm = FSM.mona.get_my_alarm_over()
elif self.type == 'underload':
alarm = FSM.mona.get_my_alarm_under()
if alarm:
self.log.debug('Recheck state: ' + str(alarm['state']))
if alarm['state'] == 'ALARM':
ipc.execute('retry')
return True
elif alarm['state'] == 'OK' or alarm['state'] == 'UNDEFINED':
ipc.execute('chillout')
return True
else:
self.log.critical('alarm ' + self.type + 'not found!!')
return False
def run(self):
self.log.debug('Connecting to zookeeper')
self.log.debug('Push sleeping state')
FSM.zoowrap.sleep()
self.log.info('Sleep state confirmed')
self.log.warning(' =====> NOW READY TO SLEEP')
while True:
self.sleep()
if FSM.quitting:
self.log.warning('Quitting...')
return
elif FSM.state == 'sleeping':
if len(FSM.novawrap.get_my_vms_dict()['ACTIVE']) > 0:
self.log.info('no more sleeping, WAKE UP')
ipc.execute('wake')
return
else:
self.log.warning(self.name+' is not the state anymore')
return
def check_all(self):
''' return sample: [{
u'lifecycle_state': None,
u'links': [...],
u'updated_timestamp': u'2015-11-29T18:41:32.000Z',
u'state_updated_timestamp': u'2015-11-29T18:41:32.000Z',
u'metrics': [{
u'dimensions': {
u'hostname': u'compute01', u'service': u'monitoring'
}, u'id': None, u'name': u'cpu.idle_perc'
}],
u'state': u'ALARM', u'link': None,
u'alarm_definition': {
u'severity': u'HIGH',
u'id': u'e6fba7c7-9d3c-41d9-bca9-2c1a99787ead',
u'links': [...],
u'name': u'01-under-alarm'
}, u'created_timestamp': u'20g15-11-29T18:40:04.000Z',
u'id': u'7b4c26c1-4ad9-404f-bac6-539307af6b00'}, {...}]
'''
FSM.mona.get_alarms()
alarm_under = FSM.mona.get_my_alarm_under()
if alarm_under:
if alarm_under['state'] == 'ALARM':
self.log.info('Underload detected')
ipc.execute('underload')
return True
else:
self.log.debug('not underload state: %s' %
alarm_under['state'])
else:
self.log.critical('alarm under not found!!')
alarm_over = FSM.mona.get_my_alarm_over()
if alarm_over:
if alarm_over['state'] == 'ALARM':
self.log.info('Overload detected')
ipc.execute('overload')
return True
else:
self.log.debug('not overload state: %s' % alarm_under['state'])
else:
self.log.critical('alarm over not found!!')
return False
def run(self):
self.log.debug('Connecting to zookeeper')
self.log.debug('Push sleeping state')
FSM.zoowrap.sleep()
self.log.info('Sleep state confirmed')
self.log.warning(' =====> NOW READY TO SLEEP')
while True:
self.sleep()
if FSM.quitting:
self.log.warning('Quitting...')
return
elif FSM.state == 'sleeping':
if len(FSM.novawrap.get_my_vms_dict()['ACTIVE']) > 0:
self.log.info('no more sleeping, WAKE UP')
ipc.execute('wake')
return
else:
self.log.warning(self.name + ' is not the state anymore')
return
def check_all(self):
''' return sample: [{
u'lifecycle_state': None,
u'links': [...],
u'updated_timestamp': u'2015-11-29T18:41:32.000Z',
u'state_updated_timestamp': u'2015-11-29T18:41:32.000Z',
u'metrics': [{
u'dimensions': {
u'hostname': u'compute01', u'service': u'monitoring'
}, u'id': None, u'name': u'cpu.idle_perc'
}],
u'state': u'ALARM', u'link': None,
u'alarm_definition': {
u'severity': u'HIGH',
u'id': u'e6fba7c7-9d3c-41d9-bca9-2c1a99787ead',
u'links': [...],
u'name': u'01-under-alarm'
}, u'created_timestamp': u'20g15-11-29T18:40:04.000Z',
u'id': u'7b4c26c1-4ad9-404f-bac6-539307af6b00'}, {...}]
'''
FSM.mona.get_alarms()
alarm_under = FSM.mona.get_my_alarm_under()
if alarm_under:
if alarm_under['state'] == 'ALARM':
self.log.info('Underload detected')
ipc.execute('underload')
return True
else:
self.log.debug('not underload state: %s' % alarm_under['state'])
else:
self.log.critical('alarm under not found!!')
alarm_over = FSM.mona.get_my_alarm_over()
if alarm_over:
if alarm_over['state'] == 'ALARM':
self.log.info('Overload detected')
ipc.execute('overload')
return True
else:
self.log.debug('not overload state: %s' % alarm_under['state'])
else:
self.log.critical('alarm over not found!!')
return False
def run(self):
# select vm to migrate
vms = self.get_my_vms()
while len(vms['ACTIVE']) > 0:
if FSM.quitting:
self.log.warning('Quitting...')
break
elif FSM.state == 'underload':
# start distributed algorithm request
if self.check_state('underload', 'OK', 'chillout'):
break
responses = self.start_requester(vms['ACTIVE'])
# choose host
candidates = self.get_hosts_from_response(responses)
if len(candidates) == 0:
self.log.info('%s: No candidates found, avoid underload' %
self.action_id)
ipc.execute('avoid')
break
else:
# vm placement
migramap = self.get_migration_map(self.action_id,
candidates,
vms)
for migramatch in migramap:
migra_id = migramatch['vm']['id']
self.move_vm(vms, migra_id, 'ACTIVE', 'PENDING')
self.migrate_active(migramatch)
if not self.wait_for_migration(migramatch):
return
self.remove_vm(vms, migra_id, 'PENDING')
self.sleep()
else:
self.log.warning(self.name+' is not the state anymore')
break
else:
self.log.info('all active vms have migrated')
'''
not working
while len(vms_dict['SHUTOFF']) > 0:
vm = vms_dict['SHUTOFF'].pop()
self.log.warning('SHUTOFF-MIGRATING %s' % vm['name'])
self.migrate_shutoff(vm)
self.wait_for_migration_shutoff(vm)
'''
self.log.debug(vms)
sleepers = FSM.zoowrap.get_sleeping()
self.log.debug(sleepers)
if len(sleepers) < len(cc.conf['compute']['managers'])-1:
self.log.warning('%s: Start sleeping' % self.action_id)
ipc.execute('sleep')
return
else:
self.log.warning('%s is the last Man standing' % cc.conf['whoami'])
ipc.execute('avoid')
return
def run(self):
# select vm to migrate
vms = self.get_my_vms()
while len(vms['ACTIVE']) > 0:
if FSM.quitting:
self.log.warning('Quitting...')
break
elif FSM.state == 'underload':
# start distributed algorithm request
if self.check_state('underload', 'OK', 'chillout'):
break
responses = self.start_requester(vms['ACTIVE'])
# choose host
candidates = self.get_hosts_from_response(responses)
if len(candidates) == 0:
self.log.info('%s: No candidates found, avoid underload' %
self.action_id)
ipc.execute('avoid')
break
else:
# vm placement
migramap = self.get_migration_map(self.action_id,
candidates, vms)
for migramatch in migramap:
migra_id = migramatch['vm']['id']
self.move_vm(vms, migra_id, 'ACTIVE', 'PENDING')
self.migrate_active(migramatch)
if not self.wait_for_migration(migramatch):
return
self.remove_vm(vms, migra_id, 'PENDING')
self.sleep()
else:
self.log.warning(self.name + ' is not the state anymore')
break
else:
self.log.info('all active vms have migrated')
'''
not working
while len(vms_dict['SHUTOFF']) > 0:
vm = vms_dict['SHUTOFF'].pop()
self.log.warning('SHUTOFF-MIGRATING %s' % vm['name'])
self.migrate_shutoff(vm)
self.wait_for_migration_shutoff(vm)
'''
self.log.debug(vms)
sleepers = FSM.zoowrap.get_sleeping()
self.log.debug(sleepers)
if len(sleepers) < len(cc.conf['compute']['managers']) - 1:
self.log.warning('%s: Start sleeping' % self.action_id)
ipc.execute('sleep')
return
else:
self.log.warning('%s is the last Man standing' %
cc.conf['whoami'])
ipc.execute('avoid')
return