def __init__(self, *args):
super(CloudManager, self).__init__(*args)
self.dispatcher = Dispatcher('Cloud Manager:Dispatcher')
register_utility(self, ICloudManager, 'Cloud Manager')
self.peer = Event.LOCALORIGIN
self.formation = []
self._scheduled = None
self.unreachable = {}
self.subscription = None
if((as_node('/services/network/https_server')).is_enabled()):
self.secure_http = True
else:
self.secure_http = False
self.channel_monitor = ChannelMonitor()
self.channel_monitor.trigger = CallbackTrigger(self.channel_monitor)
class CloudManager(CompositeNode):
implements(ICloudManager)
security = SecurityInformation.from_default()
secured_by(security)
def __init__(self, *args):
super(CloudManager, self).__init__(*args)
self.dispatcher = Dispatcher('Cloud Manager:Dispatcher')
register_utility(self, ICloudManager, 'Cloud Manager')
self.peer = Event.LOCALORIGIN
self.formation = []
self._scheduled = None
self.unreachable = {}
self.subscription = None
if((as_node('/services/network/https_server')).is_enabled()):
self.secure_http = True
else:
self.secure_http = False
self.channel_monitor = ChannelMonitor()
self.channel_monitor.trigger = CallbackTrigger(self.channel_monitor)
def stop(self):
if self.subscription:
self.remove_listener(self.subscription)
if self.channel_monitor.is_running():
self.channel_monitor.stop_monitor()
self.subscription = None
super(CloudManager, self).stop()
def is_event_valid(self,cloudevent):
portal = self.nformation.get_portal()
topic=cloudevent.topics[0]
if(topic == 'EventResend' ):
if( (portal != None ) and utils.same_host(cloudevent.origin,portal) ):
return(True)
elif(topic == 'Alarm Manager' ):
#if (self.is_peer_in_formation(cloudevent.origin) == True):
return(True)
elif(topic == 'CloudFormation' ):
return(True)
return(False)
def handle_remote_event(self, data):
cloudevent = IPickles(cPickle.loads(data))()
self.message('Handling remote event from : %s topic=%s ' %(cloudevent.origin,cloudevent.topics))
cloudevent.set_data(data)
if(self.is_event_valid(cloudevent) == False ):
self.message('Dropping the remote event from : %s topic=%s '
%(cloudevent.origin,cloudevent.topics),msglog.types.WARN)
return
self.dispatcher.dispatch(cloudevent, cloudevent.topics)
if(not ('CloudFormation' in cloudevent.topics) ):
return
'''
Dont propogate an event if we are Portal
'''
if((cloudevent.portal != None )and (utils.same_host(self.peer,cloudevent.portal)) ):
self.message('Not Propagating remote event, since I am getting it as a portal:')
return
self.propogate(cloudevent)
def send_event_to_portal(self,event,topic,target):
cloudevent = CloudEvent(
self, self.peer, [target], self.nformation.get_portal(),topic, event)
protocol = "https" if self.secure_http else "http"
notifier = CloudNotifier(self.channel_monitor,target, protocol,'/cloud', self.debug)
notifier.cloudevent = cloudevent
if not cloudevent.has_data():
cloudevent.set_data(cPickle.dumps(IPickles(cloudevent)))
clouddata = cloudevent.get_data()
notifier.notify(clouddata, self.handle_send_failure_portal,self.handle_send_success_portal)
def handle_send_failure_portal(self, notifier):
cloudevent = notifier.cloudevent
target_peer = notifier.peer
self.message('Unable to send alarm events to portal=%s ' % (target_peer),msglog.types.WARN)
def handle_send_success_portal(self,notifier):
cloudevent = notifier.cloudevent
target_peer = notifier.peer
self.message('Succesfully sent alarm events to portal=%s ' % (target_peer))
def handle_local_event(self, event, topics = []):
cloudevent = CloudEvent(
self, self.peer, self.target_formation, self.nformation.get_portal(),topics, event)
self.propogate(cloudevent)
def is_peer_in_formation(self,peer,formation=None):
if not formation:
formation = self.get_formation()
peer_ip=utils.get_ip_addr(peer)
return any(utils.same_host(peer_ip, ip) for ip in formation)
def is_host_the_portal(self,host):
portal = self.nformation.get_portal()
if not portal:
return False
return utils.same_host(portal, host)
def is_host_in_formation(self, host):
if self.is_host_the_portal(host):
return True
if self.is_peer_in_formation(host):
return True
return False
def handle_formation_update(self, cloudevent):
'''
Don't take any action like updating Cloud formation or Portal etc,
if you have got this event as a portal. The alarms are shown in the event manager
by a different mechanism.
'''
if((cloudevent.portal != None ) and (utils.same_host(self.peer,cloudevent.portal))):
self.message('Received the event as a Portal, so not going to take any action %s' % str(cloudevent))
self.message('handle_formation_update doing nothing, no change.')
return
formation = cloudevent()
if (self.is_peer_in_formation(self.peer,formation) == False):
formation = [self.peer]
self.message('Setting Cloud Formation to self.peer; no longer in Cloud.',msglog.types.INFO)
self._setup_formation(formation,cloudevent.portal)
def _setup_formation(self, formation,portal):
scheduled, self._scheduled = self._scheduled, None
if scheduled is not None:
try: scheduled.cancel()
except: pass
else: self.message('Canceled pending dispatch of formation update.')
self.nformation.set_portal(portal)
self.nformation.set_formation(formation)
self.target_formation = self.nformation.compute_targets()
self.message('Resetting unreachables during Cloud setup.')
self.reset_unreachables()
(dispatch,delay)=self.nformation.compute_dispatch_info()
if (dispatch):
self._scheduled = scheduler.after(delay, self.dispatcher.dispatch, (FormationUpdated(self),))
self.message('Scheduled dispatch in %s seconds.' % delay)
else: self.message('Formation of one peer, no Updated event generated.')
# Save the PDO, if the formation or portal has changed
if((self._pdo.formation != formation) or (self._pdo.portal != portal) or (self._pdo.peer != self.peer)):
self.message('New formation/portal found , hence pickling. New Formation is :%s portal is %s' %(str(formation),portal))
self._pdo.formation=formation[:]
self._pdo.portal=portal
self._pdo.peer=self.peer
tstart = time.time()
self._pdo.save()
tend = time.time()
self.message('New formation pickled and saved in %s seconds.' % (tend - tstart))
else:
self.message('Formation/Portal has not changed. Not pickling it. ' )
def update_formation(self, new_formation,portal):
(no_of_excluded_peers,excludes)=self.nformation.compute_excludes(new_formation)
if no_of_excluded_peers:
self.message( 'Notifying removed participants: %s' % (excludes,))
excludedevent = CloudEvent(self, self.peer, excludes,self.nformation.get_portal(),['CloudFormation'], new_formation)
self.propogate(excludedevent)
else:
self.message( 'All current Cloud member in new Cloud Formation.')
self._setup_formation(new_formation,portal)
self.handle_local_event(new_formation, ['CloudFormation'])
def handle_propogation_failure(self, notifier):
cloudevent = notifier.cloudevent
target_peer = notifier.peer
# TODO: generate comm failure error to propogate as well.
# Progpogate event to Cloud Managers target_peer would have notified.
'''
The target_peer can be portal or a peer.
If it is a portal then we will not put it in unreachables and also
we do not propogate the event.
Log if we are not connecting to the portal
'''
portal=self.nformation.get_portal()
if((portal != None ) and (utils.same_host(target_peer,portal))):
msg='Portal %s is not reachable .' % portal
self.message(msg)
return
scheduled = self.unreachable.get(target_peer)
if scheduled is not None:
scheduled.cancel()
self.message('Host %s already listed unreachable, reset scheduled retry.' % target_peer)
self.unreachable[target_peer] = scheduler.after(5 * 60, self._remove_from_unreachable, (target_peer,))
self.message('Host %s added to list of unreachable peers.' % target_peer)
self.propogate(cloudevent, target_peer)
def _remove_from_unreachable(self, peer):
if self.unreachable.has_key(peer):
del(self.unreachable[peer])
self.message('Removed "%s" from unreachable to retry.' % peer)
else: self.message('Host "%s" not in unreachable, ignoring remove.' % peer)
def reset_unreachables(self):
message = 'Resetting unreachables:\n'
unreachables = self.unreachable.items()
self.unreachable.clear()
for peer, entry in unreachables:
entry.cancel()
message += '\t- removed "%s" from unreachables;\n' % peer
message += '\t%s peers cleared from unreachables.' % len(unreachables)
self.message(message)
return len(unreachables)
def add_listener(self, callback, topic):
return self.dispatcher.register_for_topic(callback, topic)
def remove_listener(self, guid):
return self.dispatcher.unregister(guid)
def propogate(self, cloudevent, from_peer = None):
'''
No Changes - Just make sure ...
'''
if not isinstance(cloudevent, CloudEvent):
raise TypeError('Argument must be instance of CloudEvent')
if from_peer is None: from_peer = self.peer
self.message('Propogating as %s:\n\t%s...' % (from_peer, str(cloudevent)))
target_formation = cloudevent.targets
notifiers = []
targets = self.nformation.get_targets(target_formation, from_peer)
for target in targets:
if not target.strip():
self.message('Not notifying "%s" because not valid, adding its targets.' % target)
targets.extend(self.nformation.get_targets(target_formation, target))
elif not self.unreachable.has_key(target):
protocol = "https" if self.secure_http else "http"
notifier = CloudNotifier(self.channel_monitor, target,
protocol, '/cloud', self.debug)
notifier.cloudevent = cloudevent
notifiers.append(notifier)
else:
self.message('Host "%s" unreachable, adding its targets.' % target)
targets.extend(self.nformation.get_targets(target_formation, target))
if not cloudevent.has_data():
cloudevent.set_data(cPickle.dumps(IPickles(cloudevent)))
clouddata = cloudevent.get_data()
notified = []
for notifier in notifiers:
notifier.notify(clouddata, self.handle_propogation_failure)
notified.append(notifier.peer)
if(len(notified) > 0 ):
self.message('Propogate notified: %s' % (notified,))
return notified
def message(self, message, mtype = msglog.types.DB):
if mtype != msglog.types.DB or self.debug:
message = 'CloudManager(%s) - %s' % (self.peer, message)
msglog.log('broadway', mtype, message)
security.protect('add_peer', 'Configure')
def add_peer(self,peer):
formation=self.nformation.get_formation()
formation.append(peer)
portal=self.nformation.get_portal()
self.update_formation(formation,portal)
return
def get_formation(self):
formation=self.nformation.get_formation()
return(formation)
def get_portal(self):
portal=self.nformation.get_portal()
return(portal)
security.protect('set_portal', 'Configure')
def set_portal(self,portal):
formation=self.nformation.get_formation()
self.update_formation(formation,portal)
security.protect('remove_peer', 'Configure')
def remove_peer(self,peer):
formation = self.nformation.get_formation()
formation.remove(peer)
portal=self.nformation.get_portal()
self.update_formation(formation,portal)
def start(self):
# Bad self IP Address
if(self.peer == '127.0.0.1' ):
msg='Cloud facility will not function properly because of local IP address being 127.0.0.1'
self.message(msg,msglog.types.WARN)
return
if not self.channel_monitor.is_running():
self.channel_monitor.start_monitor()
self._pdo=PersistentDataObject(self)
self.message('The Cloud Manager Persistent Object is in the file :%s' %str(self._pdo.filename()),msglog.types.INFO)
migration=False
if(os.path.exists(self._pdo.filename())):
# Already Migrated
self._pdo.formation=[self.peer]
self._pdo.portal=None
self._pdo.peer=self.peer
self._pdo.load()
else:
# We save a 'default' formation and expect the Cloud Configurator to
# update the _pdo.formation via update_information API.
# The _setup_formation gets called internally from update_information
self._pdo.portal=None
self._pdo.formation=[self.peer]
self._pdo.peer=self.peer
self._pdo.save()
self._pdo.load()
migration=True
#Bad formation/peer in the PDO
if( not self._pdo.peer in self._pdo.formation ):
#Bad formation/peer
self.message('The Cloud Manager PDO in the file :%s is corrupted. Defaulting to safe configuration' %str(self._pdo.filename()),msglog.types.WARN)
self._pdo.portal=None
self._pdo.formation=[self.peer]
self._pdo.peer=self.peer
self._pdo.save()
self._pdo.load()
self.message('Hosts are :%s portal=%s self=%s' %(str(self._pdo.formation),self._pdo.portal,self._pdo.peer),msglog.types.INFO)
self.nformation=NFormation(self._pdo.formation,self.peer)
self.nformation.set_portal(self._pdo.portal)
# IP Address Change Case
if(not utils.same_host(self.peer,self._pdo.peer)):
self.message('Self address change detected old=%s new=%s. Fixing the Cloud Formation accordingly' %(str(self._pdo.peer),self.peer),msglog.types.INFO)
formation = self.nformation.get_formation()
norm_form=self.nformation.normalize_formation(formation)
# IP Address Swap
self_index=norm_form.index(self._pdo.peer)
formation.pop(self_index)
formation.insert(0,self.peer)
self.nformation.set_formation(formation)
'''
In the Case of Migration, the update_formation() API is called
by the Cloud Configurator. In the already migrated case, we call the
update_formation() with the PDO formation and Portal
'''
self.target_formation = self.nformation.compute_targets()
if(migration == False):
self.update_formation(self.nformation.get_formation(), self.nformation.get_portal())
if self.subscription is None:
self.subscription = self.add_listener(
self.handle_formation_update, 'CloudFormation')
# Send Cloud Event to all the Hosts for re-sending the alarm events
# over to the Portal again - if we are nbmm
if(self.is_host_nbmm()):
scheduler.after(10, self.request_for_resending_alarm_events)
super(CloudManager, self).start()
def is_host_nbmm(self):
devices=as_node('/interfaces').children_names()
if('relay1' in devices ):
return(False)
else:
return(True)
def get_hosts_list(self):
hosts_list=[]
all_hosts=as_node('/services/Host Manager').children_nodes()
for h in all_hosts:
hosts_list.append(h.host)
return(hosts_list)
def request_for_resending_alarm_events(self):
hosts_list=self.get_hosts_list()
for host in hosts_list:
cloudevent = CloudEvent(self, self.peer, [host],None,['EventResend'],[host])
cloudevent.set_data(cPickle.dumps(IPickles(cloudevent)))
self.send_req_for_alarm_events(host,cloudevent)
def handle_send_failure(self, notifier):
cloudevent = notifier.cloudevent
target_peer = notifier.peer
self.message('Unable to notify %s to send alarm events again ' % (target_peer),msglog.types.WARN)
def handle_send_success(self,notifier):
cloudevent = notifier.cloudevent
target_peer = notifier.peer
self.message('succesfully notified %s to send alarm events again ' % (target_peer))
def send_req_for_alarm_events(self,target,cloudevent):
from_peer=self.peer
protocol = "https" if self.secure_http else "http"
notifier = CloudNotifier(self.channel_monitor,target, protocol,'/cloud', self.debug)
notifier.cloudevent = cloudevent
clouddata = cloudevent.get_data()
notifier.notify(clouddata, self.handle_send_failure,self.handle_send_success)
