def __init__(self, *args): super(CloudManager, self).__init__(*args) self.dispatcher = Dispatcher('Cloud Manager:Dispatcher') register_utility(self, ICloudManager, 'Cloud Manager') self.peer = Event.LOCALORIGIN self.formation = [] self._scheduled = None self.unreachable = {} self.subscription = None if((as_node('/services/network/https_server')).is_enabled()): self.secure_http = True else: self.secure_http = False self.channel_monitor = ChannelMonitor() self.channel_monitor.trigger = CallbackTrigger(self.channel_monitor)
class CloudManager(CompositeNode): implements(ICloudManager) security = SecurityInformation.from_default() secured_by(security) def __init__(self, *args): super(CloudManager, self).__init__(*args) self.dispatcher = Dispatcher('Cloud Manager:Dispatcher') register_utility(self, ICloudManager, 'Cloud Manager') self.peer = Event.LOCALORIGIN self.formation = [] self._scheduled = None self.unreachable = {} self.subscription = None if((as_node('/services/network/https_server')).is_enabled()): self.secure_http = True else: self.secure_http = False self.channel_monitor = ChannelMonitor() self.channel_monitor.trigger = CallbackTrigger(self.channel_monitor) def stop(self): if self.subscription: self.remove_listener(self.subscription) if self.channel_monitor.is_running(): self.channel_monitor.stop_monitor() self.subscription = None super(CloudManager, self).stop() def is_event_valid(self,cloudevent): portal = self.nformation.get_portal() topic=cloudevent.topics[0] if(topic == 'EventResend' ): if( (portal != None ) and utils.same_host(cloudevent.origin,portal) ): return(True) elif(topic == 'Alarm Manager' ): #if (self.is_peer_in_formation(cloudevent.origin) == True): return(True) elif(topic == 'CloudFormation' ): return(True) return(False) def handle_remote_event(self, data): cloudevent = IPickles(cPickle.loads(data))() self.message('Handling remote event from : %s topic=%s ' %(cloudevent.origin,cloudevent.topics)) cloudevent.set_data(data) if(self.is_event_valid(cloudevent) == False ): self.message('Dropping the remote event from : %s topic=%s ' %(cloudevent.origin,cloudevent.topics),msglog.types.WARN) return self.dispatcher.dispatch(cloudevent, cloudevent.topics) if(not ('CloudFormation' in cloudevent.topics) ): return ''' Dont propogate an event if we are Portal ''' if((cloudevent.portal != None )and (utils.same_host(self.peer,cloudevent.portal)) ): self.message('Not Propagating remote event, since I am getting it as a portal:') return self.propogate(cloudevent) def send_event_to_portal(self,event,topic,target): cloudevent = CloudEvent( self, self.peer, [target], self.nformation.get_portal(),topic, event) protocol = "https" if self.secure_http else "http" notifier = CloudNotifier(self.channel_monitor,target, protocol,'/cloud', self.debug) notifier.cloudevent = cloudevent if not cloudevent.has_data(): cloudevent.set_data(cPickle.dumps(IPickles(cloudevent))) clouddata = cloudevent.get_data() notifier.notify(clouddata, self.handle_send_failure_portal,self.handle_send_success_portal) def handle_send_failure_portal(self, notifier): cloudevent = notifier.cloudevent target_peer = notifier.peer self.message('Unable to send alarm events to portal=%s ' % (target_peer),msglog.types.WARN) def handle_send_success_portal(self,notifier): cloudevent = notifier.cloudevent target_peer = notifier.peer self.message('Succesfully sent alarm events to portal=%s ' % (target_peer)) def handle_local_event(self, event, topics = []): cloudevent = CloudEvent( self, self.peer, self.target_formation, self.nformation.get_portal(),topics, event) self.propogate(cloudevent) def is_peer_in_formation(self,peer,formation=None): if not formation: formation = self.get_formation() peer_ip=utils.get_ip_addr(peer) return any(utils.same_host(peer_ip, ip) for ip in formation) def is_host_the_portal(self,host): portal = self.nformation.get_portal() if not portal: return False return utils.same_host(portal, host) def is_host_in_formation(self, host): if self.is_host_the_portal(host): return True if self.is_peer_in_formation(host): return True return False def handle_formation_update(self, cloudevent): ''' Don't take any action like updating Cloud formation or Portal etc, if you have got this event as a portal. The alarms are shown in the event manager by a different mechanism. ''' if((cloudevent.portal != None ) and (utils.same_host(self.peer,cloudevent.portal))): self.message('Received the event as a Portal, so not going to take any action %s' % str(cloudevent)) self.message('handle_formation_update doing nothing, no change.') return formation = cloudevent() if (self.is_peer_in_formation(self.peer,formation) == False): formation = [self.peer] self.message('Setting Cloud Formation to self.peer; no longer in Cloud.',msglog.types.INFO) self._setup_formation(formation,cloudevent.portal) def _setup_formation(self, formation,portal): scheduled, self._scheduled = self._scheduled, None if scheduled is not None: try: scheduled.cancel() except: pass else: self.message('Canceled pending dispatch of formation update.') self.nformation.set_portal(portal) self.nformation.set_formation(formation) self.target_formation = self.nformation.compute_targets() self.message('Resetting unreachables during Cloud setup.') self.reset_unreachables() (dispatch,delay)=self.nformation.compute_dispatch_info() if (dispatch): self._scheduled = scheduler.after(delay, self.dispatcher.dispatch, (FormationUpdated(self),)) self.message('Scheduled dispatch in %s seconds.' % delay) else: self.message('Formation of one peer, no Updated event generated.') # Save the PDO, if the formation or portal has changed if((self._pdo.formation != formation) or (self._pdo.portal != portal) or (self._pdo.peer != self.peer)): self.message('New formation/portal found , hence pickling. New Formation is :%s portal is %s' %(str(formation),portal)) self._pdo.formation=formation[:] self._pdo.portal=portal self._pdo.peer=self.peer tstart = time.time() self._pdo.save() tend = time.time() self.message('New formation pickled and saved in %s seconds.' % (tend - tstart)) else: self.message('Formation/Portal has not changed. Not pickling it. ' ) def update_formation(self, new_formation,portal): (no_of_excluded_peers,excludes)=self.nformation.compute_excludes(new_formation) if no_of_excluded_peers: self.message( 'Notifying removed participants: %s' % (excludes,)) excludedevent = CloudEvent(self, self.peer, excludes,self.nformation.get_portal(),['CloudFormation'], new_formation) self.propogate(excludedevent) else: self.message( 'All current Cloud member in new Cloud Formation.') self._setup_formation(new_formation,portal) self.handle_local_event(new_formation, ['CloudFormation']) def handle_propogation_failure(self, notifier): cloudevent = notifier.cloudevent target_peer = notifier.peer # TODO: generate comm failure error to propogate as well. # Progpogate event to Cloud Managers target_peer would have notified. ''' The target_peer can be portal or a peer. If it is a portal then we will not put it in unreachables and also we do not propogate the event. Log if we are not connecting to the portal ''' portal=self.nformation.get_portal() if((portal != None ) and (utils.same_host(target_peer,portal))): msg='Portal %s is not reachable .' % portal self.message(msg) return scheduled = self.unreachable.get(target_peer) if scheduled is not None: scheduled.cancel() self.message('Host %s already listed unreachable, reset scheduled retry.' % target_peer) self.unreachable[target_peer] = scheduler.after(5 * 60, self._remove_from_unreachable, (target_peer,)) self.message('Host %s added to list of unreachable peers.' % target_peer) self.propogate(cloudevent, target_peer) def _remove_from_unreachable(self, peer): if self.unreachable.has_key(peer): del(self.unreachable[peer]) self.message('Removed "%s" from unreachable to retry.' % peer) else: self.message('Host "%s" not in unreachable, ignoring remove.' % peer) def reset_unreachables(self): message = 'Resetting unreachables:\n' unreachables = self.unreachable.items() self.unreachable.clear() for peer, entry in unreachables: entry.cancel() message += '\t- removed "%s" from unreachables;\n' % peer message += '\t%s peers cleared from unreachables.' % len(unreachables) self.message(message) return len(unreachables) def add_listener(self, callback, topic): return self.dispatcher.register_for_topic(callback, topic) def remove_listener(self, guid): return self.dispatcher.unregister(guid) def propogate(self, cloudevent, from_peer = None): ''' No Changes - Just make sure ... ''' if not isinstance(cloudevent, CloudEvent): raise TypeError('Argument must be instance of CloudEvent') if from_peer is None: from_peer = self.peer self.message('Propogating as %s:\n\t%s...' % (from_peer, str(cloudevent))) target_formation = cloudevent.targets notifiers = [] targets = self.nformation.get_targets(target_formation, from_peer) for target in targets: if not target.strip(): self.message('Not notifying "%s" because not valid, adding its targets.' % target) targets.extend(self.nformation.get_targets(target_formation, target)) elif not self.unreachable.has_key(target): protocol = "https" if self.secure_http else "http" notifier = CloudNotifier(self.channel_monitor, target, protocol, '/cloud', self.debug) notifier.cloudevent = cloudevent notifiers.append(notifier) else: self.message('Host "%s" unreachable, adding its targets.' % target) targets.extend(self.nformation.get_targets(target_formation, target)) if not cloudevent.has_data(): cloudevent.set_data(cPickle.dumps(IPickles(cloudevent))) clouddata = cloudevent.get_data() notified = [] for notifier in notifiers: notifier.notify(clouddata, self.handle_propogation_failure) notified.append(notifier.peer) if(len(notified) > 0 ): self.message('Propogate notified: %s' % (notified,)) return notified def message(self, message, mtype = msglog.types.DB): if mtype != msglog.types.DB or self.debug: message = 'CloudManager(%s) - %s' % (self.peer, message) msglog.log('broadway', mtype, message) security.protect('add_peer', 'Configure') def add_peer(self,peer): formation=self.nformation.get_formation() formation.append(peer) portal=self.nformation.get_portal() self.update_formation(formation,portal) return def get_formation(self): formation=self.nformation.get_formation() return(formation) def get_portal(self): portal=self.nformation.get_portal() return(portal) security.protect('set_portal', 'Configure') def set_portal(self,portal): formation=self.nformation.get_formation() self.update_formation(formation,portal) security.protect('remove_peer', 'Configure') def remove_peer(self,peer): formation = self.nformation.get_formation() formation.remove(peer) portal=self.nformation.get_portal() self.update_formation(formation,portal) def start(self): # Bad self IP Address if(self.peer == '127.0.0.1' ): msg='Cloud facility will not function properly because of local IP address being 127.0.0.1' self.message(msg,msglog.types.WARN) return if not self.channel_monitor.is_running(): self.channel_monitor.start_monitor() self._pdo=PersistentDataObject(self) self.message('The Cloud Manager Persistent Object is in the file :%s' %str(self._pdo.filename()),msglog.types.INFO) migration=False if(os.path.exists(self._pdo.filename())): # Already Migrated self._pdo.formation=[self.peer] self._pdo.portal=None self._pdo.peer=self.peer self._pdo.load() else: # We save a 'default' formation and expect the Cloud Configurator to # update the _pdo.formation via update_information API. # The _setup_formation gets called internally from update_information self._pdo.portal=None self._pdo.formation=[self.peer] self._pdo.peer=self.peer self._pdo.save() self._pdo.load() migration=True #Bad formation/peer in the PDO if( not self._pdo.peer in self._pdo.formation ): #Bad formation/peer self.message('The Cloud Manager PDO in the file :%s is corrupted. Defaulting to safe configuration' %str(self._pdo.filename()),msglog.types.WARN) self._pdo.portal=None self._pdo.formation=[self.peer] self._pdo.peer=self.peer self._pdo.save() self._pdo.load() self.message('Hosts are :%s portal=%s self=%s' %(str(self._pdo.formation),self._pdo.portal,self._pdo.peer),msglog.types.INFO) self.nformation=NFormation(self._pdo.formation,self.peer) self.nformation.set_portal(self._pdo.portal) # IP Address Change Case if(not utils.same_host(self.peer,self._pdo.peer)): self.message('Self address change detected old=%s new=%s. Fixing the Cloud Formation accordingly' %(str(self._pdo.peer),self.peer),msglog.types.INFO) formation = self.nformation.get_formation() norm_form=self.nformation.normalize_formation(formation) # IP Address Swap self_index=norm_form.index(self._pdo.peer) formation.pop(self_index) formation.insert(0,self.peer) self.nformation.set_formation(formation) ''' In the Case of Migration, the update_formation() API is called by the Cloud Configurator. In the already migrated case, we call the update_formation() with the PDO formation and Portal ''' self.target_formation = self.nformation.compute_targets() if(migration == False): self.update_formation(self.nformation.get_formation(), self.nformation.get_portal()) if self.subscription is None: self.subscription = self.add_listener( self.handle_formation_update, 'CloudFormation') # Send Cloud Event to all the Hosts for re-sending the alarm events # over to the Portal again - if we are nbmm if(self.is_host_nbmm()): scheduler.after(10, self.request_for_resending_alarm_events) super(CloudManager, self).start() def is_host_nbmm(self): devices=as_node('/interfaces').children_names() if('relay1' in devices ): return(False) else: return(True) def get_hosts_list(self): hosts_list=[] all_hosts=as_node('/services/Host Manager').children_nodes() for h in all_hosts: hosts_list.append(h.host) return(hosts_list) def request_for_resending_alarm_events(self): hosts_list=self.get_hosts_list() for host in hosts_list: cloudevent = CloudEvent(self, self.peer, [host],None,['EventResend'],[host]) cloudevent.set_data(cPickle.dumps(IPickles(cloudevent))) self.send_req_for_alarm_events(host,cloudevent) def handle_send_failure(self, notifier): cloudevent = notifier.cloudevent target_peer = notifier.peer self.message('Unable to notify %s to send alarm events again ' % (target_peer),msglog.types.WARN) def handle_send_success(self,notifier): cloudevent = notifier.cloudevent target_peer = notifier.peer self.message('succesfully notified %s to send alarm events again ' % (target_peer)) def send_req_for_alarm_events(self,target,cloudevent): from_peer=self.peer protocol = "https" if self.secure_http else "http" notifier = CloudNotifier(self.channel_monitor,target, protocol,'/cloud', self.debug) notifier.cloudevent = cloudevent clouddata = cloudevent.get_data() notifier.notify(clouddata, self.handle_send_failure,self.handle_send_success)