class Logger(ServiceNode): ## # @author Craig Warren # @param config # @return None def configure(self,config): ServiceNode.configure(self,config) ## # @author Craig Warren # starts the logger service # @return None def start(self): ServiceNode.start(self) # this will correctly add the msglog as a child # to the logger. if 'msglog' not in self.children_names(): columns = mpx.lib.msglog.get_columns() log = Log() log.configure({'name':'msglog', 'parent':self}) for c in columns: column = mpx.lib.factory('mpx.service.logger.column') config = c.configuration() config['parent'] = log column.configure(config) self._logs = PersistentDataObject(self) self._logs.names = [] self._logs.load() for name in self._logs.names: if ((not mpx.lib.log.log_exists(name)) and (name not in self.children_names())): log = mpx.lib.log.log(name) log.destroy() del(log) self._logs.names = [] for child in self.children_nodes(): if not isinstance(child, Alias): # Don't manage other managers' logs... self._logs.names.append(child.name) self._logs.save() ## # @author Craig Warren # stops the logger service # @return None def stop(self): return ServiceNode.stop(self) ## # @author Craig Warren # @param log_name # the name of the log to return # @return Log # returns the log if it can't find the log it # returns None def get_log(self,log_name): for child in self.children_nodes(): if child.name == log_name: return child return None
class LastAlarm(CompositeNode,EventConsumerMixin): def __init__(self): self._last_alarm = None self._started = 0 CompositeNode.__init__(self) EventConsumerMixin.__init__(self,self._alarm_triggered) def configure(self, config): CompositeNode.configure(self, config) def configuration(self): config = CompositeNode.configuration(self) return config def start(self): self._pdo = PersistentDataObject(self) self._pdo.last_dictionary = None self._pdo.load() self._started = 1 self.parent.event_subscribe(self,AlarmTriggerEvent) CompositeNode.start(self) def stop(self): selt._started = 0 self.parent.cancel(self,AlarmTriggerEvent) CompositeNode.stop(self) def _alarm_triggered(self, alarm): self._last_alarm = alarm self._pdo.last_dictionary = alarm.dictionary() self._pdo.save() def get(self, skipCache=0): return self._last_alarm def get_dictionary(self): return self._pdo.last_dictionary
class SimplePersistentValue(SimpleValue): def configure(self, config): SimpleValue.configure(self, config) self._pdo = PersistentDataObject(self) self._pdo.value = None self._pdo.conversion = None self._pdo.load() conversion = _get_name(self.conversion) if (self._pdo.value == None or self._pdo.conversion != conversion): self._pdo.value = self.value self._pdo.conversion = conversion self._pdo.save() else: self.value = self._pdo.value def configuration(self): self.value = self._pdo.value return SimpleValue.configuration(self) def set(self, value, asyncOK=1): SimpleValue.set(self, value, asyncOK) self._pdo.value = self.value self._pdo.save() def get(self, skipCache=0): return self._pdo.value
def _certificate_maintenance(self): previous = PersistentDataObject(self) previous.cert_config = None previous.key_file = None previous.server_cert = None previous.cert_fingerprint = None previous.load() c = certificate.CertificateConfiguration(self) config = {'C':self.country} config['ST'] = self.state config['L'] = self.city config['O'] = self.organization config['OU'] = self.organizational_unit config['CN'] = self.common_name config['emailAddress'] = self.email c.configure(config) cert_fingerprint = makecert.get_fingerprint(self.server_cert) if previous.cert_fingerprint == cert_fingerprint: msglog.log('broadway', msglog.types.INFO, 'Certificate Fingerprint Match!!!!' ) else: msglog.log('broadway', msglog.types.INFO, 'Certificate Fingerprint Mismatch!!!!' ) if c == previous.cert_config and \ previous.key_file == self.key_file and \ previous.cert_fingerprint == cert_fingerprint and \ not certificate.is_outdated(self.server_cert): msglog.log('broadway', msglog.types.INFO, 'Using existing certificate') return msglog.log('broadway', msglog.types.INFO, 'Generating new certificate') filename = os.path.join(properties.TEMP_DIR, 'cert_config.tmp') file = open(filename, 'w') c.formatted_output_to_file(file) try: failed = 1 makecert.create_from_file(filename, self.key_file, self.server_cert) failed = 0 msglog.log('broadway', msglog.types.INFO, 'Certificate generated') except: msglog.exception() msglog.log('broadway', msglog.types.WARN, 'Certificate generation failed') file.close() os.remove(filename) if not failed: previous.cert_config = c.configuration() previous.key_file = self.key_file previous.server_cert = self.server_cert previous.cert_fingerprint = makecert.get_fingerprint(self.server_cert) previous.save() return
class LastAlarm(CompositeNode, EventConsumerMixin): def __init__(self): self._last_alarm = None self._started = 0 CompositeNode.__init__(self) EventConsumerMixin.__init__(self, self._alarm_triggered) def configure(self, config): CompositeNode.configure(self, config) def configuration(self): config = CompositeNode.configuration(self) return config def start(self): self._pdo = PersistentDataObject(self) self._pdo.last_dictionary = None self._pdo.load() self._started = 1 self.parent.event_subscribe(self, AlarmTriggerEvent) CompositeNode.start(self) def stop(self): selt._started = 0 self.parent.cancel(self, AlarmTriggerEvent) CompositeNode.stop(self) def _alarm_triggered(self, alarm): self._last_alarm = alarm self._pdo.last_dictionary = alarm.dictionary() self._pdo.save() def get(self, skipCache=0): return self._last_alarm def get_dictionary(self): return self._pdo.last_dictionary
class SimplePersistentValue(SimpleValue): def configure(self, config): SimpleValue.configure(self, config) self._pdo = PersistentDataObject(self) self._pdo.value = None self._pdo.conversion = None self._pdo.load() conversion = _get_name(self.conversion) if (self._pdo.value == None or self._pdo.conversion != conversion): self._pdo.value = self.value self._pdo.conversion = conversion self._pdo.save() else: self.value = self._pdo.value def configuration(self): self.value = self._pdo.value return SimpleValue.configuration(self) def set(self,value,asyncOK=1): SimpleValue.set(self, value, asyncOK) self._pdo.value = self.value self._pdo.save() def get(self, skipCache=0): return self._pdo.value
def __save(self): result = PersistentDataObject.save(self) self.__snapshot(self.saved()) return result
class CloudManager(CompositeNode): implements(ICloudManager) security = SecurityInformation.from_default() secured_by(security) def __init__(self, *args): super(CloudManager, self).__init__(*args) self.dispatcher = Dispatcher('Cloud Manager:Dispatcher') register_utility(self, ICloudManager, 'Cloud Manager') self.peer = Event.LOCALORIGIN self.formation = [] self._scheduled = None self.unreachable = {} self.subscription = None if((as_node('/services/network/https_server')).is_enabled()): self.secure_http = True else: self.secure_http = False self.channel_monitor = ChannelMonitor() self.channel_monitor.trigger = CallbackTrigger(self.channel_monitor) def stop(self): if self.subscription: self.remove_listener(self.subscription) if self.channel_monitor.is_running(): self.channel_monitor.stop_monitor() self.subscription = None super(CloudManager, self).stop() def is_event_valid(self,cloudevent): portal = self.nformation.get_portal() topic=cloudevent.topics[0] if(topic == 'EventResend' ): if( (portal != None ) and utils.same_host(cloudevent.origin,portal) ): return(True) elif(topic == 'Alarm Manager' ): #if (self.is_peer_in_formation(cloudevent.origin) == True): return(True) elif(topic == 'CloudFormation' ): return(True) return(False) def handle_remote_event(self, data): cloudevent = IPickles(cPickle.loads(data))() self.message('Handling remote event from : %s topic=%s ' %(cloudevent.origin,cloudevent.topics)) cloudevent.set_data(data) if(self.is_event_valid(cloudevent) == False ): self.message('Dropping the remote event from : %s topic=%s ' %(cloudevent.origin,cloudevent.topics),msglog.types.WARN) return self.dispatcher.dispatch(cloudevent, cloudevent.topics) if(not ('CloudFormation' in cloudevent.topics) ): return ''' Dont propogate an event if we are Portal ''' if((cloudevent.portal != None )and (utils.same_host(self.peer,cloudevent.portal)) ): self.message('Not Propagating remote event, since I am getting it as a portal:') return self.propogate(cloudevent) def send_event_to_portal(self,event,topic,target): cloudevent = CloudEvent( self, self.peer, [target], self.nformation.get_portal(),topic, event) protocol = "https" if self.secure_http else "http" notifier = CloudNotifier(self.channel_monitor,target, protocol,'/cloud', self.debug) notifier.cloudevent = cloudevent if not cloudevent.has_data(): cloudevent.set_data(cPickle.dumps(IPickles(cloudevent))) clouddata = cloudevent.get_data() notifier.notify(clouddata, self.handle_send_failure_portal,self.handle_send_success_portal) def handle_send_failure_portal(self, notifier): cloudevent = notifier.cloudevent target_peer = notifier.peer self.message('Unable to send alarm events to portal=%s ' % (target_peer),msglog.types.WARN) def handle_send_success_portal(self,notifier): cloudevent = notifier.cloudevent target_peer = notifier.peer self.message('Succesfully sent alarm events to portal=%s ' % (target_peer)) def handle_local_event(self, event, topics = []): cloudevent = CloudEvent( self, self.peer, self.target_formation, self.nformation.get_portal(),topics, event) self.propogate(cloudevent) def is_peer_in_formation(self,peer,formation=None): if not formation: formation = self.get_formation() peer_ip=utils.get_ip_addr(peer) return any(utils.same_host(peer_ip, ip) for ip in formation) def is_host_the_portal(self,host): portal = self.nformation.get_portal() if not portal: return False return utils.same_host(portal, host) def is_host_in_formation(self, host): if self.is_host_the_portal(host): return True if self.is_peer_in_formation(host): return True return False def handle_formation_update(self, cloudevent): ''' Don't take any action like updating Cloud formation or Portal etc, if you have got this event as a portal. The alarms are shown in the event manager by a different mechanism. ''' if((cloudevent.portal != None ) and (utils.same_host(self.peer,cloudevent.portal))): self.message('Received the event as a Portal, so not going to take any action %s' % str(cloudevent)) self.message('handle_formation_update doing nothing, no change.') return formation = cloudevent() if (self.is_peer_in_formation(self.peer,formation) == False): formation = [self.peer] self.message('Setting Cloud Formation to self.peer; no longer in Cloud.',msglog.types.INFO) self._setup_formation(formation,cloudevent.portal) def _setup_formation(self, formation,portal): scheduled, self._scheduled = self._scheduled, None if scheduled is not None: try: scheduled.cancel() except: pass else: self.message('Canceled pending dispatch of formation update.') self.nformation.set_portal(portal) self.nformation.set_formation(formation) self.target_formation = self.nformation.compute_targets() self.message('Resetting unreachables during Cloud setup.') self.reset_unreachables() (dispatch,delay)=self.nformation.compute_dispatch_info() if (dispatch): self._scheduled = scheduler.after(delay, self.dispatcher.dispatch, (FormationUpdated(self),)) self.message('Scheduled dispatch in %s seconds.' % delay) else: self.message('Formation of one peer, no Updated event generated.') # Save the PDO, if the formation or portal has changed if((self._pdo.formation != formation) or (self._pdo.portal != portal) or (self._pdo.peer != self.peer)): self.message('New formation/portal found , hence pickling. New Formation is :%s portal is %s' %(str(formation),portal)) self._pdo.formation=formation[:] self._pdo.portal=portal self._pdo.peer=self.peer tstart = time.time() self._pdo.save() tend = time.time() self.message('New formation pickled and saved in %s seconds.' % (tend - tstart)) else: self.message('Formation/Portal has not changed. Not pickling it. ' ) def update_formation(self, new_formation,portal): (no_of_excluded_peers,excludes)=self.nformation.compute_excludes(new_formation) if no_of_excluded_peers: self.message( 'Notifying removed participants: %s' % (excludes,)) excludedevent = CloudEvent(self, self.peer, excludes,self.nformation.get_portal(),['CloudFormation'], new_formation) self.propogate(excludedevent) else: self.message( 'All current Cloud member in new Cloud Formation.') self._setup_formation(new_formation,portal) self.handle_local_event(new_formation, ['CloudFormation']) def handle_propogation_failure(self, notifier): cloudevent = notifier.cloudevent target_peer = notifier.peer # TODO: generate comm failure error to propogate as well. # Progpogate event to Cloud Managers target_peer would have notified. ''' The target_peer can be portal or a peer. If it is a portal then we will not put it in unreachables and also we do not propogate the event. Log if we are not connecting to the portal ''' portal=self.nformation.get_portal() if((portal != None ) and (utils.same_host(target_peer,portal))): msg='Portal %s is not reachable .' % portal self.message(msg) return scheduled = self.unreachable.get(target_peer) if scheduled is not None: scheduled.cancel() self.message('Host %s already listed unreachable, reset scheduled retry.' % target_peer) self.unreachable[target_peer] = scheduler.after(5 * 60, self._remove_from_unreachable, (target_peer,)) self.message('Host %s added to list of unreachable peers.' % target_peer) self.propogate(cloudevent, target_peer) def _remove_from_unreachable(self, peer): if self.unreachable.has_key(peer): del(self.unreachable[peer]) self.message('Removed "%s" from unreachable to retry.' % peer) else: self.message('Host "%s" not in unreachable, ignoring remove.' % peer) def reset_unreachables(self): message = 'Resetting unreachables:\n' unreachables = self.unreachable.items() self.unreachable.clear() for peer, entry in unreachables: entry.cancel() message += '\t- removed "%s" from unreachables;\n' % peer message += '\t%s peers cleared from unreachables.' % len(unreachables) self.message(message) return len(unreachables) def add_listener(self, callback, topic): return self.dispatcher.register_for_topic(callback, topic) def remove_listener(self, guid): return self.dispatcher.unregister(guid) def propogate(self, cloudevent, from_peer = None): ''' No Changes - Just make sure ... ''' if not isinstance(cloudevent, CloudEvent): raise TypeError('Argument must be instance of CloudEvent') if from_peer is None: from_peer = self.peer self.message('Propogating as %s:\n\t%s...' % (from_peer, str(cloudevent))) target_formation = cloudevent.targets notifiers = [] targets = self.nformation.get_targets(target_formation, from_peer) for target in targets: if not target.strip(): self.message('Not notifying "%s" because not valid, adding its targets.' % target) targets.extend(self.nformation.get_targets(target_formation, target)) elif not self.unreachable.has_key(target): protocol = "https" if self.secure_http else "http" notifier = CloudNotifier(self.channel_monitor, target, protocol, '/cloud', self.debug) notifier.cloudevent = cloudevent notifiers.append(notifier) else: self.message('Host "%s" unreachable, adding its targets.' % target) targets.extend(self.nformation.get_targets(target_formation, target)) if not cloudevent.has_data(): cloudevent.set_data(cPickle.dumps(IPickles(cloudevent))) clouddata = cloudevent.get_data() notified = [] for notifier in notifiers: notifier.notify(clouddata, self.handle_propogation_failure) notified.append(notifier.peer) if(len(notified) > 0 ): self.message('Propogate notified: %s' % (notified,)) return notified def message(self, message, mtype = msglog.types.DB): if mtype != msglog.types.DB or self.debug: message = 'CloudManager(%s) - %s' % (self.peer, message) msglog.log('broadway', mtype, message) security.protect('add_peer', 'Configure') def add_peer(self,peer): formation=self.nformation.get_formation() formation.append(peer) portal=self.nformation.get_portal() self.update_formation(formation,portal) return def get_formation(self): formation=self.nformation.get_formation() return(formation) def get_portal(self): portal=self.nformation.get_portal() return(portal) security.protect('set_portal', 'Configure') def set_portal(self,portal): formation=self.nformation.get_formation() self.update_formation(formation,portal) security.protect('remove_peer', 'Configure') def remove_peer(self,peer): formation = self.nformation.get_formation() formation.remove(peer) portal=self.nformation.get_portal() self.update_formation(formation,portal) def start(self): # Bad self IP Address if(self.peer == '127.0.0.1' ): msg='Cloud facility will not function properly because of local IP address being 127.0.0.1' self.message(msg,msglog.types.WARN) return if not self.channel_monitor.is_running(): self.channel_monitor.start_monitor() self._pdo=PersistentDataObject(self) self.message('The Cloud Manager Persistent Object is in the file :%s' %str(self._pdo.filename()),msglog.types.INFO) migration=False if(os.path.exists(self._pdo.filename())): # Already Migrated self._pdo.formation=[self.peer] self._pdo.portal=None self._pdo.peer=self.peer self._pdo.load() else: # We save a 'default' formation and expect the Cloud Configurator to # update the _pdo.formation via update_information API. # The _setup_formation gets called internally from update_information self._pdo.portal=None self._pdo.formation=[self.peer] self._pdo.peer=self.peer self._pdo.save() self._pdo.load() migration=True #Bad formation/peer in the PDO if( not self._pdo.peer in self._pdo.formation ): #Bad formation/peer self.message('The Cloud Manager PDO in the file :%s is corrupted. Defaulting to safe configuration' %str(self._pdo.filename()),msglog.types.WARN) self._pdo.portal=None self._pdo.formation=[self.peer] self._pdo.peer=self.peer self._pdo.save() self._pdo.load() self.message('Hosts are :%s portal=%s self=%s' %(str(self._pdo.formation),self._pdo.portal,self._pdo.peer),msglog.types.INFO) self.nformation=NFormation(self._pdo.formation,self.peer) self.nformation.set_portal(self._pdo.portal) # IP Address Change Case if(not utils.same_host(self.peer,self._pdo.peer)): self.message('Self address change detected old=%s new=%s. Fixing the Cloud Formation accordingly' %(str(self._pdo.peer),self.peer),msglog.types.INFO) formation = self.nformation.get_formation() norm_form=self.nformation.normalize_formation(formation) # IP Address Swap self_index=norm_form.index(self._pdo.peer) formation.pop(self_index) formation.insert(0,self.peer) self.nformation.set_formation(formation) ''' In the Case of Migration, the update_formation() API is called by the Cloud Configurator. In the already migrated case, we call the update_formation() with the PDO formation and Portal ''' self.target_formation = self.nformation.compute_targets() if(migration == False): self.update_formation(self.nformation.get_formation(), self.nformation.get_portal()) if self.subscription is None: self.subscription = self.add_listener( self.handle_formation_update, 'CloudFormation') # Send Cloud Event to all the Hosts for re-sending the alarm events # over to the Portal again - if we are nbmm if(self.is_host_nbmm()): scheduler.after(10, self.request_for_resending_alarm_events) super(CloudManager, self).start() def is_host_nbmm(self): devices=as_node('/interfaces').children_names() if('relay1' in devices ): return(False) else: return(True) def get_hosts_list(self): hosts_list=[] all_hosts=as_node('/services/Host Manager').children_nodes() for h in all_hosts: hosts_list.append(h.host) return(hosts_list) def request_for_resending_alarm_events(self): hosts_list=self.get_hosts_list() for host in hosts_list: cloudevent = CloudEvent(self, self.peer, [host],None,['EventResend'],[host]) cloudevent.set_data(cPickle.dumps(IPickles(cloudevent))) self.send_req_for_alarm_events(host,cloudevent) def handle_send_failure(self, notifier): cloudevent = notifier.cloudevent target_peer = notifier.peer self.message('Unable to notify %s to send alarm events again ' % (target_peer),msglog.types.WARN) def handle_send_success(self,notifier): cloudevent = notifier.cloudevent target_peer = notifier.peer self.message('succesfully notified %s to send alarm events again ' % (target_peer)) def send_req_for_alarm_events(self,target,cloudevent): from_peer=self.peer protocol = "https" if self.secure_http else "http" notifier = CloudNotifier(self.channel_monitor,target, protocol,'/cloud', self.debug) notifier.cloudevent = cloudevent clouddata = cloudevent.get_data() notifier.notify(clouddata, self.handle_send_failure,self.handle_send_success)
class EnergywiseManager(CompositeNode): def __init__(self): CompositeNode.__init__(self) self._pdo_lock = Lock() self._pdo = None self.__running = False self.debug = 0 return def configure(self,config): if self.debug: msglog.log('EnergywiseManager:', msglog.types.INFO, 'Inside configure' ) CompositeNode.configure(self, config) set_attribute(self, 'debug', 0, config, int) return def configuration(self): config = CompositeNode.configuration(self) get_attribute(self, 'debug', config, str) return config # def configure_trend_in_switches(self, start_node, frequency): # for child in start_node.children_nodes(): # if child.children_nodes(): # self.configure_trend_in_switches(child, frequency) # else: # reached upto leaf, each energywise switch has trends as child # child.new_trend(frequency) #return def delete_trend_configuration(self, trend_domain): self._pdo_lock.acquire() try: if self._pdo.trends.has_key(trend_domain): # stop logging as well del self._pdo.trends[trend_domain] self._pdo.save() finally: self._pdo_lock.release() return def delete_trends(self, trendList): if self.debug: msglog.log('EnergywiseManager:', msglog.types.INFO, 'Inside delete_trends' ) for domain in trendList.split(':'): if domain: domain_node = as_node(domain) domain_node.delete_trend() self.delete_trend_configuration(domain) return def start(self): if self.__running: return if self.debug: msglog.log('EnergywiseManager :', msglog.types.INFO, 'Inside start' ) CompositeNode.start(self) # start_node = as_node('/services/EnergywiseManager/') # self.configure_trend_in_switches(start_node, 60) self.__running = True self._pdo_lock.acquire() self._pdo = PersistentDataObject(self, dmtype=GC_NEVER) self._pdo.trends = {} self._pdo.load() self._pdo_lock.release() if self.has_child('trends'): self.trends = self.get_child('trends') else: self.trends = CompositeNode() self.trends.configure({'parent':self, 'name':'trends'}) self.trends.start() # start trending for saved domains for domain,freq in self._pdo.trends.items(): try: start_node = as_node(domain) # self.configure_trend_in_switches( start_node,freq ) start_node.new_trend(freq) except: self.delete_trend_configuration(domain) return def get_trends(self): return self._pdo.trends.items() def add_trend_configuration(self, trend_period, trend_domain): self._pdo_lock.acquire() self._pdo.trends[trend_domain] = trend_period self._pdo.save() self._pdo_lock.release() return def save_trends(self, trend_list): # Traverse through _pdo.items and check if new domain is either subset # of any configured or superset. # If subset return with msg already covered and dont save this # If superset then configure new ones and delete subset from # _pdo.items '''Adding and saving trends''' for point in reversed(trend_list): point_period = point['frequency'] point_domain = point['domain'] for saved_domain,saved_period in tuple(self._pdo.trends.items()): if saved_domain == point_domain: if saved_period != point_period: self.delete_trend_configuration(saved_domain) break if not self._pdo.trends.has_key(point_domain): # add this trend try: domain_node = as_node(point_domain) if isinstance(domain_node,EnergywiseSwitch) or isinstance(domain_node,EnergywiseDomain): self.add_trend_configuration(point_period, point_domain) domain_node.new_trend(point_period) except Exception: msglog.exception() msglog.log( "Energywise",msglog.types.ERR, "Failed to create trend for %r every %r seconds" %(point_domain,point_period) ) return def stop(self): CompositeNode.stop(self) self.__running = False return
def save(self): PersistentDataObject.save(self) msglog.log('interfaces....BBMD', msglog.types.INFO, 'Saved BBMD table to Persistent Storage')
import inspect from mpx.lib.persistence import storage from mpx.lib.persistence import datatypes from mpx.lib.persistent import PersistentDataObject as PDO data = dict([(str(key), string.ascii_letters) for key in range(10000)]) pdo = PDO('many-key-test') pdict = datatypes.PersistentDictionary('many-key-test') pdo.load() pdodict = pdo.__dict__ items = data.items() pdostart = time.time() for key, value in items: pdodict[key] = value pdo.save() pdostop = time.time() pdictstart = time.time() for key, value in items: pdict[key] = value pdictstop = time.time() print 'Took %0.4f seconds to set/save %d PDO attributes' % (pdostop - pdostart, len(items)) print 'Took %0.4f seconds to set/save %d PDict items' % ( pdictstop - pdictstart, len(items)) import time
def save(self): PersistentDataObject.save(self) msglog.log("interfaces....BBMD", msglog.types.INFO, "Saved BBMD table to Persistent Storage")
class SynchronizedExporter(Exporter,EventConsumerMixin): def __init__(self): self.running = 0 self._lock = Lock() Exporter.__init__(self) EventConsumerMixin.__init__(self,self.handle_log,self.handle_error) def debug_information(self,message): if self.debug: debug = '%s Exporter => %s' % (self.name,message) msglog.log('broadway',msglog.types.DB,debug) def handle_log(self,event): self._event_count += 1 self.debug_information('Log entry event caught.') if self._event_count >= self.log_multiple: self.debug_information('Going to start export thread.') if self._lock.acquire(0): try: thread = Thread(name=self.name, target=self.go, args=(event.values[0],)) thread.start() self._event_count = 0 finally: self._lock.release() else: msglog.log('broadway',msglog.types.WARN, ('Last export still active, ' + 'skipping current request.')) def handle_error(self,exc): msglog.exception(exc) def configure(self, config): set_attribute(self,'log_multiple',1,config,int) set_attribute(self,'timeout',60,config,int) set_attribute(self,'connection_node','/services/network',config) set_attribute(self,'connection_attempts',3,config,int) Exporter.configure(self, config) def configuration(self): config = Exporter.configuration(self) get_attribute(self,'log_multiple',config,str) get_attribute(self,'connection_node',config) get_attribute(self,'connection_attempts',config) get_attribute(self,'timeout',config,int) return config def start(self): Exporter.start(self) if not self.running: self.running = 1 self.connection = as_node(self.connection_node) self._event_count = self.log_multiple - 1 self._time_keeper = PersistentDataObject(self) self._time_keeper.start_time = 0 self._time_keeper.load() self._period = self.parent.parent.period self.parent.parent.event_subscribe(self, LogAddEntryEvent) else: raise EAlreadyRunning def stop(self): self.running = 0 def scheduled_time(self): return self._end_time def go(self, end_time): self.debug_information('Exporting.') self._lock.acquire() try: self._end_time = end_time self._export(end_time) self._end_time = None self.debug_information('Done Exporting.') except: msglog.exception() self._lock.release() def _export(self,end_time): attempts = 0 connected = 0 while attempts < self.connection_attempts: self.debug_information('Acquiring connection...') try: connected = self.connection.acquire() except: msglog.exception() if connected: self.debug_information('Connection acquired.') break self.debug_information('Failed to acquire.') attempts += 1 else: self.debug_information('Connection failed, aborting.') raise EConnectionError('Failed to connect %s times' % attempts) try: last_break = 0 end = end_time start_time = self._time_keeper.start_time while start_time <= end_time: self.debug_information('Getting data from %s to %s.' % (start_time,end)) data = self.log.get_range('timestamp',start_time,end) if not data: self.debug_information('No Data to export.') raise ENoData('timestamp',start_time,end) try: self.debug_information('Calling format.') output = self.formatter.format(data) self.debug_information('Calling transport.') self.transporter.transport(output) self.debug_information('Done transporting.') start_time = end + self._period except EBreakupTransfer, e: entry = e.break_at self.debug_information('Breaking up transfer.') if entry['timestamp'] == last_break: # prevents loop where transporter is just failing. raise EIOError('EBreakupTransfer not progressing.') last_break = entry['timestamp'] end = last_break - self._period msglog.log('broadway',msglog.types.WARN, 'Breaking up data transfer at %s.' % end) else: end = end_time self._time_keeper.start_time = start_time self._time_keeper.save() finally: if connected: self.connection.release()
def set_meta_value(self,name,value): self.meta[name] = value PersistentDataObject.save(self)
class SynchronizedExporter(Exporter, EventConsumerMixin): def __init__(self): self.running = 0 self._lock = Lock() Exporter.__init__(self) EventConsumerMixin.__init__(self, self.handle_log, self.handle_error) def debug_information(self, message): if self.debug: debug = '%s Exporter => %s' % (self.name, message) msglog.log('broadway', msglog.types.DB, debug) def handle_log(self, event): self._event_count += 1 self.debug_information('Log entry event caught.') if self._event_count >= self.log_multiple: self.debug_information('Going to start export thread.') if self._lock.acquire(0): try: thread = Thread(name=self.name, target=self.go, args=(event.values[0], )) thread.start() self._event_count = 0 finally: self._lock.release() else: msglog.log('broadway', msglog.types.WARN, ('Last export still active, ' + 'skipping current request.')) def handle_error(self, exc): msglog.exception(exc) def configure(self, config): set_attribute(self, 'log_multiple', 1, config, int) set_attribute(self, 'timeout', 60, config, int) set_attribute(self, 'connection_node', '/services/network', config) set_attribute(self, 'connection_attempts', 3, config, int) Exporter.configure(self, config) def configuration(self): config = Exporter.configuration(self) get_attribute(self, 'log_multiple', config, str) get_attribute(self, 'connection_node', config) get_attribute(self, 'connection_attempts', config) get_attribute(self, 'timeout', config, int) return config def start(self): Exporter.start(self) if not self.running: self.running = 1 self.connection = as_node(self.connection_node) self._event_count = self.log_multiple - 1 self._time_keeper = PersistentDataObject(self) self._time_keeper.start_time = 0 self._time_keeper.load() self._period = self.parent.parent.period self.parent.parent.event_subscribe(self, LogAddEntryEvent) else: raise EAlreadyRunning def stop(self): self.running = 0 def scheduled_time(self): return self._end_time def go(self, end_time): self.debug_information('Exporting.') self._lock.acquire() try: self._end_time = end_time self._export(end_time) self._end_time = None self.debug_information('Done Exporting.') except: msglog.exception() self._lock.release() def _export(self, end_time): attempts = 0 connected = 0 while attempts < self.connection_attempts: self.debug_information('Acquiring connection...') try: connected = self.connection.acquire() except: msglog.exception() if connected: self.debug_information('Connection acquired.') break self.debug_information('Failed to acquire.') attempts += 1 else: self.debug_information('Connection failed, aborting.') raise EConnectionError('Failed to connect %s times' % attempts) try: last_break = 0 end = end_time start_time = self._time_keeper.start_time while start_time <= end_time: self.debug_information('Getting data from %s to %s.' % (start_time, end)) data = self.log.get_range('timestamp', start_time, end) if not data: self.debug_information('No Data to export.') raise ENoData('timestamp', start_time, end) try: self.debug_information('Calling format.') output = self.formatter.format(data) self.debug_information('Calling transport.') self.transporter.transport(output) self.debug_information('Done transporting.') start_time = end + self._period except EBreakupTransfer, e: entry = e.break_at self.debug_information('Breaking up transfer.') if entry['timestamp'] == last_break: # prevents loop where transporter is just failing. raise EIOError('EBreakupTransfer not progressing.') last_break = entry['timestamp'] end = last_break - self._period msglog.log('broadway', msglog.types.WARN, 'Breaking up data transfer at %s.' % end) else: end = end_time self._time_keeper.start_time = start_time self._time_keeper.save() finally: if connected: self.connection.release()
class FTPTransporter(Transporter): _last = None ## # Configure object. # # @key url The url that data is to be sent # to. For example: ftp.hostname.com/tmp/. # @key username The username for the FTP session. # @key password The password for the FTP session. # @key file_prefix The prefix for files that are # created when data is uploaded. # @key file_suffix The suffix to be appended to all # files created when data is uploaded. # @default .dat # @key name_scheme Naming scheme to be used for created # each file. # @value timestamp Insert timestamp between file_prefix # and file_suffix for each upload. # @value count Insert incrmemental count of uploads between # prefix and suffix. # @value none Do not use a naming scheme. Each upload # will overwrite the previous upload. # @default timestamp. # def configure(self, config): set_attribute(self, 'host', REQUIRED, config) set_attribute(self, 'port', 21, config, int) set_attribute(self, 'directory', '', config) set_attribute(self, 'username', REQUIRED, config) set_attribute(self, 'password', REQUIRED, config) #CSCtn64870 if (config.has_key('timeout') and config['timeout'] == ''): config['timeout'] = 'None' set_attribute(self, 'timeout', None, config, float) set_attribute(self, 'file_prefix', 'cisco', config) set_attribute(self, 'file_suffix', '.dat', config) set_attribute(self, 'name_scheme', 'timestamp', config) set_attribute(self, 'timestamp_format', '%s', config) set_attribute(self, 'passive_mode', 1, config, as_boolean) set_attribute(self, 'file_append', 0, config, as_boolean) Transporter.configure(self, config) if self._last is None: self._last = PersistentDataObject(self) self._last.filename = None self._last.count = 1 self._last.load() def configuration(self): config = Transporter.configuration(self) get_attribute(self, 'host', config) get_attribute(self, 'port', config, str) get_attribute(self, 'directory', config) get_attribute(self, 'username', config) get_attribute(self, 'password', config) get_attribute(self, 'timeout', config, str) get_attribute(self, 'file_prefix', config) get_attribute(self, 'file_suffix', config) get_attribute(self, 'name_scheme', config) get_attribute(self, 'timestamp_format', config) get_attribute(self, 'passive_mode', config, as_onoff) get_attribute(self, 'file_append', config, str) return config def transport(self, data): filename = self._generate_filename() if type(data) == type(''): data = StringIO(data) ftp = ftplib.FTP() ftp.connect(self.host, self.port, self.timeout) finished = 0 try: ftp.login(self.username, self.password) ftp.set_pasv(self.passive_mode != 0) if self.file_append and not self.name_scheme: ftp.storlines('APPE ' + self._full_file_name(filename), data) else: ftp.storlines('STOR ' + self._full_file_name(filename), data) self._last.save() finished = 1 data.close() finally: if not finished: # quit hangs is exception. ftp.close() else: try: ftp.quit() except: ftp.close() def _generate_filename(self): append = '' filename = self.file_prefix if self.name_scheme == 'incremental': append = '%s' % self._last.count elif self.name_scheme == 'timestamp': filetime = self.parent.time_function(self.parent.scheduled_time()) filename += time.strftime(self.timestamp_format, filetime) append = '_%s' % (self._last.count + 1) if filename != self._last.filename: self._last.count = 0 append = '' self._last.count += 1 self._last.filename = filename return filename + append + self.file_suffix def _full_file_name(self, filename): if self.directory: if self.directory[-1:] == '/': filename = self.directory + filename else: filename = self.directory + '/' + filename return filename
def set_meta_value(self, name, value): self.meta[name] = value PersistentDataObject.save(self)
import inspect from mpx.lib.persistence import storage from mpx.lib.persistence import datatypes from mpx.lib.persistent import PersistentDataObject as PDO data = dict([(str(key), string.ascii_letters) for key in range(10000)]) pdo = PDO('many-key-test') pdict = datatypes.PersistentDictionary('many-key-test') pdo.load() pdodict = pdo.__dict__ items = data.items() pdostart = time.time() for key,value in items: pdodict[key] = value pdo.save() pdostop = time.time() pdictstart = time.time() for key,value in items: pdict[key] = value pdictstop = time.time() print 'Took %0.4f seconds to set/save %d PDO attributes' % (pdostop - pdostart, len(items)) print 'Took %0.4f seconds to set/save %d PDict items' % (pdictstop - pdictstart, len(items))
class Logger(ServiceNode): ## # @author Craig Warren # @param config # @return None def configure(self, config): ServiceNode.configure(self, config) ## # @author Craig Warren # starts the logger service # @return None def start(self): ServiceNode.start(self) # this will correctly add the msglog as a child # to the logger. if 'msglog' not in self.children_names(): columns = mpx.lib.msglog.get_columns() log = Log() log.configure({'name': 'msglog', 'parent': self}) for c in columns: column = mpx.lib.factory('mpx.service.logger.column') config = c.configuration() config['parent'] = log column.configure(config) self._logs = PersistentDataObject(self) self._logs.names = [] self._logs.load() for name in self._logs.names: if ((not mpx.lib.log.log_exists(name)) and (name not in self.children_names())): log = mpx.lib.log.log(name) log.destroy() del (log) self._logs.names = [] for child in self.children_nodes(): if not isinstance(child, Alias): # Don't manage other managers' logs... self._logs.names.append(child.name) self._logs.save() ## # @author Craig Warren # stops the logger service # @return None def stop(self): return ServiceNode.stop(self) ## # @author Craig Warren # @param log_name # the name of the log to return # @return Log # returns the log if it can't find the log it # returns None def get_log(self, log_name): for child in self.children_nodes(): if child.name == log_name: return child return None
class XMLFormatter(Formatter): MIME_TYPE='text/xml' def __init__(self): Formatter.__init__(self) self._channels = {} # {name:{uom:,meastype:,Delta:,Totalized:,key:}} self._exception_log = None # NEVER CREATE A PDO BEFORE THE NODE IS INSERTED IN THE NODE TREE! self._PDO = None return ## # @param config # @key timestamp_format the timestamp format string example: Y-%m-%dT%H:%M:%S. # @key info the information that will be placed in the info attribute of the # data tag def configure(self, config): Formatter.configure(self, config) set_attribute(self, 'debug_lvl', 0, config, int) set_attribute(self, 'timestamp_format', '%Y-%m-%dT%H:%M:%S', config) set_attribute(self, 'pretty_format',0,config,as_boolean) set_attribute(self, 'location_info','DefaultLocationInfo',config) set_attribute(self, 'location_key','DefaultLocationKey',config) set_attribute(self, 'panel_info','DefaultPanelInfo',config) set_attribute(self, 'panel_key','DefaultPanelKey',config) set_attribute(self, 'capture_period',24.0,config,float) # capture period preceding data transmission time (hrs) set_attribute(self, 'exception_log_url','/services/logger/fsg_exception_log',config) ## # @returns returns the configuratation def configuration(self): config = Formatter.configuration(self) get_attribute(self, 'debug_lvl', config, int) get_attribute(self, 'timestamp_format', config) get_attribute(self, 'pretty_format',config,str) get_attribute(self, 'location_info',config) get_attribute(self, 'location_key',config) get_attribute(self, 'panel_info',config) get_attribute(self, 'panel_key',config) get_attribute(self, 'capture_period',config,float) # capture period preceding data transmission time (hrs) get_attribute(self, 'exception_log_url',config) return config def start(self): self._PDO = PersistentDataObject(self,dmtype=GC_NEVER) self._PDO.exception_log_last_time = 0.0 self._PDO.load() # Scan subtree of grandparent logger for channel (column) 'fsg_attrs' # nodes containing info required for FSG Demo, so that we don't have # to do the scan every time format() is called: self._channels = {} columns_node = self.parent.parent.parent.get_child('columns') column_nodes = columns_node.children_nodes() for column_node in column_nodes: if column_node.name == 'timestamp': continue assert isinstance(column_node, ChannelAttrsColumn) \ or isinstance(column_node, ChannelAttrsDeltaColumn), \ 'Column %s should be class ChannelAttrsColumn, but is class %s' \ % (column_node.name, column_node.__class__.__name__) self._channels[column_node.name] = { 'channel_node':column_node,'values':[] } self._exception_log = None try: self._exception_log = as_node(self.exception_log_url) except ENoSuchName: pass return ## # cancel(): # Called by exporter if attempted transport fails, to clear out pre-formatted # data waiting in self._channels value dicts. Else, the pre-formatted data # in self._channels is still present at next attempt, and will cause transport # of multiple copies of same data: # def cancel(self): for channel_dict in self._channels.values(): channel_dict['values'] = [] return ## # @param data list of dictionary values to be converted in to XML format. # @param pretty_format 0,1 optional parameter to return pretty xml, xml that has # carriage returns in it # @default 0 # @note timestamp MUST be on of the dictionary keys. # @trhows EIncompatibleFormat if timestamp is not a key in a dictionary entry. def format(self, data, pretty_format=None): # Organize all log data (list of time-based dicts) into a dict of # point-based lists. (Dict of lists could get REALLY large; may # need to do only one point at a time... # self._channels:K=col_name,V=col_dict # col_dict:K='column_node':,'values':list_of_2tuples # list_of_2tuples: [(timestamp,value),] # Only want records for preceding self.capture_period-hr period: end_time = time.time() start_time = self.parent.last_time() # ASSUME that parent is a periodic exporter... # Comment out line below, in favor of line above, because FSG tends to # disable their FTP server (effectively) for days at a time, but still # want all the data gathered during those blackout periods to go to the # FTP server when the server reappears with respect to the Mediator. # This change means that the FTP server recvs table-formatted data all # the way back to the last successful export, regardless of the # actual size of that data: #start_time = end_time - (self.capture_period * 3600.0) data_to_send = 0 data = data[:] self.debug_print('Data: %s' % data,None,1) removed_channels = [] for log_rec_dict in data: timestamp = log_rec_dict['timestamp'] if (timestamp < start_time) \ or (timestamp > end_time): continue for channel_name in log_rec_dict.keys(): if channel_name == 'timestamp': continue if not self._channels.has_key(channel_name): if not channel_name in removed_channels: msglog.log('fsg:xml_formatter',msglog.types.ERR, \ 'Channel %s has been removed from the configuration.' \ % channel_name) removed_channels.append(channel_name) continue data_to_send = 1 self._channels[channel_name]['values'].append((timestamp,log_rec_dict[channel_name],)) channel_names = self._channels.keys() # it's a list # Organize all data from exception log, if any: exception_dicts = {} # K:trigger name, V:time-sorted list of 2tuples # (timestamp, message) if not self._exception_log is None: if self._PDO.exception_log_last_time > start_time: start_time = self._PDO.exception_log_last_time + 0.00001 # do not re-send already-sent data exception_data = self._exception_log.get_range('timestamp',start_time,end_time) for log_rec_dict in exception_data: trigger_node_url = log_rec_dict['trigger_node_url'] trigger_node = as_node(trigger_node_url) assert isinstance(trigger_node, FsgComparisonTrigger), \ 'Node %s should be FsgComparisonTrigger, is %s' \ % (trigger_node.name, trigger_node.__class__) timestamp = log_rec_dict['timestamp'] trigger_node_msg = log_rec_dict['trigger_node_msg'] if not exception_dicts.has_key(trigger_node_url): exception_dicts[trigger_node_url] = {'trigger_node_url':trigger_node_url,'timestamps':[(timestamp,trigger_node_msg,)]} else: exception_dicts[trigger_node_url]['timestamps'].append((timestamp,trigger_node_msg,)) self._PDO.exception_log_last_time = timestamp self._PDO.save() if (data_to_send == 0) and (len(exception_dicts) == 0): msglog.log('fsg:xml_formatter',msglog.types.INFO,'No data or exceptions to send.') return None # nothing to send # Create an output stream to minimize the combined size of the XML # file and the remaining point_dicts contents during formatting: stream = StreamWithCallback(self.output_callback) stream.set_meta('channel_names',channel_names) stream.set_meta('exception_data',exception_dicts.values()) # pass in a list of "values" (dicts), to allow easy iteration stream.set_meta('index',0) # number of point time-value lists written to XML output stream formatter = SGMLFormatter() # Write opening tags: formatter.open_tag('data', info=self.location_info, key=self.location_key ) formatter.open_tag('device', info=self.panel_info, key=self.panel_key ) output = formatter.output() self.debug_print(output,None,1) stream.write(output) stream.set_meta('formatter',formatter) stream.set_meta('remaining', '') data_mode = 'channels' if data_to_send == 0: data_mode = 'exceptions' # no data for channels, so skip 'em stream.set_meta('data_mode',data_mode) return stream def output_callback(self, stream): remaining = stream.get_meta_value('remaining') if remaining: count = stream.write(remaining) remaining = remaining[count:] stream.set_meta('remaining',remaining) if remaining: return None data_mode = stream.get_meta_value('data_mode') if data_mode == 'close': stream.close() return del remaining index = stream.get_meta_value('index') formatter = stream.get_meta_value('formatter') data_mode = stream.get_meta_value('data_mode') if data_mode == 'channels': channel_names = stream.get_meta_value('channel_names') for i in range(index,index+5): # try writing 5 channel time-value lists at a time channel_dict = {} try: channel_dict = self._channels[channel_names[i]] except IndexError: # no more data available; move on to exceptions data_mode = 'exceptions' stream.set_meta('data_mode', data_mode) break channel_node = channel_dict['channel_node'] formatter.open_tag('channel',name=channel_node.channel_name, uom=channel_node.uom, meastype=channel_node.meastype, Delta=channel_node.Delta, Totalized=channel_node.Totalized, key=channel_node.key,) stream.set_meta('index',i+1) for timestamp,value in channel_dict['values']: ts = self.parent.time_function(timestamp) ts_str = time.strftime(self.timestamp_format, ts) formatter.open_tag('value',timestamp=ts_str) formatter.add_text(str(value)) formatter.close_tag('value') formatter.close_tag('channel') output = formatter.output() self.debug_print(output,None,1) channel_dict['values'] = [] # save memory: data is now in stream or remaining count = stream.write(output) if count != len(output): stream.set_meta('remaining',output[count:]) return None if data_mode == 'exceptions': exception_data = stream.get_meta_value('exception_data') for i in range(0,5): # try writing 5 exception lists at a time exception_dict = {} try: exception_dict = exception_data[i] except IndexError: # no more exceptions available; close out XML file try: formatter.close_tag('device') formatter.close_tag('data') output = formatter.output() self.debug_print(output,None,1) count = stream.write(output) if count != len(output): stream.set_meta('remaining',output[count:]) data_mode = 'close' stream.set_meta('data_mode', data_mode) return None stream.close() return None except: msglog.exception() return None trigger_node_url = exception_dict['trigger_node_url'] trigger_node = as_node(trigger_node_url) formatter.open_tag('exception',name=trigger_node.name, key=trigger_node.key,) stream.set_meta('index',i+1) for exc_timestamp,msg in exception_dict['timestamps']: ts = self.parent.time_function(exc_timestamp) ts_str = time.strftime(self.timestamp_format, ts) formatter.open_tag('value',timestamp=ts_str) formatter.add_text(msg) formatter.close_tag('value') formatter.close_tag('exception') output = formatter.output() self.debug_print(output,None,1) count = stream.write(output) if count != len(output): stream.set_meta('remaining',output[count:]) return None del exception_data[0:5] # save memory: exception data is now in XML output stream return None def debug_print(self, msg_fmt_str, msg_value_tuple=None, msg_lvl=1): if msg_lvl <= self.debug_lvl: if msg_value_tuple is None: prn_msg = 'FsgXmlFmttr: ' + msg_fmt_str else: prn_msg = 'FsgXmlFmttr: ' + (msg_fmt_str % msg_value_tuple) print prn_msg self.parent.msglog(prn_msg) return
class ExportersConfigurator(CompositeNode): security = SecurityInformation.from_default() secured_by(security) def __init__(self, *args): self._pdo_lock = Lock() self.manager = None super(ExportersConfigurator, self).__init__(*args) def configure(self, config): self.setattr('path', config.get('path','/exportconfig')) self.setattr('container', config.get('container','/services/Alarm Exporters')) self.secured = as_internal_node("/services").secured super(ExportersConfigurator, self).configure(config) def configuration(self): config = super(ExportersConfigurator, self).configuration() config['path'] = self.getattr('path') config['container'] = self.getattr('container') return config def start(self): self.container = self.nodespace.as_node(self.container) self._pdo_lock.acquire() try: self._pdo = PersistentDataObject(self, dmtype=GC_NEVER) self._pdo.exporters = {} self._pdo.load() exporterdumps = self._pdo.exporters.values() finally: self._pdo_lock.release() super(ExportersConfigurator, self).start() tstart = time.time() for exporterdump in exporterdumps: IPickles(cPickle.loads(exporterdump))() tend = time.time() tlapse = tend - tstart msglog.log('broadway', msglog.types.INFO, 'Exporter Configurator loaded ' '%s exporters in %s seconds.' % (len(exporterdumps), tlapse)) self.manager = self.container def stop(self): super(ExportersConfigurator, self).stop() self.container = None def match(self, path): return path.startswith(self.path) security.protect('create_node', 'Configure') def create_node(self, name, config=()): config = dict(config) config.setdefault("name", name) config.setdefault("parent", self.manager) exporter = self.manager.nodespace.create_node(AlarmExporter) exporter.configure(config) exporter.start() self.updatepdo() return exporter.name security.protect('remove_node', 'Configure') def remove_node(self, name): exporter = self.manager.get_child(name) exporter.prune() self.updatepdo() return exporter.name security.protect('configure_node', 'Configure') def configure_node(self, name, config): exporter = self.manager.get_child(name) exporter.configure(config) self.updatepdo() return exporter.name security.protect('node_configuration', 'View') def node_configuration(self, name, extended=False): exporter = self.manager.get_child(name) return exporter.configuration() security.protect('configure_formatter', 'Configure') def configure_formatter(self, exporter, config): return self.configure_node(exporter, {"formatter": config}) security.protect('formatter_configuration', 'View') def formatter_configuration(self, exporter, extended=False): return self.node_configuration(exporter).get("formatter", {}) security.protect('configure_transporter', 'Configure') def configure_transporter(self, exporter, config): return self.configure_node(exporter, {"transporter": config}) security.protect('transporter_configuration', 'View') def transporter_configuration(self, exporter, extended=False): return self.node_configuration(exporter).get("transporter", {}) security.protect('trigger_configuration', 'View') def trigger_configuration(self, name=None): manager = self.nodespace.as_node('/services/Alarm Manager') sources = [manager] + manager.get_alarms() configuration = dict([(source.url, []) for source in sources]) if name: exporter = self.manager.get_child(name) configuration.update(exporter.trigger_configuration()) configs = [] for source,events in configuration.items(): configs.append({"source": source, "events": events}) return configs security.protect('configure_triggers', 'Configure') def configure_triggers(self, name, triggers=()): configuration = {} for config in triggers: configuration[config["source"]] = config["events"] exporter = self.manager.get_child(name) exporter.configure_triggers(configuration) self.updatepdo() security.protect('get_node_names', 'View') def get_node_names(self): return self.manager.children_names() def updatepdo(self): exporters = {} self._pdo_lock.acquire() try: for exporter in self.manager.get_exporters(): exporters[exporter.name] = cPickle.dumps(IPickles(exporter)) self._pdo.exporters = exporters self._pdo.save() finally: self._pdo_lock.release() def handle_request(self, request): update_pdo = False response = Response(request) request_data = request.get_post_data_as_dictionary() request_data.update(request.get_query_string_as_dictionary()) if request_data.has_key('add'): adapt = self.create_exporter("New Exporter") elif request_data.has_key('remove'): name = urllib.unquote_plus(request_data['remove'][0]) self.remove_exporter(name) adapt = self.container elif request_data.has_key('edit'): name = urllib.unquote_plus(request_data['edit'][0]) update_pdo = False adapt = self.container.get_exporter(name) elif request_data.has_key('configure'): name = urllib.unquote_plus(request_data['configure'][0]) exporter = self.container.get_exporter(name) config = {'Exporter': {}, 'Formatter': {}, 'Transporter': {}} for attrname in request_data.keys(): splitname = attrname.split('.') if len(splitname) == 2 and config.has_key(splitname[0]): config[splitname[0]][splitname[1]] = urllib.unquote_plus(request_data[attrname][0]) exportconfig = config['Exporter'] exportconfig['formatter'] = config['Formatter'] exportconfig['transporter'] = config['Transporter'] exporter.configure(exportconfig) update_pdo = True adapt = exporter else: adapt = self.container if request_data.has_key('actionName'): target = urllib.unquote_plus(request_data.get('target')[0]) action = urllib.unquote_plus(request_data.get('actionName')[0]) params = map(urllib.unquote_plus, request_data.get('params')) exporter = self.container.get_exporter(target) method = getattr(exporter, action) result = method(*params) update_pdo = True if update_pdo: self.updatepdo() webadapter = IWebContent(adapt) response.send(webadapter.render())
class Control(CompositeNode): ## # This attribute is used in the introspective generation # of configuration data. __module__ = mpx.service.control.__name__ def __init__(self): CompositeNode.__init__(self) self._status = 'initialized' self._stale_apps = [] def configure(self, config): self._pdo = PersistentDataObject(self) self._pdo.stats_dict = { } #used to detect changes to xml files based on timestamp. Tuple (modify time, pickle string) self._pdo.load() # write_priority can be set for the entire control service, though # it may be specialized at the individual application node level. set_attribute(self, 'write_priority', 9, config, int) CompositeNode.configure(self, config) def configuration(self): config = CompositeNode.configuration(self) self.pdo_file = self._pdo.filename() get_attribute(self, 'write_priority', config) get_attribute(self, 'pdo_file', config, str) return config def _save_pdo( self): #no locking needed since the load and save cannot overlap start_time = time.time() self._pdo.save() msglog.log( self.as_node_url(), msglog.types.INFO, 'Control service configuration data saved in: %s seconds' % (str(time.time() - start_time), )) def start(self): self._status = 'starting' self.stats_dict = {} #clear out stats dict to force reload of app self.application_change_detector(1) #starting def _start(self): CompositeNode.start(self) #get the children ready for a trip... #now that the children are started, go back through the list and finish up the "graphical compile" for n in self.children_nodes(): if n.hasattr('map_output_connections'): n.map_output_connections() for n in self.children_nodes(): if n.hasattr('map_reference_output_connections'): n.map_reference_output_connections() for n in self.children_nodes(): if n.hasattr('resolve_inputs'): n.resolve_inputs() for n in self.children_nodes(): if n.hasattr('prepare_run_list'): n.prepare_run_list() for n in self.children_nodes(): if n.hasattr('trigger_run_list'): n.trigger_run_list() def prune_orphaned_schedules(self): # remove schedules under /services/time/local/TIM that have no app manager = as_node('/services/time/local') if manager.has_child('TIM'): try: sh = as_node('/services/time/local/TIM') name_header = 'RZSched_' # create list of RZSched_'s under the TIM node schedules = filter( lambda k: k[:len(name_header)] == name_header, sh.children_names()) # compare appname after RZSched_, upto : with our children names orphans = filter( lambda k: k.split('_')[1].split(':')[0] not in self. children_names(), schedules) for o in orphans: try: sh.get_child(o).prune() msglog.log('Graphical Control:', 'pruned orphaned schedule: ', o) except: msglog.exception() if len(orphans): sh.save_schedule() except: msglog.exception() def check_and_load_application_files(self, starting=0): app_reloaded = starting #return value to signal that the children need to be started save_pdo = 0 #flag to control saving config data to pdo files = os.listdir( config_path) #/var/mpx/config/services/control (usually) xml_filenames = [] for f in files: if f.find('.xml') > 0 and len(f) == ( f.find('.xml') + 4 ): #any xml file in this folder is assumed to be a control app xml_filenames.append(f) modify_time = os.stat(config_path + f)[8] stale_pdo = True no_stats_pdo = True if f in self._pdo.stats_dict: #check for change since last time no_stats_pdo = False if self._pdo.stats_dict[f][0] == modify_time: stale_pdo = False #old news, no change detected #self.stats_dict[f]=modify_time if starting or no_stats_pdo or ( stale_pdo and ALLOW_APP_RELOAD): #need to (re)load application if app_reloaded == 0: #only stop all application nodes for the first detected change try: self._status = 'Stopping %s' % (f, ) msglog.log( self.as_node_url(), msglog.types.INFO, 'Stage 0: Stop Application templates.') for c in self.children_nodes(): if hasattr(c, '_stop_running_app'): c._stop_running_app() except: msglog.exception() app_reloaded = 1 #no need to "stop" for any other app changes self._status = 'Loading %s' % (f, ) try: root = None if not stale_pdo: #so no change was detected, we are starting up the framework try: #to get the pickled config data rather than load the xml again msglog.log( self.as_node_url(), msglog.types.INFO, 'Stage 1: XML unchanged. Loading configuration data from PDO: %s' % (f, )) root = cPickle.loads( self._pdo.stats_dict[f][1]) except: msglog.exception() msglog.log( self.as_node_url(), msglog.types.WARN, 'Stage 1: Unable to reload config data. Next, try XML file.' ) if root is None: msglog.log( self.as_node_url(), msglog.types.INFO, 'Stage 1: Parsing configuration xml file: %s' % (f, )) root = parse_xml(config_path + f) self._pdo.stats_dict[f] = (modify_time, cPickle.dumps(root)) save_pdo = 1 if f in self._stale_apps: self._stale_apps.remove(f) #now we have the root configuration. Turn it into configured nodes module = root.get_config().get('module', None) if module == 'mpx.ion.rz.rzhost_node.RzhostNode': load_rz_root(root, self) elif module == 'mpx.service.control.graphical.ApplicationNode': load_tim_root(root, self) else: raise EInvalidValue() except Exception, e: msglog.exception() pass elif stale_pdo: if not f in self._stale_apps: msglog.log( self.as_node_url(), msglog.types.INFO, 'Application %s has been modified, please restart the framework.' % (f, )) self._stale_apps.append(f) try: self.get_child(f.split('.')[0]).set_stale_flag() except: msglog.exception() #clear out any leftover pdo for deleted files for k in self._pdo.stats_dict.keys(): try: if k not in xml_filenames: save_pdo = 1 #force save of modififed pdo del self._pdo.stats_dict[k] n = k.split('.xml')[0] if self.has_child( n): # prune any running app who's file was deleted try: self.get_child(n).prune() self.prune_orphaned_schedules() except: msglog.exception() except: pass if save_pdo: thread_pool.LOW.queue_noresult(self._save_pdo) return app_reloaded
class ExportersConfigurator(CompositeNode): security = SecurityInformation.from_default() secured_by(security) def __init__(self, *args): self._pdo_lock = Lock() self.manager = None super(ExportersConfigurator, self).__init__(*args) def configure(self, config): self.setattr('path', config.get('path', '/exportconfig')) self.setattr('container', config.get('container', '/services/Alarm Exporters')) self.secured = as_internal_node("/services").secured super(ExportersConfigurator, self).configure(config) def configuration(self): config = super(ExportersConfigurator, self).configuration() config['path'] = self.getattr('path') config['container'] = self.getattr('container') return config def start(self): self.container = self.nodespace.as_node(self.container) self._pdo_lock.acquire() try: self._pdo = PersistentDataObject(self, dmtype=GC_NEVER) self._pdo.exporters = {} self._pdo.load() exporterdumps = self._pdo.exporters.values() finally: self._pdo_lock.release() super(ExportersConfigurator, self).start() tstart = time.time() for exporterdump in exporterdumps: IPickles(cPickle.loads(exporterdump))() tend = time.time() tlapse = tend - tstart msglog.log( 'broadway', msglog.types.INFO, 'Exporter Configurator loaded ' '%s exporters in %s seconds.' % (len(exporterdumps), tlapse)) self.manager = self.container def stop(self): super(ExportersConfigurator, self).stop() self.container = None def match(self, path): return path.startswith(self.path) security.protect('create_node', 'Configure') def create_node(self, name, config=()): config = dict(config) config.setdefault("name", name) config.setdefault("parent", self.manager) exporter = self.manager.nodespace.create_node(AlarmExporter) exporter.configure(config) exporter.start() self.updatepdo() return exporter.name security.protect('remove_node', 'Configure') def remove_node(self, name): exporter = self.manager.get_child(name) exporter.prune() self.updatepdo() return exporter.name security.protect('configure_node', 'Configure') def configure_node(self, name, config): exporter = self.manager.get_child(name) exporter.configure(config) self.updatepdo() return exporter.name security.protect('node_configuration', 'View') def node_configuration(self, name, extended=False): exporter = self.manager.get_child(name) return exporter.configuration() security.protect('configure_formatter', 'Configure') def configure_formatter(self, exporter, config): return self.configure_node(exporter, {"formatter": config}) security.protect('formatter_configuration', 'View') def formatter_configuration(self, exporter, extended=False): return self.node_configuration(exporter).get("formatter", {}) security.protect('configure_transporter', 'Configure') def configure_transporter(self, exporter, config): return self.configure_node(exporter, {"transporter": config}) security.protect('transporter_configuration', 'View') def transporter_configuration(self, exporter, extended=False): return self.node_configuration(exporter).get("transporter", {}) security.protect('trigger_configuration', 'View') def trigger_configuration(self, name=None): manager = self.nodespace.as_node('/services/Alarm Manager') sources = [manager] + manager.get_alarms() configuration = dict([(source.url, []) for source in sources]) if name: exporter = self.manager.get_child(name) configuration.update(exporter.trigger_configuration()) configs = [] for source, events in configuration.items(): configs.append({"source": source, "events": events}) return configs security.protect('configure_triggers', 'Configure') def configure_triggers(self, name, triggers=()): configuration = {} for config in triggers: configuration[config["source"]] = config["events"] exporter = self.manager.get_child(name) exporter.configure_triggers(configuration) self.updatepdo() security.protect('get_node_names', 'View') def get_node_names(self): return self.manager.children_names() def updatepdo(self): exporters = {} self._pdo_lock.acquire() try: for exporter in self.manager.get_exporters(): exporters[exporter.name] = cPickle.dumps(IPickles(exporter)) self._pdo.exporters = exporters self._pdo.save() finally: self._pdo_lock.release() def handle_request(self, request): update_pdo = False response = Response(request) request_data = request.get_post_data_as_dictionary() request_data.update(request.get_query_string_as_dictionary()) if request_data.has_key('add'): adapt = self.create_exporter("New Exporter") elif request_data.has_key('remove'): name = urllib.unquote_plus(request_data['remove'][0]) self.remove_exporter(name) adapt = self.container elif request_data.has_key('edit'): name = urllib.unquote_plus(request_data['edit'][0]) update_pdo = False adapt = self.container.get_exporter(name) elif request_data.has_key('configure'): name = urllib.unquote_plus(request_data['configure'][0]) exporter = self.container.get_exporter(name) config = {'Exporter': {}, 'Formatter': {}, 'Transporter': {}} for attrname in request_data.keys(): splitname = attrname.split('.') if len(splitname) == 2 and config.has_key(splitname[0]): config[splitname[0]][splitname[1]] = urllib.unquote_plus( request_data[attrname][0]) exportconfig = config['Exporter'] exportconfig['formatter'] = config['Formatter'] exportconfig['transporter'] = config['Transporter'] exporter.configure(exportconfig) update_pdo = True adapt = exporter else: adapt = self.container if request_data.has_key('actionName'): target = urllib.unquote_plus(request_data.get('target')[0]) action = urllib.unquote_plus(request_data.get('actionName')[0]) params = map(urllib.unquote_plus, request_data.get('params')) exporter = self.container.get_exporter(target) method = getattr(exporter, action) result = method(*params) update_pdo = True if update_pdo: self.updatepdo() webadapter = IWebContent(adapt) response.send(webadapter.render())
class EnergywiseManager(CompositeNode): def __init__(self): CompositeNode.__init__(self) self._pdo_lock = Lock() self._pdo = None self.__running = False self.debug = 0 return def configure(self, config): if self.debug: msglog.log('EnergywiseManager:', msglog.types.INFO, 'Inside configure') CompositeNode.configure(self, config) set_attribute(self, 'debug', 0, config, int) return def configuration(self): config = CompositeNode.configuration(self) get_attribute(self, 'debug', config, str) return config # def configure_trend_in_switches(self, start_node, frequency): # for child in start_node.children_nodes(): # if child.children_nodes(): # self.configure_trend_in_switches(child, frequency) # else: # reached upto leaf, each energywise switch has trends as child # child.new_trend(frequency) #return def delete_trend_configuration(self, trend_domain): self._pdo_lock.acquire() try: if self._pdo.trends.has_key(trend_domain): # stop logging as well del self._pdo.trends[trend_domain] self._pdo.save() finally: self._pdo_lock.release() return def delete_trends(self, trendList): if self.debug: msglog.log('EnergywiseManager:', msglog.types.INFO, 'Inside delete_trends') for domain in trendList.split(':'): if domain: domain_node = as_node(domain) domain_node.delete_trend() self.delete_trend_configuration(domain) return def start(self): if self.__running: return if self.debug: msglog.log('EnergywiseManager :', msglog.types.INFO, 'Inside start') CompositeNode.start(self) # start_node = as_node('/services/EnergywiseManager/') # self.configure_trend_in_switches(start_node, 60) self.__running = True self._pdo_lock.acquire() self._pdo = PersistentDataObject(self, dmtype=GC_NEVER) self._pdo.trends = {} self._pdo.load() self._pdo_lock.release() if self.has_child('trends'): self.trends = self.get_child('trends') else: self.trends = CompositeNode() self.trends.configure({'parent': self, 'name': 'trends'}) self.trends.start() # start trending for saved domains for domain, freq in self._pdo.trends.items(): try: start_node = as_node(domain) # self.configure_trend_in_switches( start_node,freq ) start_node.new_trend(freq) except: self.delete_trend_configuration(domain) return def get_trends(self): return self._pdo.trends.items() def add_trend_configuration(self, trend_period, trend_domain): self._pdo_lock.acquire() self._pdo.trends[trend_domain] = trend_period self._pdo.save() self._pdo_lock.release() return def save_trends(self, trend_list): # Traverse through _pdo.items and check if new domain is either subset # of any configured or superset. # If subset return with msg already covered and dont save this # If superset then configure new ones and delete subset from # _pdo.items '''Adding and saving trends''' for point in reversed(trend_list): point_period = point['frequency'] point_domain = point['domain'] for saved_domain, saved_period in tuple(self._pdo.trends.items()): if saved_domain == point_domain: if saved_period != point_period: self.delete_trend_configuration(saved_domain) break if not self._pdo.trends.has_key(point_domain): # add this trend try: domain_node = as_node(point_domain) if isinstance(domain_node, EnergywiseSwitch) or isinstance( domain_node, EnergywiseDomain): self.add_trend_configuration(point_period, point_domain) domain_node.new_trend(point_period) except Exception: msglog.exception() msglog.log( "Energywise", msglog.types.ERR, "Failed to create trend for %r every %r seconds" % (point_domain, point_period)) return def stop(self): CompositeNode.stop(self) self.__running = False return
class GarbageCollector(ServiceNode): def __init__(self): ServiceNode.__init__(self) self.debug = 0 self._registered = () self._did_save = 0 self._post_configuration=0 if self.debug: print 'Initialized _registered to [] for %s.' % self def singleton_unload_hook(self): return ## # @param config # @return None def configure(self,config): ServiceNode.configure(self,config) def configuration(self): if self.debug: print 'In GarbageCollector:configuration().' config = ServiceNode.configuration(self) return config ## # starts the data manager service # @return None def start(self): from mpx.lib.persistent import PersistentDataObject ServiceNode.start(self) if self.debug: print 'Garbage Collector Starting!' self._data = PersistentDataObject(self,dmtype=GC_NEVER) self._data.registered = [] self._data.load() if self.debug: print 'GC Data is %s.' % self._data if self.debug: print 'GC Data._reg is %s.' % self._data.registered ## # stops the data manager service # @return None def stop(self): return ServiceNode.stop(self) ## # set_faillist is the hook which allows the system to inform the data # manager about which nodes failed to start up. Each list item should # be a dictionary with the following members: # name - the name of the node (without parent information) # parent - the parent of the node (with any relevant parent information, # e.g. /services/com1 # type - what type of failure occured. Acceptable values are # load and config. def set_faillist(self, faillist): if self.debug: print 'Got faillist of %s.' % faillist if self.debug: print 'Got reglist of %s.' % self._registered old_registered = self._data.registered[:] # By now, everyone should have had a chance to start up. # @fixme (real soon now, I promise): Use the cool new method that # Mark and Shane suggested to consume an event from the root node # when all nodes have been started as a trigger for starting # the garbage collection process. self._data.registered = list(self._registered) # OK, now process our lists and see who is naughty and who is # nice. if self.debug: print '---- Starting to Process Potential Reaping List ----' for regnode in old_registered: purge_type = regnode['type'] filename = regnode['filename'] nodename = regnode['nodename'] # If we are never supposed to purge this data, then don't bother # to do any more checking if purge_type == GC_NEVER: if self.debug: print '%s: Skipping because it is GC_NEVER.' % nodename continue if self.debug: print '%s: Checking.' % nodename node_did_register = 0 node_did_fail = 0 node_did_fail_on_config = 0 node_did_fail_on_load = 0 node_did_fail_on_start = 0 parent_did_fail = 0 should_purge = 0 # If this node has registered with us, then we assume that # it started up and is present, etc. This might not always # be the correct thing to do, but for now it seems like the # correct enough thing to do and should keep performance # from becoming an issue. if regnode in self._registered: if self.debug: print '%s: Appears to be happy.' % nodename node_did_register = 1 else: # Check to see if the node or one of it's ancestors failed for failnode in faillist: curnode = failnode['name'] curpare = failnode['parent'] curtype = failnode['type'] if curpare == '/': curpath = curpare + curnode else: curpath = curpare + '/' + curnode if self.debug: print 'curpath is %s and nodename is %s.' % (curpath, nodename) if nodename == curpath: if self.debug: print 'We got a match, %s failed because of %s.' % ( nodename, curtype) if curtype == 'load': node_did_fail_on_load = 1 elif curtype == 'config': node_did_fail_on_config = 1 else: raise 'Unrecognized failure type: %s.' % curtype # Don't need to check any further break else: if self._path_is_parent(curpath, nodename): if self.debug: print 'We found a parent who failed: %s.' % curpath parent_did_fail = 1 # Don't need to check any further break if node_did_fail_on_config or node_did_fail_on_load: node_did_fail = 1 # If the node didn't fail in load or config, but it didn't register either, # then check to see if perhaps it exists, but didn't start. We detect this # by doing an as_node on it. If this succeeds, we can check the node's state. # If it doesn't succeed, then we can pretty safely assume that the node # has been delete (or, unfortunately, is auto-discovered). if not node_did_fail and not node_did_register: try: x = as_node(nodename) node_did_fail_on_start = 1 node_did_fail = 1 if self.debug: print 'We were able to instantiate node: %s.' % nodename except: if self.debug: print 'Failed to instantiate node: %s.' % nodename # The node doesn't seem to exist at all. Let the following code # draw the appropriate conclusions. pass if not node_did_register: if self.debug: print 'node_did_fail_on_load: %d.' % node_did_fail_on_load if self.debug: print 'node_did_fail_on_config: %d.' % node_did_fail_on_config if self.debug: print 'node_did_fail_on_start: %d.' % node_did_fail_on_start if self.debug: print 'node_did_fail: %d.' % node_did_fail if self.debug: print 'parent_did_fail: %d.' % parent_did_fail if self.debug: print 'purge_type: %d.' % purge_type # OK, the node didn't register. Check to see what we've # been told to do in this case. if node_did_fail and (purge_type == GC_ONFAILURE): should_purge = 1 # For now, purge even if it was a parent who failed and purge_type # is GC_ONFAILURE. @fixme: We need to think about if this is what # we want to do or not. if parent_did_fail and (purge_type == GC_ONFAILURE): should_purge = 1 # If the node did not register and neither it nor a parent # failed to start, then we assume that it has been deleted. # Note: This does not seem to be correct for auto-discovered # nodes, so we need a better way of detecting this case. if (not node_did_fail) and (not parent_did_fail) and (purge_type == GC_ONDELETE): should_purge = 1 # If the node did not register and we aren't going to purge it, then # save it's registration information so that if circumstances change, # we can consider purging it at some later date. if (not node_did_register) and (not should_purge): if self.debug: print '%s did not register, but we are registering for it.' % nodename self._data.registered.append(regnode) # OK, we've figured out that we should purge this persistent # data. Go ahead and do so. if should_purge: if os.access(filename, os.F_OK): if self.debug: print 'We decided we should purge the following file: %s.' % filename msglog.log('garbage_collector',msglog.types.INFO, 'Purging the following persistent data file: %s on behalf of %s.' % (filename, nodename)) try: os.remove(filename) except: msglog.log('garbage_collector',msglog.types.INFO, 'Got exception trying to remove persistent data: %s.' % filename) msglog.exception('garbage_collector') else: if self.debug: print '%s: Will SAVE the following file: %s.' % (nodename, filename) if self.debug: print '---- Done Processing Potential Reaping List ----' # Now, at long last, persist our own data. self._data.save() self._did_save = 1 self._post_configuration=1 def register(self, nodename, filename, type=None): # Default to GC_ONDELETE if type == None: type = GC_ONDELETE if self.debug: print '%s: Registered with type of %d.' % (nodename, type) if self._post_configuration: self._data.registered+=( {'nodename':nodename, 'filename':filename, 'type':type}, ) else: self._registered += ( {'nodename':nodename, 'filename':filename, 'type':type}, ) # If we have already saved our data, but just received a new registration, # then save it again. if self._did_save: self._data.save() def _path_is_parent(self, path, node): # If they are the same, we aren't talking a parent/child relationship here if path == node: return 0 strind = string.find(node, path) if strind == -1: return 0 if strind == 0: return 1 # If we got something other than -1 or 0 here, strange things are # happening. Dump a message to msglog so that whatever is wrong # can be fixed. msglog.log( 'garbage_collector',msglog.types.INFO, '_path_is_parent: Found %s at a weird spot in %s.' % (node, parent) ) return 1 ## # Return a tuple of dict()s describing all the registered PDOs. # @note DO NOT MODIFY THE DICT()s IN THE TUPLE! def registered_pdo_tuple(self): return self._registered
class Control(CompositeNode): ## # This attribute is used in the introspective generation # of configuration data. __module__ = mpx.service.control.__name__ def __init__(self): CompositeNode.__init__(self) self._status = 'initialized' self._stale_apps = [] def configure(self, config): self._pdo = PersistentDataObject(self) self._pdo.stats_dict = {} #used to detect changes to xml files based on timestamp. Tuple (modify time, pickle string) self._pdo.load() # write_priority can be set for the entire control service, though # it may be specialized at the individual application node level. set_attribute(self, 'write_priority', 9, config, int) CompositeNode.configure(self, config) def configuration(self): config = CompositeNode.configuration(self) self.pdo_file = self._pdo.filename() get_attribute(self, 'write_priority', config) get_attribute(self, 'pdo_file', config, str) return config def _save_pdo(self): #no locking needed since the load and save cannot overlap start_time = time.time() self._pdo.save() msglog.log(self.as_node_url(),msglog.types.INFO, 'Control service configuration data saved in: %s seconds' % (str(time.time() - start_time),)) def start(self): self._status = 'starting' self.stats_dict = {} #clear out stats dict to force reload of app self.application_change_detector(1) #starting def _start(self): CompositeNode.start(self) #get the children ready for a trip... #now that the children are started, go back through the list and finish up the "graphical compile" for n in self.children_nodes(): if n.hasattr('map_output_connections'): n.map_output_connections() for n in self.children_nodes(): if n.hasattr('map_reference_output_connections'): n.map_reference_output_connections() for n in self.children_nodes(): if n.hasattr('resolve_inputs'): n.resolve_inputs() for n in self.children_nodes(): if n.hasattr('prepare_run_list'): n.prepare_run_list() for n in self.children_nodes(): if n.hasattr('trigger_run_list'): n.trigger_run_list() def prune_orphaned_schedules(self): # remove schedules under /services/time/local/TIM that have no app manager = as_node('/services/time/local') if manager.has_child('TIM'): try: sh = as_node('/services/time/local/TIM') name_header = 'RZSched_' # create list of RZSched_'s under the TIM node schedules = filter(lambda k:k[:len(name_header)] == name_header, sh.children_names()) # compare appname after RZSched_, upto : with our children names orphans = filter(lambda k:k.split('_')[1].split(':')[0] not in self.children_names(), schedules) for o in orphans: try: sh.get_child(o).prune() msglog.log('Graphical Control:', 'pruned orphaned schedule: ', o) except: msglog.exception() if len(orphans): sh.save_schedule() except: msglog.exception() def check_and_load_application_files(self, starting=0): app_reloaded = starting #return value to signal that the children need to be started save_pdo = 0 #flag to control saving config data to pdo files = os.listdir(config_path) #/var/mpx/config/services/control (usually) xml_filenames = [] for f in files: if f.find('.xml') > 0 and len(f) == (f.find('.xml') + 4): #any xml file in this folder is assumed to be a control app xml_filenames.append(f) modify_time = os.stat(config_path + f)[8] stale_pdo = True no_stats_pdo = True if f in self._pdo.stats_dict: #check for change since last time no_stats_pdo = False if self._pdo.stats_dict[f][0] == modify_time: stale_pdo = False #old news, no change detected #self.stats_dict[f]=modify_time if starting or no_stats_pdo or (stale_pdo and ALLOW_APP_RELOAD): #need to (re)load application if app_reloaded == 0: #only stop all application nodes for the first detected change try: self._status = 'Stopping %s' % (f,) msglog.log(self.as_node_url(),msglog.types.INFO, 'Stage 0: Stop Application templates.') for c in self.children_nodes(): if hasattr(c, '_stop_running_app'): c._stop_running_app() except: msglog.exception() app_reloaded = 1 #no need to "stop" for any other app changes self._status = 'Loading %s' % (f,) try: root = None if not stale_pdo: #so no change was detected, we are starting up the framework try: #to get the pickled config data rather than load the xml again msglog.log(self.as_node_url(),msglog.types.INFO, 'Stage 1: XML unchanged. Loading configuration data from PDO: %s' % (f,)) root = cPickle.loads(self._pdo.stats_dict[f][1]) except: msglog.exception() msglog.log(self.as_node_url(),msglog.types.WARN, 'Stage 1: Unable to reload config data. Next, try XML file.') if root is None: msglog.log(self.as_node_url(),msglog.types.INFO, 'Stage 1: Parsing configuration xml file: %s' % (f,)) root = parse_xml(config_path + f) self._pdo.stats_dict[f] = (modify_time, cPickle.dumps(root)) save_pdo = 1 if f in self._stale_apps: self._stale_apps.remove(f) #now we have the root configuration. Turn it into configured nodes module = root.get_config().get('module', None) if module == 'mpx.ion.rz.rzhost_node.RzhostNode': load_rz_root(root, self) elif module == 'mpx.service.control.graphical.ApplicationNode': load_tim_root(root, self) else: raise EInvalidValue() except Exception, e: msglog.exception() pass elif stale_pdo: if not f in self._stale_apps: msglog.log(self.as_node_url(), msglog.types.INFO, 'Application %s has been modified, please restart the framework.' % (f,)) self._stale_apps.append(f) try: self.get_child(f.split('.')[0]).set_stale_flag() except: msglog.exception() #clear out any leftover pdo for deleted files for k in self._pdo.stats_dict.keys(): try: if k not in xml_filenames: save_pdo = 1 #force save of modififed pdo del self._pdo.stats_dict[k] n = k.split('.xml')[0] if self.has_child(n): # prune any running app who's file was deleted try: self.get_child(n).prune() self.prune_orphaned_schedules() except: msglog.exception() except: pass if save_pdo: thread_pool.LOW.queue_noresult(self._save_pdo) return app_reloaded