def log(self,severity,msg,remote=None): """ The remote flag should allow operations done outside of QMainThread to be logged """ remote = remote if remote is not None else self.remote if remote: self.emit(Qt.SIGNAL('logging'),severity,msg,False) return if msg == self.last_msg: msg = '+1' else: self.last_msg = msg if self.logger: try: if severity not in self.log_objs: self.log_objs[severity] = \ getattr(self.logger,severity,(lambda m,s=severity:'%s:%s: %s'%(s.upper(),F.time2str(),m))) self.log_objs[severity](msg) except: pass if self.dialog(): if msg!='+1': msg = '%s:%s: %s'%(severity.upper(),F.time2str(),msg) if self.filters: msg = (F.filtersmart(msg,self.filters) or [''])[0] if msg: self.dialog().append(msg)
def start_devices(self,regexp = '*', force = False, do_init = False, do_restart = False): #devs = fn.tango.get_class_devices('HdbEventSubscriber') devs = self.get_archivers() if regexp: devs = fn.filtersmart(devs,regexp) off = sorted(set(d for d in devs if not fn.check_device(d))) if off and do_restart: print('Restarting %s Archiving Servers ...'%self.db_name) astor = fn.Astor() astor.load_from_devs_list(list(off)) astor.stop_servers() fn.wait(3.) astor.start_servers() fn.wait(3.) for d in devs: try: dp = fn.get_device(d, keep=True) if do_init: dp.init() if force or dp.attributenumber != dp.attributestartednumber: off.append(d) print('%s.Start()' % d) dp.start() except Exception,e: self.warning('start_archivers(%s) failed: %s' % (d,e))
def log(self, severity, msg, remote=None): """ The remote flag should allow operations done outside of QMainThread to be logged """ remote = remote if remote is not None else self.remote if remote: self.emit(Qt.SIGNAL('logging'), severity, msg, False) return if msg == self.last_msg: msg = '+1' else: self.last_msg = msg if self.logger: try: if severity not in self.log_objs: self.log_objs[severity] = \ getattr(self.logger,severity,(lambda m,s=severity:'%s:%s: %s'%(s.upper(),F.time2str(),m))) self.log_objs[severity](msg) except: pass if self.dialog(): if msg != '+1': msg = '%s:%s: %s' % (severity.upper(), F.time2str(), msg) if self.filters: msg = (F.filtersmart(msg, self.filters) or [''])[0] if msg: self.dialog().append(msg)
def start_devices(self, regexp='*', force=False, do_init=False, do_restart=False): #devs = fn.tango.get_class_devices('HdbEventSubscriber') devs = self.get_archivers() if regexp: devs = fn.filtersmart(devs, regexp) off = sorted(set(d for d in devs if not fn.check_device(d))) if off and do_restart: print('Restarting %s Archiving Servers ...' % self.db_name) astor = fn.Astor() astor.load_from_devs_list(list(off)) astor.stop_servers() fn.wait(3.) astor.start_servers() fn.wait(3.) for d in devs: try: dp = fn.get_device(d) if do_init: dp.init() if force or dp.attributenumber != dp.attributestartednumber: off.append(d) print('%s.Start()' % d) dp.start() except Exception, e: self.warning('start_archivers(%s) failed: %s' % (d, e))
def get_attributes_failed(self, regexp='*', timeout=3600, from_db=True): if from_db: timeout = fn.now() - timeout attrs = self.get_attributes(True) attrs = fn.filtersmart(attrs, regexp) print('get_attributes_failed([%d])' % len(attrs)) print(attrs) vals = self.load_last_values(attrs) return sorted(t for t in vals if not t[1] or t[1][0] < timeout) else: # Should inspect the Subscribers Error Lists raise Exception('NotImplemented')
def __init__(self, mask=None): #mask = mask or 'FolderDS/*' self.mask = mask ProxiesDict.__init__(self) devs = tango.get_class_devices('FolderDS') extra = fn.get_database().get_class_property('FolderDS', ['ExtraDevices']) devs.extend(extra.get('ExtraDevices', [])) if mask: devs = fn.filtersmart(devs, mask) self.hosts = fn.defaultdict(list) for d in devs: self[d]
def get_attributes(self, active=None, regexp=''): """ Alias for Reader API """ if active: r = self.get_archived_attributes() else: # Inactive attributes must be read from Database r = self.get_attribute_names(False) r = sorted(set( fn.tango.get_full_name(a, fqdn=True).lower() for a in r)) return sorted(fn.filtersmart(r, regexp) if regexp else r)
def get_archivers_for_attributes(attrs=[], archs='archiving/es/*'): """ This method returns matching archivers for a list of attributes in simplename format (no tango host). It applies AttributeFilters as defined in Tango DB (sorted) """ if isString(attrs): attrs = ft.find_attributes(attrs) else: attrs = attrs or get_schema_attributes('*') devattrs = fn.dicts.defaultdict(set) [devattrs[a.rsplit('/', 1)[0]].add(a) for a in attrs] if isSequence(archs): archs = '(%s)' % ')|('.join(archs) filters = get_archivers_filters(archs) r = devattrs.keys() archattrs = {} for i, k in enumerate(filters): v = filters[k] # k is the archiver name k = fn.tango.parse_tango_model(k, fqdn=True).fullname if 'DEFAULT' in v: df = k else: #filtersmart(list,regexp): returns a clsearch on the list m = fn.filtersmart(r, v) currattrs = set(fn.join(*[devattrs[d] for d in m])) if len(currattrs): print(k, len(currattrs), sorted(set(i.split('/')[-2] for i in m))) archattrs[k] = currattrs print('\n') r = [a for a in r if a not in m] if i == len(filters) - 1: k = df m = r currattrs = fn.join(*[devattrs[d] for d in m]) if len(currattrs): print(k, len(currattrs), sorted(set(i.split('/')[-2] for i in m))) archattrs[k] = currattrs return archattrs
def get_attribute_names(self, active=False, regexp=''): t0 = fn.now() if not active: attributes = [ a[0].lower() for a in self.Query('select att_name from att_conf') ] [ self.get_attr_id_type_table(a) for a in attributes if a not in self.attributes ] r = self.attributes.keys() else: r = self.get_archived_attributes() r = sorted(fn.filtersmart(r, regexp) if regexp else r) self.debug('get attribute names took %d ms' % (1e3 * (fn.now() - t0))) return r
def get_archivers_for_attributes(attrs=[],archs='archiving/es/*'): """ This method returns matching archivers for a list of attributes in simplename format (no tango host). It applies AttributeFilters as defined in Tango DB (sorted) """ if isString(attrs): attrs = ft.find_attributes(attrs) else: attrs = attrs or get_schema_attributes('*') devattrs = fn.dicts.defaultdict(set) [devattrs[a.rsplit('/',1)[0]].add(a) for a in attrs]; if isSequence(archs): archs = '(%s)'%')|('.join(archs) filters = get_archivers_filters(archs) r = devattrs.keys() archattrs = {} for i,k in enumerate(filters): v = filters[k] k = fn.tango.parse_tango_model(k, fqdn = True).fullname if 'DEFAULT' in v: df = k else: m = fn.filtersmart(r,v) currattrs = set(fn.join(*[devattrs[d] for d in m])) if len(currattrs): print(k,len(currattrs),sorted(set(i.split('/')[-2] for i in m))) archattrs[k] = currattrs print('\n') r = [a for a in r if a not in m] if i == len(filters)-1: k = df m = r currattrs = fn.join(*[devattrs[d] for d in m]) if len(currattrs): print(k,len(currattrs),sorted(set(i.split('/')[-2] for i in m))) archattrs[k] = currattrs return archattrs
def log(self,severity,msg): if msg == self.last_msg: msg = '+1' else: self.last_msg = msg if self.logger: try: if severity not in self.log_objs: self.log_objs[severity] = \ getattr(self.logger,severity.lower(), lambda m,s=severity:'%s:%s: %s'% (s.upper(),fn.time2str(),m)) self.log_objs[severity](msg) except: pass if self.dialog(): if msg!='+1': msg = '%s:%s: %s'%(severity.upper(),fn.time2str(),msg) if self.filters: msg = (fn.filtersmart(msg,self.filters) or [''])[0] if msg: if len(self.instances())>1: msg = self.tango_host+':'+msg self.dialog().append(msg)
def log(self, severity, msg): if msg == self.last_msg: msg = '+1' else: self.last_msg = msg if self.logger: try: if severity not in self.log_objs: self.log_objs[severity] = \ getattr(self.logger,severity.lower(), lambda m,s=severity:'%s:%s: %s'% (s.upper(),fn.time2str(),m)) self.log_objs[severity](msg) except: pass if self.dialog(): if msg != '+1': msg = '%s:%s: %s' % (severity.upper(), fn.time2str(), msg) if self.filters: msg = (fn.filtersmart(msg, self.filters) or [''])[0] if msg: if len(self.instances()) > 1: msg = self.tango_host + ':' + msg self.dialog().append(msg)
def check_attributes(self,attrs = '', load = False, t0 = 0): db,t0,result,vals = self,t0 or fn.now(),{},{} print('Checking %s' % str(db)) if fn.isDictionary(attrs): attrs,vals = attrs.keys(),attrs if isinstance(vals.values()[0],dict): vals = dict((k,v.values()[0]) for k,v in vals.items()) else: if fn.isString(attrs): attrs = fn.filtersmart(db.get_attributes(),attrs) load = True if load: [vals.update(db.load_last_values(a)) for a in attrs] print('\t%d attributes'%len(attrs)) result['attrs'] = attrs result['vals'] = vals result['novals'] = [a for a,v in vals.items() if not v] result['nones'],result['down'],result['lost'] = [],[],[] for a,v in vals.items(): if not v or [1] is None: if not fn.read_attribute(a): #USE read not check!! result['down'].append(a) else: result['novals' if not v else 'nones'].append(a) elif v[0] < (t0 - 7200): result['lost'].append(a) print('\t%d attributes have no values'%len(result['novals'])) print('\t%d attributes are not readable'%len(result['down'])) print('\t%d attributes are not updated'%len(result['lost'])) print('\t%d attributes have None values'%len(result['nones'])) return result
def check_attributes(self, attrs='', load=False, t0=0): db, t0, result, vals = self, t0 or fn.now(), {}, {} print('Checking %s' % str(db)) if fn.isDictionary(attrs): attrs, vals = attrs.keys(), attrs if isinstance(vals.values()[0], dict): vals = dict((k, v.values()[0]) for k, v in vals.items()) else: if fn.isString(attrs): attrs = fn.filtersmart(db.get_attributes(), attrs) load = True if load: [vals.update(db.load_last_values(a)) for a in attrs] print('\t%d attributes' % len(attrs)) result['attrs'] = attrs result['vals'] = vals result['novals'] = [a for a, v in vals.items() if not v] result['nones'], result['down'], result['lost'] = [], [], [] for a, v in vals.items(): if not v or [1] is None: if not fn.read_attribute(a): #USE read not check!! result['down'].append(a) else: result['novals' if not v else 'nones'].append(a) elif v[0] < (t0 - 7200): result['lost'].append(a) print('\t%d attributes have no values' % len(result['novals'])) print('\t%d attributes are not readable' % len(result['down'])) print('\t%d attributes are not updated' % len(result['lost'])) print('\t%d attributes have None values' % len(result['nones'])) return result
def get_attributes_errors(self, regexp='*', timeout=3 * 3600, from_db=False, extend=False): """ Returns a dictionary {attribute, error/last value} If from_db=True and extend=True, it performs a full attribute check """ if regexp == '*': self.status = fn.defaultdict(list) if from_db or extend: timeout = fn.now() - timeout attrs = self.get_attributes(True) attrs = fn.filtersmart(attrs, regexp) print('get_attributes_errors([%d/%d])' % (len(attrs), len(self.attributes))) vals = self.load_last_values(attrs) for a, v in vals.items(): if v and v[0] > timeout: self.status['Updated'].append(a) if v[1] is not None: self.status['Readable'].append(a) else: rv = fn.read_attribute(a) if rv is not None: self.status['WrongNone'].append(a) else: self.status['None'].append(a) vals.pop(a) if not extend: self.status['NotUpdated'] = vals.keys() else: for a, v in vals.items(): c = fn.check_attribute(a) if c is None: vals[a] = 'Unreadable' self.status['Unreadable'].append(a) elif isinstance(c, Exception): vals[a] = str(c) self.status['Exception'].append(a) else: ev = fn.tango.check_attribute_events(a) if not ev: vals[a] = 'NoEvents' self.status['NoEvents'].append(a) else: d = self.get_attribute_archiver(a) e = self.get_archiver_errors(d) if a in e: vals[a] = e[a] self.status['ArchiverError'].append(a) else: rv = fn.read_attribute(a) if v and str(rv) == str(v[1]): vals[a] = 'NotChanged' self.status['NotChanged'].append(a) else: self.status['NotUpdated'].append(a) if regexp == '*': for k, v in self.status.items(): print('%s: %s' % (k, len(v))) return vals else: # Should inspect the Subscribers Error Lists vals = dict() for d in self.get_archivers(): err = self.get_archiver_errors(d) for a, e in err.items(): if fn.clmatch(regexp, a): vals[a] = e return vals
def get_attributes_errors(self, regexp='*', timeout=3*3600, from_db=False, extend = False): """ Returns a dictionary {attribute, error/last value} If from_db=True and extend=True, it performs a full attribute check """ if regexp == '*': self.status = fn.defaultdict(list) if from_db or extend: timeout = fn.now()-timeout attrs = self.get_attributes(True) attrs = fn.filtersmart(attrs,regexp) print('get_attributes_errors([%d/%d])' % (len(attrs),len(self.attributes))) vals = self.load_last_values(attrs) for a,v in vals.items(): if v and v[0] > timeout: self.status['Updated'].append(a) if v[1] is not None: self.status['Readable'].append(a) else: rv = fn.read_attribute(a) if rv is not None: self.status['WrongNone'].append(a) else: self.status['None'].append(a) vals.pop(a) if not extend: self.status['NotUpdated'] = vals.keys() else: for a,v in vals.items(): c = fn.check_attribute(a) if c is None: vals[a] = 'Unreadable' self.status['Unreadable'].append(a) elif isinstance(c,Exception): vals[a] = str(c) self.status['Exception'].append(a) else: ev = fn.tango.check_attribute_events(a) if not ev: vals[a] = 'NoEvents' self.status['NoEvents'].append(a) else: d = self.get_attribute_archiver(a) e = self.get_archiver_errors(d) if a in e: vals[a] = e[a] self.status['ArchiverError'].append(a) else: rv = fn.read_attribute(a) if v and str(rv) == str(v[1]): vals[a] = 'NotChanged' self.status['NotChanged'].append(a) else: self.status['NotUpdated'].append(a) if regexp == '*': for k,v in self.status.items(): print('%s: %s' % (k,len(v))) return vals else: # Should inspect the Subscribers Error Lists vals = dict() for d in self.get_archivers(): err = self.get_archiver_errors(d) for a,e in err.items(): if fn.clmatch(regexp,a): vals[a] = e return vals
def get_module_callables(self,module=None): r = TestModule.get_module_callables(self,module) fs = self.include + ['!%s'%e for e in self.exclude] r = fn.filtersmart(r,fs) return r
def showFile(self): mask = str(self.mask2.text()) data = fn.filtersmart(self.rawdata.split('\n'),mask) self.data.setText('\n'.join(data))
def showFile(self): mask = str(self.mask2.text()) data = fn.filtersmart(self.rawdata.split('\n'), mask) self.data.setText('\n'.join(data))
def get_module_callables(self, module=None): r = TestModule.get_module_callables(self, module) fs = self.include + ['!%s' % e for e in self.exclude] r = fn.filtersmart(r, fs) return r
def get_submodules(self, module=None, nr=0): r = TestModule.get_submodules(self, module, nr) fs = self.include + ['!%s' % e for e in self.exclude] r = fn.filtersmart(r, fs) return r
def get_submodules(self,module=None,nr=0): r = TestModule.get_submodules(self,module,nr) fs = self.include + ['!%s'%e for e in self.exclude] r = fn.filtersmart(r,fs) return r