def log(self,severity,msg,remote=None):
"""
The remote flag should allow operations done outside of QMainThread to be logged
"""
remote = remote if remote is not None else self.remote
if remote:
self.emit(Qt.SIGNAL('logging'),severity,msg,False)
return
if msg == self.last_msg:
msg = '+1'
else:
self.last_msg = msg
if self.logger:
try:
if severity not in self.log_objs: self.log_objs[severity] = \
getattr(self.logger,severity,(lambda m,s=severity:'%s:%s: %s'%(s.upper(),F.time2str(),m)))
self.log_objs[severity](msg)
except: pass
if self.dialog():
if msg!='+1':
msg = '%s:%s: %s'%(severity.upper(),F.time2str(),msg)
if self.filters:
msg = (F.filtersmart(msg,self.filters) or [''])[0]
if msg:
self.dialog().append(msg)
def start_devices(self,regexp = '*', force = False,
do_init = False, do_restart = False):
#devs = fn.tango.get_class_devices('HdbEventSubscriber')
devs = self.get_archivers()
if regexp:
devs = fn.filtersmart(devs,regexp)
off = sorted(set(d for d in devs if not fn.check_device(d)))
if off and do_restart:
print('Restarting %s Archiving Servers ...'%self.db_name)
astor = fn.Astor()
astor.load_from_devs_list(list(off))
astor.stop_servers()
fn.wait(3.)
astor.start_servers()
fn.wait(3.)
for d in devs:
try:
dp = fn.get_device(d, keep=True)
if do_init:
dp.init()
if force or dp.attributenumber != dp.attributestartednumber:
off.append(d)
print('%s.Start()' % d)
dp.start()
except Exception,e:
self.warning('start_archivers(%s) failed: %s' % (d,e))
def log(self, severity, msg, remote=None):
"""
The remote flag should allow operations done outside of QMainThread to be logged
"""
remote = remote if remote is not None else self.remote
if remote:
self.emit(Qt.SIGNAL('logging'), severity, msg, False)
return
if msg == self.last_msg:
msg = '+1'
else:
self.last_msg = msg
if self.logger:
try:
if severity not in self.log_objs: self.log_objs[severity] = \
getattr(self.logger,severity,(lambda m,s=severity:'%s:%s: %s'%(s.upper(),F.time2str(),m)))
self.log_objs[severity](msg)
except:
pass
if self.dialog():
if msg != '+1':
msg = '%s:%s: %s' % (severity.upper(), F.time2str(), msg)
if self.filters:
msg = (F.filtersmart(msg, self.filters) or [''])[0]
if msg:
self.dialog().append(msg)
def start_devices(self,
regexp='*',
force=False,
do_init=False,
do_restart=False):
#devs = fn.tango.get_class_devices('HdbEventSubscriber')
devs = self.get_archivers()
if regexp:
devs = fn.filtersmart(devs, regexp)
off = sorted(set(d for d in devs if not fn.check_device(d)))
if off and do_restart:
print('Restarting %s Archiving Servers ...' % self.db_name)
astor = fn.Astor()
astor.load_from_devs_list(list(off))
astor.stop_servers()
fn.wait(3.)
astor.start_servers()
fn.wait(3.)
for d in devs:
try:
dp = fn.get_device(d)
if do_init:
dp.init()
if force or dp.attributenumber != dp.attributestartednumber:
off.append(d)
print('%s.Start()' % d)
dp.start()
except Exception, e:
self.warning('start_archivers(%s) failed: %s' % (d, e))
def get_attributes_failed(self, regexp='*', timeout=3600, from_db=True):
if from_db:
timeout = fn.now() - timeout
attrs = self.get_attributes(True)
attrs = fn.filtersmart(attrs, regexp)
print('get_attributes_failed([%d])' % len(attrs))
print(attrs)
vals = self.load_last_values(attrs)
return sorted(t for t in vals if not t[1] or t[1][0] < timeout)
else:
# Should inspect the Subscribers Error Lists
raise Exception('NotImplemented')
def __init__(self, mask=None):
#mask = mask or 'FolderDS/*'
self.mask = mask
ProxiesDict.__init__(self)
devs = tango.get_class_devices('FolderDS')
extra = fn.get_database().get_class_property('FolderDS',
['ExtraDevices'])
devs.extend(extra.get('ExtraDevices', []))
if mask: devs = fn.filtersmart(devs, mask)
self.hosts = fn.defaultdict(list)
for d in devs:
self[d]
def get_attributes(self, active=None, regexp=''):
"""
Alias for Reader API
"""
if active:
r = self.get_archived_attributes()
else:
# Inactive attributes must be read from Database
r = self.get_attribute_names(False)
r = sorted(set(
fn.tango.get_full_name(a, fqdn=True).lower() for a in r))
return sorted(fn.filtersmart(r, regexp) if regexp else r)
def get_archivers_for_attributes(attrs=[], archs='archiving/es/*'):
"""
This method returns matching archivers for a list of attributes
in simplename format (no tango host).
It applies AttributeFilters as defined in Tango DB (sorted)
"""
if isString(attrs):
attrs = ft.find_attributes(attrs)
else:
attrs = attrs or get_schema_attributes('*')
devattrs = fn.dicts.defaultdict(set)
[devattrs[a.rsplit('/', 1)[0]].add(a) for a in attrs]
if isSequence(archs): archs = '(%s)' % ')|('.join(archs)
filters = get_archivers_filters(archs)
r = devattrs.keys()
archattrs = {}
for i, k in enumerate(filters):
v = filters[k] # k is the archiver name
k = fn.tango.parse_tango_model(k, fqdn=True).fullname
if 'DEFAULT' in v:
df = k
else:
#filtersmart(list,regexp): returns a clsearch on the list
m = fn.filtersmart(r, v)
currattrs = set(fn.join(*[devattrs[d] for d in m]))
if len(currattrs):
print(k, len(currattrs),
sorted(set(i.split('/')[-2] for i in m)))
archattrs[k] = currattrs
print('\n')
r = [a for a in r if a not in m]
if i == len(filters) - 1:
k = df
m = r
currattrs = fn.join(*[devattrs[d] for d in m])
if len(currattrs):
print(k, len(currattrs),
sorted(set(i.split('/')[-2] for i in m)))
archattrs[k] = currattrs
return archattrs
def get_attribute_names(self, active=False, regexp=''):
t0 = fn.now()
if not active:
attributes = [
a[0].lower()
for a in self.Query('select att_name from att_conf')
]
[
self.get_attr_id_type_table(a) for a in attributes
if a not in self.attributes
]
r = self.attributes.keys()
else:
r = self.get_archived_attributes()
r = sorted(fn.filtersmart(r, regexp) if regexp else r)
self.debug('get attribute names took %d ms' % (1e3 * (fn.now() - t0)))
return r
def get_archivers_for_attributes(attrs=[],archs='archiving/es/*'):
"""
This method returns matching archivers for a list of attributes
in simplename format (no tango host).
It applies AttributeFilters as defined in Tango DB (sorted)
"""
if isString(attrs):
attrs = ft.find_attributes(attrs)
else:
attrs = attrs or get_schema_attributes('*')
devattrs = fn.dicts.defaultdict(set)
[devattrs[a.rsplit('/',1)[0]].add(a) for a in attrs];
if isSequence(archs): archs = '(%s)'%')|('.join(archs)
filters = get_archivers_filters(archs)
r = devattrs.keys()
archattrs = {}
for i,k in enumerate(filters):
v = filters[k]
k = fn.tango.parse_tango_model(k, fqdn = True).fullname
if 'DEFAULT' in v:
df = k
else:
m = fn.filtersmart(r,v)
currattrs = set(fn.join(*[devattrs[d] for d in m]))
if len(currattrs):
print(k,len(currattrs),sorted(set(i.split('/')[-2] for i in m)))
archattrs[k] = currattrs
print('\n')
r = [a for a in r if a not in m]
if i == len(filters)-1:
k = df
m = r
currattrs = fn.join(*[devattrs[d] for d in m])
if len(currattrs):
print(k,len(currattrs),sorted(set(i.split('/')[-2] for i in m)))
archattrs[k] = currattrs
return archattrs
def log(self,severity,msg):
if msg == self.last_msg:
msg = '+1'
else:
self.last_msg = msg
if self.logger:
try:
if severity not in self.log_objs: self.log_objs[severity] = \
getattr(self.logger,severity.lower(),
lambda m,s=severity:'%s:%s: %s'%
(s.upper(),fn.time2str(),m))
self.log_objs[severity](msg)
except: pass
if self.dialog():
if msg!='+1':
msg = '%s:%s: %s'%(severity.upper(),fn.time2str(),msg)
if self.filters:
msg = (fn.filtersmart(msg,self.filters) or [''])[0]
if msg:
if len(self.instances())>1: msg = self.tango_host+':'+msg
self.dialog().append(msg)
def log(self, severity, msg):
if msg == self.last_msg:
msg = '+1'
else:
self.last_msg = msg
if self.logger:
try:
if severity not in self.log_objs: self.log_objs[severity] = \
getattr(self.logger,severity.lower(),
lambda m,s=severity:'%s:%s: %s'%
(s.upper(),fn.time2str(),m))
self.log_objs[severity](msg)
except:
pass
if self.dialog():
if msg != '+1':
msg = '%s:%s: %s' % (severity.upper(), fn.time2str(), msg)
if self.filters:
msg = (fn.filtersmart(msg, self.filters) or [''])[0]
if msg:
if len(self.instances()) > 1: msg = self.tango_host + ':' + msg
self.dialog().append(msg)
def check_attributes(self,attrs = '', load = False, t0 = 0):
db,t0,result,vals = self,t0 or fn.now(),{},{}
print('Checking %s' % str(db))
if fn.isDictionary(attrs):
attrs,vals = attrs.keys(),attrs
if isinstance(vals.values()[0],dict):
vals = dict((k,v.values()[0]) for k,v in vals.items())
else:
if fn.isString(attrs):
attrs = fn.filtersmart(db.get_attributes(),attrs)
load = True
if load:
[vals.update(db.load_last_values(a)) for a in attrs]
print('\t%d attributes'%len(attrs))
result['attrs'] = attrs
result['vals'] = vals
result['novals'] = [a for a,v in vals.items() if not v]
result['nones'],result['down'],result['lost'] = [],[],[]
for a,v in vals.items():
if not v or [1] is None:
if not fn.read_attribute(a): #USE read not check!!
result['down'].append(a)
else:
result['novals' if not v else 'nones'].append(a)
elif v[0] < (t0 - 7200):
result['lost'].append(a)
print('\t%d attributes have no values'%len(result['novals']))
print('\t%d attributes are not readable'%len(result['down']))
print('\t%d attributes are not updated'%len(result['lost']))
print('\t%d attributes have None values'%len(result['nones']))
return result
def check_attributes(self, attrs='', load=False, t0=0):
db, t0, result, vals = self, t0 or fn.now(), {}, {}
print('Checking %s' % str(db))
if fn.isDictionary(attrs):
attrs, vals = attrs.keys(), attrs
if isinstance(vals.values()[0], dict):
vals = dict((k, v.values()[0]) for k, v in vals.items())
else:
if fn.isString(attrs):
attrs = fn.filtersmart(db.get_attributes(), attrs)
load = True
if load:
[vals.update(db.load_last_values(a)) for a in attrs]
print('\t%d attributes' % len(attrs))
result['attrs'] = attrs
result['vals'] = vals
result['novals'] = [a for a, v in vals.items() if not v]
result['nones'], result['down'], result['lost'] = [], [], []
for a, v in vals.items():
if not v or [1] is None:
if not fn.read_attribute(a): #USE read not check!!
result['down'].append(a)
else:
result['novals' if not v else 'nones'].append(a)
elif v[0] < (t0 - 7200):
result['lost'].append(a)
print('\t%d attributes have no values' % len(result['novals']))
print('\t%d attributes are not readable' % len(result['down']))
print('\t%d attributes are not updated' % len(result['lost']))
print('\t%d attributes have None values' % len(result['nones']))
return result
def get_attributes_errors(self,
regexp='*',
timeout=3 * 3600,
from_db=False,
extend=False):
"""
Returns a dictionary {attribute, error/last value}
If from_db=True and extend=True, it performs a full attribute check
"""
if regexp == '*':
self.status = fn.defaultdict(list)
if from_db or extend:
timeout = fn.now() - timeout
attrs = self.get_attributes(True)
attrs = fn.filtersmart(attrs, regexp)
print('get_attributes_errors([%d/%d])' %
(len(attrs), len(self.attributes)))
vals = self.load_last_values(attrs)
for a, v in vals.items():
if v and v[0] > timeout:
self.status['Updated'].append(a)
if v[1] is not None:
self.status['Readable'].append(a)
else:
rv = fn.read_attribute(a)
if rv is not None:
self.status['WrongNone'].append(a)
else:
self.status['None'].append(a)
vals.pop(a)
if not extend:
self.status['NotUpdated'] = vals.keys()
else:
for a, v in vals.items():
c = fn.check_attribute(a)
if c is None:
vals[a] = 'Unreadable'
self.status['Unreadable'].append(a)
elif isinstance(c, Exception):
vals[a] = str(c)
self.status['Exception'].append(a)
else:
ev = fn.tango.check_attribute_events(a)
if not ev:
vals[a] = 'NoEvents'
self.status['NoEvents'].append(a)
else:
d = self.get_attribute_archiver(a)
e = self.get_archiver_errors(d)
if a in e:
vals[a] = e[a]
self.status['ArchiverError'].append(a)
else:
rv = fn.read_attribute(a)
if v and str(rv) == str(v[1]):
vals[a] = 'NotChanged'
self.status['NotChanged'].append(a)
else:
self.status['NotUpdated'].append(a)
if regexp == '*':
for k, v in self.status.items():
print('%s: %s' % (k, len(v)))
return vals
else:
# Should inspect the Subscribers Error Lists
vals = dict()
for d in self.get_archivers():
err = self.get_archiver_errors(d)
for a, e in err.items():
if fn.clmatch(regexp, a):
vals[a] = e
return vals
def get_attributes_errors(self, regexp='*', timeout=3*3600,
from_db=False, extend = False):
"""
Returns a dictionary {attribute, error/last value}
If from_db=True and extend=True, it performs a full attribute check
"""
if regexp == '*':
self.status = fn.defaultdict(list)
if from_db or extend:
timeout = fn.now()-timeout
attrs = self.get_attributes(True)
attrs = fn.filtersmart(attrs,regexp)
print('get_attributes_errors([%d/%d])'
% (len(attrs),len(self.attributes)))
vals = self.load_last_values(attrs)
for a,v in vals.items():
if v and v[0] > timeout:
self.status['Updated'].append(a)
if v[1] is not None:
self.status['Readable'].append(a)
else:
rv = fn.read_attribute(a)
if rv is not None:
self.status['WrongNone'].append(a)
else:
self.status['None'].append(a)
vals.pop(a)
if not extend:
self.status['NotUpdated'] = vals.keys()
else:
for a,v in vals.items():
c = fn.check_attribute(a)
if c is None:
vals[a] = 'Unreadable'
self.status['Unreadable'].append(a)
elif isinstance(c,Exception):
vals[a] = str(c)
self.status['Exception'].append(a)
else:
ev = fn.tango.check_attribute_events(a)
if not ev:
vals[a] = 'NoEvents'
self.status['NoEvents'].append(a)
else:
d = self.get_attribute_archiver(a)
e = self.get_archiver_errors(d)
if a in e:
vals[a] = e[a]
self.status['ArchiverError'].append(a)
else:
rv = fn.read_attribute(a)
if v and str(rv) == str(v[1]):
vals[a] = 'NotChanged'
self.status['NotChanged'].append(a)
else:
self.status['NotUpdated'].append(a)
if regexp == '*':
for k,v in self.status.items():
print('%s: %s' % (k,len(v)))
return vals
else:
# Should inspect the Subscribers Error Lists
vals = dict()
for d in self.get_archivers():
err = self.get_archiver_errors(d)
for a,e in err.items():
if fn.clmatch(regexp,a):
vals[a] = e
return vals
def get_module_callables(self,module=None): r = TestModule.get_module_callables(self,module) fs = self.include + ['!%s'%e for e in self.exclude] r = fn.filtersmart(r,fs) return r
def showFile(self):
mask = str(self.mask2.text())
data = fn.filtersmart(self.rawdata.split('\n'),mask)
self.data.setText('\n'.join(data))
def showFile(self):
mask = str(self.mask2.text())
data = fn.filtersmart(self.rawdata.split('\n'), mask)
self.data.setText('\n'.join(data))
def get_module_callables(self, module=None):
r = TestModule.get_module_callables(self, module)
fs = self.include + ['!%s' % e for e in self.exclude]
r = fn.filtersmart(r, fs)
return r
def get_submodules(self, module=None, nr=0):
r = TestModule.get_submodules(self, module, nr)
fs = self.include + ['!%s' % e for e in self.exclude]
r = fn.filtersmart(r, fs)
return r
def get_submodules(self,module=None,nr=0): r = TestModule.get_submodules(self,module,nr) fs = self.include + ['!%s'%e for e in self.exclude] r = fn.filtersmart(r,fs) return r
