def get_only_scalar_attributes(model, exclude_strings=True):
"""
This method will filter out all attributes that are either Arrays or Strings.
Model can be a device name, a regexp expression or a list of attributes.
"""
if fun.isSequence(model):
attrs = model
elif fun.isRegexp(model):
attrs = fandango.get_matching_attributes(model)
else:
attrs = fandango.get_matching_attributes(model + '/*')
exclude = [list] + ([str] if exclude_strings else [])
return [a for a in attrs if get_attribute_pytype(a) not in exclude]
def get_only_scalar_attributes(model,exclude_strings=True):
"""
This method will filter out all attributes that are either Arrays or Strings.
Model can be a device name, a regexp expression or a list of attributes.
"""
if fun.isSequence(model):
attrs = model
elif fun.isRegexp(model):
attrs = fandango.get_matching_attributes(model)
else:
attrs = fandango.get_matching_attributes(model+'/*')
exclude = [list] + ([str] if exclude_strings else [])
return [a for a in attrs if get_attribute_pytype(a) not in exclude]
def apply_search(self):
signs = "==|=|<|>"
txt = str(self.search.text() or '*')
if fandango.clsearch(signs, txt):
t = fandango.re.split(signs, txt, 1)[0]
txt, formula = t, txt.replace(t, '')
else:
formula = ''
txt = txt.replace(' ', '*').strip()
txt = self.preffix + txt + self.suffix
model = fandango.get_matching_attributes(txt)
if self.labels and '/' in txt:
dev, attr = txt.rsplit('/', 1)
model.extend(
fandango.tango.get_matching_device_attribute_labels(
dev, attr).keys())
if formula and model:
nm = []
for m in model:
try:
f = '%s %s' % (fandango.read_attribute(m), formula)
if eval(f):
nm.append(m)
except:
pass
model = nm
self.form.setModel(model)
def check_archiving_performance(schema='hdb',attributes=[],period=24*3600*90,\
exclude=['*/waveid','*/wavename','*/elotech-*'],action=False,trace=True):
import PyTangoArchiving as pta
import fandango as fn
ti = fn.now()
api = pta.api(schema)
check = dict()
period = 24*3600*period if period < 1000 else (24*period if period<3600 else period)
attributes = fn.get_matching_attributes(attributes) if fn.isString(attributes) else map(str.lower,attributes)
tattrs = [a for a in api if not attributes or a in attributes]
excluded = [a for a in tattrs if any(fn.clmatch(e,a) for e in exclude)]
tattrs = [a for a in tattrs if a not in excluded]
#Getting Tango devices currently not running
alldevs = set(t.rsplit('/',1)[0] for t in tattrs if api[t].archiver)
tdevs = filter(fn.check_device,alldevs)
nodevs = [d for d in alldevs if d not in tdevs]
#Updating data from archiving config tables
if not attributes:
tattrs = sorted(a for a in api if a.rsplit('/',1)[0] in tdevs)
tattrs = [a for a in tattrs if not any(fn.clmatch(e,a) for e in exclude)]
print('%d attributes will not be checked (excluded or device not running)'%(len(api)-len(tattrs)))
tarch = sorted(a for a in api if api[a].archiver)
tnoread = sorted(t for t in tarch if t not in tattrs)
check.update((t,None) for t in tnoread)
#Getting attributes archived in the past and not currently active
tmiss = [t for t in tattrs if not api[t].archiver]
check.update((t,fn.check_attribute(t,readable=True)) for t in tmiss)
tmiss = [t for t in tmiss if check[t]]
tmarray = [t for t in tmiss if fn.isString(check[t].value) or fn.isSequence(check[t].value)]
tmscalar = [t for t in tmiss if t not in tmarray]
#Getting updated tables from database
tups = pta.utils.get_table_updates(schema)
# Some tables do not update MySQL index tables
t0 = [a for a in tarch if a in tattrs and not tups[api[a].table]]
check.update((t,check_attribute(a,readable=True)) for t in t0 if not check.get(t))
t0 = [t for t in t0 if check[t]]
print('%d/%d archived attributes have indexes not updated ...'%(len(t0),len(tarch)))
if t0 and len(t0)<100:
vs = api.load_last_values(t0);
tups.update((api[t].table,api[t].last_date) for t in t0)
tnotup = [a for a in tarch if tups[api[a].table]<fn.now()-1800]
check.update((t,1) for t in tarch if t not in tnotup)
#Updating readable attributes (all updated are considered as readable)
tread = sorted(t for t in tattrs if t not in tnoread)
for t in tattrs:
if t not in check:
check[t] = fn.check_attribute(t,readable=True)
tread = sorted(t for t in tattrs if check[t])
tnoread.extend(t for t in tread if not check[t])
tnoread = sorted(set(tnoread))
#tread contains all readable attributes from devices with some attribute archived
#tnoread contains all unreadable attributes from already archived
#Calcullating all final stats
#tok will be all archivable attributes that are archived
#tnotup = [a for a in tnotup if check[a]]
#tok = [t for t in tread if t in tarch and t not in tnotup]
tok = [t for t in tarch if t not in tnotup]
readarch = [a for a in tread if a in tarch]
treadnotup = [t for t in readarch if t in tnotup] #tnotup contains only data from tarch
tokread = [t for t in readarch if t not in tnotup] #Useless, all archived are considered readable
tarray = [t for t in tarch if check[t] and get_attribute_pytype(t) in (str,list)]
removed = [a for a in tattrs if not api[a].archiver and tups[api[a].table]>fn.now()-period]
result = fn.Struct()
result.Excluded = excluded
result.Schema = schema
result.All = api.keys()
result.Archived = tarch
result.Readable = tread
result.ArchivedAndReadable = readarch
result.Updated = tok #tokread
result.Lost = treadnotup
result.Removed = removed
result.TableUpdates = tups
result.NotUpdated = tnotup
result.Missing = tmiss
result.MissingScalars = tmscalar
result.MissingArrays = tmarray
result.ArchivedArray = tarray
result.Unreadable = tnoread
result.DeviceNotRunning = nodevs
get_ratio = lambda a,b:float(len(a))/float(len(b))
result.ArchRatio = get_ratio([t for t in readarch if t not in tnotup],readarch)
result.ReadRatio = get_ratio(result.Readable,tattrs)
result.LostRatio = get_ratio([a for a in tread if a in tnotup],tread)
result.MissRatio = get_ratio([a for a in tread if a not in tarch],tread)
result.OkRatio = 1.0-result.LostRatio-result.MissRatio
result.Summary = '\n'.join((
('Checking archiving of %s attributes'%(len(attributes) if attributes else schema))
,('%d attributes in %s, %d are currently active'%(len(api),schema,len(tarch)))
,('%d devices with %d archived attributes are not running'%(len(nodevs),len([a for a in api if a.rsplit('/',1) in nodevs])))
,('%d archived attributes (%2.1f %%) are unreadable! (check and remove)'%(len(tnoread),1e2*get_ratio(tnoread,tarch)))
,('%d readable attributes are not archived'%(len(tmiss)))
,('%d attributes (readable or not) are updated (%2.1f %% of all readables)'%(len(tok),1e2*result.OkRatio))
,('-'*80)
,('%d archived attributes (readable or not) are not updated!'%len(tnotup))
,('%d archived and readable attributes are not updated! (check and restart?)'%len(treadnotup))
,('-'*80)
,('%d readable attributes have been removed in the last %d days!'%(len(removed),period/(24*3600)))
,('%d readable scalar attributes are not being archived (not needed anymore?)'%len(tmscalar))
,('%d readable array attributes are not being archived (Ok)'%len(tmarray))
,('%d readable array attributes are archived (Expensive)'%len(tarray))
,('')))
if trace: print(result.Summary)
print('%d readable lost,Ok = %2.1f%%, %2.1f %% over all Readables (%2.1f %% of total)'%\
(len(treadnotup),1e2*result.ArchRatio,1e2*result.OkRatio,1e2*result.ReadRatio))
if action:
print('NO ACTIONS ARE GONNA BE EXECUTED, AS THESE ARE ONLY RECOMMENDATIONS')
print("""
api = PyTangoArchiving.ArchivingAPI('%s')
lostdevs = sorted(set(api[a].archiver for a in result.NotUpdated))
print(lostdevs)
if lostdevs < a_reasonable_number:
astor = fn.Astor()
astor.load_from_devs_list(lostdevs)
astor.stop_servers()
fn.time.sleep(10.)
astor.start_servers()
"""%schema)
if trace: print('finished in %d seconds'%(fn.now()-ti))
return result
import fandango as fn, fandango.callbacks as fc
attrs = map(str.lower, fn.get_matching_attributes('bl00*eps*plc*01/*'))
a = 'bl00/ct/eps-plc-01/State'
t0 = fn.now()
sources = dict(
(d, fc.EventSource(d, asynchronous=True)) for d in sorted(attrs))
#sources.values()[10].read(synch=True)
cache = fn.CaselessDict()
def hook(src, t, value):
cache[src.normal_name] = (value)
el = fc.EventListener('A Queue for All')
el.set_value_hook(hook)
print('Subscribing %d attributes' % len(attrs))
[s.addListener(el) for k, s in sorted(sources.items())]
print('Subscription took %f seconds' % (fn.now() - t0))
t0 = fn.now()
print('Waiting ...')
while len(cache) < len(attrs):
if fn.now() > t0 + 18.:
break
fn.wait(1.)