def trigger_callbacks(self, regs=None):
"""
regs = list of addresses that changed
"""
if not self.callbacks:
return
for key, cb in self.callbacks.items():
try:
rs, push, org = regs, False, cb
if fn.isSequence(cb):
rs = regs and [r for r in regs if r in cb]
if not regs or rs or (len(cb) == 1
and fn.isCallable(cb[0])):
cb = cb[-1]
else:
continue
msg = ('%s: %s.trigger_callbacks(%s,%s): %s:%s' %
(fn.time2str(), self.name, fn.shortstr(
regs, 40), rs, key, org))
if self.plc_obj is not None:
self.plc_obj.debug(msg)
else:
print(msg)
if fn.isCallable(cb):
cb(key) #,push=push)
else:
cb = getattr(cb, 'push_event',
getattr(cb, 'event_received', None))
cb and cb(key)
fn.wait(1.e-4)
except Exception as e:
print(fn.except2str())
print('%s.callback(%s,%s): %s' % (self, key, cb, e))
def start_devices(self,regexp = '*', force = False,
do_init = False, do_restart = False):
#devs = fn.tango.get_class_devices('HdbEventSubscriber')
devs = self.get_archivers()
if regexp:
devs = fn.filtersmart(devs,regexp)
off = sorted(set(d for d in devs if not fn.check_device(d)))
if off and do_restart:
print('Restarting %s Archiving Servers ...'%self.db_name)
astor = fn.Astor()
astor.load_from_devs_list(list(off))
astor.stop_servers()
fn.wait(3.)
astor.start_servers()
fn.wait(3.)
for d in devs:
try:
dp = fn.get_device(d, keep=True)
if do_init:
dp.init()
if force or dp.attributenumber != dp.attributestartednumber:
off.append(d)
print('%s.Start()' % d)
dp.start()
except Exception,e:
self.warning('start_archivers(%s) failed: %s' % (d,e))
def __test__(*args):
t0 = fd.now()
args = args or ['*value*', '20']
opts = dict.fromkeys(a.strip('-') for a in args if a.startswith('-'))
args = [a for a in args if a not in opts]
scope = args[0]
tlimit = int((args[1:] or ['20'])[0])
if opts:
opts = dict(o.split('=') if '=' in o else (o, True) for o in opts)
opts.update((o, fd.str2type(v)) for o, v in opts.items())
print('AlarmView(Test,\t'
'\tscope=%s,\n\ttlimit=%s,\n\t**%s)\n' % (scope, tlimit, opts))
if opts.get('d', False):
th = TangoAttribute.get_thread()
th.set_period_ms(500)
th.setLogLevel('DEBUG')
verbose = opts.get('v', 2)
view = AlarmView('Test', scope=scope, verbose=verbose, **opts)
print('\n'.join('>' * 80 for i in range(4)))
cols = 'sortkey', 'tag', 'state', 'active', 'time', 'severity'
while fd.now() < (t0 + tlimit):
fd.wait(3.)
print('\n' + '<' * 80)
l = view.sort(as_text={'cols': cols})
print('\n'.join(l))
print('AlarmView.__test__(%s) finished after %d seconds' %
(args[0], fd.now() - t0))
def run():
### LAUNCH DEVICES
dspath = path.join(path.dirname(path.abspath(panic.__file__)),
'ds','PyAlarm.py')
fn.log.info('MAIL_TEST: RUN '+'>'*40)
os.system(dspath+' test_mail -v2 &')
fn.wait(30.)
def trigger_callbacks(self, regs=None):
"""
regs = list of addresses that changed
"""
if not self.callbacks:
return
for key, cb in self.callbacks.items():
try:
push = False
if fun.isSequence(cb):
if not regs or any(r in cb for r in regs):
#print('%s: %s.trigger(%s): callback(%s):%s' %
#(fun.time2str(),self,key,cb,regs))
cb, push = cb[-1], True
else:
continue
if fun.isCallable(cb):
cb(key, push=push)
else:
cb = getattr(cb, 'push_event',
getattr(cb, 'event_received', None))
cb and cb(key)
fandango.wait(1.e-4)
except Exception as e:
print(fandango.except2str())
print('%s.callback(%s): %s' % (self, cb, e))
def start_devices(self,
regexp='*',
force=False,
do_init=False,
do_restart=False):
#devs = fn.tango.get_class_devices('HdbEventSubscriber')
devs = self.get_archivers()
if regexp:
devs = fn.filtersmart(devs, regexp)
off = sorted(set(d for d in devs if not fn.check_device(d)))
if off and do_restart:
print('Restarting %s Archiving Servers ...' % self.db_name)
astor = fn.Astor()
astor.load_from_devs_list(list(off))
astor.stop_servers()
fn.wait(3.)
astor.start_servers()
fn.wait(3.)
for d in devs:
try:
dp = fn.get_device(d)
if do_init:
dp.init()
if force or dp.attributenumber != dp.attributestartednumber:
off.append(d)
print('%s.Start()' % d)
dp.start()
except Exception, e:
self.warning('start_archivers(%s) failed: %s' % (d, e))
def readComm(self, commCode, READ=True, emulation=False):
## A WAIT TIME HAS BEEN NECESSARY BEFORE READING THE BUFFER
# This wait is divided in smaller periods
# In each period is tested what has been received from the serial port
# The wait will finish when after receiving some information there's silence again
t0, result, retries = fandango.now(),'',0
if not hasattr(self,'_Dcache'):
self._Dcache = {}
if emulation and commCode in self._Dcache:
return self._Dcache[commCode]
wtime = 0.0; result = ""; rec = ""; lastrec = ""; div=4.;
before=time.time(); after=before+0.001
while wtime<self.waitTime and not (not len(rec) \
and len(lastrec.replace(commCode,'').replace('\r','').replace('\n',''))):
#if self.trace and retries:
# print('In readComm(%s)(%d) Waiting %fs for answer ...'
# %(commCode,retries,self.waitTime))
retries += 1
#The wait condition aborts after waitTime or nothing read after something different from \r or \n.
#Between reads a pause of TimeWait/10.0 is performed.
#I've tried to make smaller the time that the thread spends waiting for an answer ... more attempts to improve has been inefficient do to imprecission of the time.sleep method
last=before
after=time.time()
pause = self.waitTime/div - (after-before)
fandango.wait(max(pause,0)) #time.sleep(max(pause,0))
before=time.time();
lastrec=lastrec+rec
if self.getSerialClass() == 'PySerial':
nchars = self.dp.read_attribute('InputBuffer').value
rec = self.dp.command_inout("Read",nchars)
else: #Class is 'Serial'
#rec = self.dp.command_inout("DevSerReadRaw")
rec = self.dp.command_inout("DevSerReadString",0)
rec.rstrip().lstrip()
lrclean = lastrec.replace(commCode,'').replace('\r','').replace('\n','')
rrclean = rec.replace('\r','\\r').replace('\n','\\n')
if self.trace and rrclean:
#self.debug
print( 'received('+str(wtime)+';'+str(after-last)+';'
+str(len(lrclean))+';'+str(len(rec))+"): '" + rrclean+"'")
result += rec
wtime += self.waitTime/div
self._Dcache[commCode] = result
readtime = fandango.now()-t0
self.maxreadtime = max((self.maxreadtime,readtime))
if self.trace:
print('ReadComm(%s) = %s done in %f seconds (max = %f, + %f)' %
(commCode,result.strip(),readtime,self.maxreadtime,fandango.now()-self.lasttime))
return result
def run(self):
### LAUNCH DEVICES
dspath = path.join(path.dirname(path.abspath(panic.__file__)), 'ds',
'PyAlarm.py')
self.log('RUN ' + '>' * 40)
for s in self.servers:
os.system(dspath + ' ' + s + ' -v2 &')
fn.wait(30.)
def restart_servers(servers=[], host=''):
if not servers:
servers = get_servers_status()['restart']
astor = fandango.Astor()
astor.load_from_servers_list(servers)
astor.stop_servers()
print('waiting ...')
fandango.wait(10.)
for s in astor:
host = host or astor[s].host
print('Starting %s at %s' % (s, host))
astor.start_servers(s, host=host)
return
def delete_device(self):
# PROTECTED REGION ID(SchemaManager.delete_device) ENABLED START #
try:
print('-'*80)
print("[Device delete_device method] for %s"%self.get_name())
self.set_state(PyTango.DevState.INIT)
if self.threadDict and self.threadDict.alive():
self.threadDict.stop()
print('waiting ...')
#Waiting longer times does not avoid segfault (sigh)
fn.wait(3.)
except:
traceback.print_exc()
print('-'*80)
def delete_device(self):
# PROTECTED REGION ID(SchemaManager.delete_device) ENABLED START #
try:
print('-' * 80)
print("[Device delete_device method] for %s" % self.get_name())
self.set_state(PyTango.DevState.INIT)
if self.threadDict and self.threadDict.alive():
self.threadDict.stop()
print('waiting ...')
#Waiting longer times does not avoid segfault (sigh)
fn.wait(3.)
except:
traceback.print_exc()
print('-' * 80)
def restart_attribute(self, attr, d=''):
try:
d = self.get_attribute_archiver(attr)
print('%s.restart_attribute(%s)' % (d, attr))
dp = fn.get_device(d, keep=True)
if not fn.check_device(dp):
self.start_devices('(.*/)?' + d, do_restart=True)
dp.AttributeStop(attr)
fn.wait(.1)
dp.AttributeStart(attr)
except:
print('%s.AttributeStart(%s) failed!' % (d, attr))
def restart_attribute(self,attr, d=''):
try:
d = self.get_attribute_archiver(attr)
print('%s.restart_attribute(%s)' % (d,attr))
dp = fn.get_device(d, keep=True)
if not fn.check_device(dp):
self.start_devices('(.*/)?'+d,do_restart=True)
dp.AttributeStop(attr)
fn.wait(.1)
dp.AttributeStart(attr)
except:
print('%s.AttributeStart(%s) failed!'%(d,attr))
def stop_periodic_archiving(self, attribute):
try:
attribute = parse_tango_model(attribute,
fqdn=True).fullname.lower()
arch = self.get_periodic_attribute_archiver(attribute)
if not arch:
self.warning('%s is not archived!' % attribute)
else:
self.info('Removing %s from %s' % (attribute, arch))
dp = fn.get_device(archiver)
v = dp.AttributeRemove([attribute, str(int(float(period)))])
dp.UpdateAttributeList()
fn.wait(wait)
return v
except:
self.warning('stop_periodic_archiving(%s) failed!' %
(attribute, traceback.format_exc()))
def restart_periodic_archiving(self, attribute):
try:
attribute = parse_tango_model(attribute,
fqdn=True).fullname.lower()
arch = self.get_periodic_attribute_archiver(attribute)
if not arch:
self.warning('%s is not archived!' % attribute)
else:
self.info('Restarting %s at %s' % (attribute, arch))
dp = fn.get_device(archiver)
v = dp.AttributeStop(attribute)
dp.ResetErrorAttributes()
fn.wait(wait)
v = dp.AttributeStart(attribute)
return v
except:
self.warning('restart_periodic_archiving(%s) failed!' %
(attribute, traceback.format_exc()))
def GetAttDataBetweenDates(self, argin):
"""
Arguments to be AttrName, StartDate, StopDate, Synchronous
If Synchronous is missing or False, data is buffered into attributes, which names are returned
If True or Yes, all the data is returned when ready
Data returned will be (rows,[t0,v0,t1,v1,t2,v2,...])
"""
print time.ctime() + "In ", self.get_name(
), "::GetAttDataBetweenDates(%s)" % argin
# Add your own code here
size = 0
aname = argin[0]
tag = self.attr2tag(aname)
dates = self.dates2times(argin[1:3])
RW = False
synch = fn.searchCl('yes|true', str(argin[3:4]))
attrs = [tag, tag + '_r', tag + '_w', tag +
'_t'] if RW else [tag, tag + '_r', tag + '_w', tag + '_t']
self.reader.get_attribute_values(
aname, (lambda v: self.reader_hook(aname, v)),
dates[0],
dates[1],
decimate=True,
cache=self.UseApiCache)
self.counter += 1
print(self.counter)
argout = [fn.shape(attrs), [a for a in attrs]]
if not synch:
print '\t%s' % argout
return argout
else:
while not self.IsDataReady(aname):
fandango.wait(0.1)
data = self.AttrData[aname][-1]
for t, v in data:
argout.append(t)
argout.extend(fn.toSequence(v))
return [fn.shape(data), argout]
def restart_attribute(self, attr, d=''):
"""
execute AttributeStop/Start on subscriber device
"""
try:
a = self.is_attribute_archived(attr)
if not a:
raise Exception('%s is not archived!' % attr)
attr = a
d = self.get_attribute_archiver(attr)
print('%s.restart_attribute(%s)' % (d, attr))
dp = fn.get_device(d, keep=True)
if not fn.check_device(dp):
self.start_devices('(.*/)?' + d, do_restart=True)
dp.AttributeStop(attr)
fn.wait(10.)
dp.AttributeStart(attr)
except:
print('%s.AttributeStart(%s) failed!' % (d, attr))
def restart_attributes(self, attributes=None, timewait=0.5):
if attributes is None:
attributes = self.get_attributes_not_updated()
todo = []
for a in attributes:
a = self.is_attribute_archived(a)
if a:
todo.append(a)
else:
self.warning('%s is not archived!' % a)
devs = dict(fn.kmap(self.get_attribute_archiver, todo))
for a, d in fn.randomize(sorted(devs.items())):
if not fn.check_device(d):
self.start_devices('(.*/)?' + d, do_restart=True)
else:
dp = fn.get_device(d, keep=True)
dp.AttributeStop(a)
fn.wait(timewait)
fn.wait(10. * timewait)
for a, d in devs.items():
dp = fn.get_device(d, keep=True)
dp.AttributeStart(a)
fn.wait(timewait)
print('%d attributes restarted' % len(attributes))
def add_periodic_attribute(self,
attribute,
period,
archiver=None,
wait=3.,
force=False):
if not force and period < 500:
raise Exception('periods below 500 ms are not allowed!')
attribute = parse_tango_model(attribute, fqdn=True).fullname.lower()
arch = self.get_periodic_attribute_archiver(attribute)
if arch:
print('%s is already archived by %s!' % (attribute, arch))
p = self.get_periodic_attribute_period(attribute)
if p == period:
return False
else:
archiver = arch
archiver = archiver or self.get_next_periodic_archiver(
attrexp=fn.tango.get_dev_name(attribute) + '/*')
if not self.is_attribute_archived(attribute):
self.info('Attribute %s does not exist in %s database, adding it' %
(attribute, self.db_name))
subs = [d for d in self.get_subscribers() if 'null' in d.lower()]
self.add_attribute(attribute,
archiver=(subs[0] if subs else None),
code_event=True,
context='SERVICE')
self.info('%s.AttributeAdd(%s,%s)' % (archiver, attribute, period))
dp = fn.get_device(archiver, keep=True)
dp.set_timeout_millis(30000)
v = dp.AttributeAdd([attribute, str(int(float(period)))])
fn.wait(wait)
return v
def GetAttDataBetweenDates(self, argin):
"""
Arguments to be AttrName, StartDate, StopDate, Synchronous
If Synchronous is missing or False, data is buffered into attributes, which names are returned
If True or Yes, all the data is returned when ready
Data returned will be (rows,[t0,v0,t1,v1,t2,v2,...])
"""
print time.ctime()+"In ", self.get_name(), "::GetAttDataBetweenDates(%s)"%argin
# Add your own code here
size = 0
aname = argin[0]
tag = self.attr2tag(aname)
dates = self.dates2times(argin[1:3])
RW = False
synch = fn.searchCl('yes|true',str(argin[3:4]))
attrs = [tag,tag+'_r',tag+'_w',tag+'_t'] if RW else [tag,tag+'_r',tag+'_w',tag+'_t']
self.reader.get_attribute_values(aname,
(lambda v: self.reader_hook(aname,v)),dates[0],dates[1],
decimate=True, cache=self.UseApiCache)
self.counter+=1
print(self.counter)
argout = [fn.shape(attrs),[a for a in attrs]]
if not synch:
print '\t%s'%argout
return argout
else:
while not self.IsDataReady(aname):
fandango.wait(0.1)
data = self.AttrData[aname][-1]
for t,v in data:
argout.append(t)
argout.extend(fn.toSequence(v))
return [fn.shape(data),argout]
def hook(src, t, value):
cache[src.normal_name] = (value)
el = fc.EventListener('A Queue for All')
el.set_value_hook(hook)
print('Subscribing %d attributes' % len(attrs))
[s.addListener(el) for k, s in sorted(sources.items())]
print('Subscription took %f seconds' % (fn.now() - t0))
t0 = fn.now()
print('Waiting ...')
while len(cache) < len(attrs):
if fn.now() > t0 + 18.:
break
fn.wait(1.)
print('Attributes upated in %f seconds' % (fn.now() - t0))
def print_all():
for i, t in enumerate(sorted(cache.items())):
k, v = t
print('%s/%s: %s = %s' % (i, len(attrs), k, str(v)[:40]))
print_all()
print('%d attributes were not read' % (len(attrs) - len(cache)))
print(sorted(a for a in attrs if a not in cache.keys()))
print('%d Nones' % len([v for v in cache.values() if v is None]))
def check_archiving_schema(
schema='hdb',
attributes=[],values={},
ti = None,
period = 7200,
old_period=24*3600*90,\
exclude=['*/waveid','*/wavename','*/elotech-*'],
use_index = True,
loads = True,
action=False,
trace=True,
export=None):
ti = fn.now() if ti is None else str2time(ti) if isString(ti) else ti
api = pta.api(schema)
is_hpp = isinstance(api, pta.HDBpp)
check = dict()
old_period = 24*3600*old_period if old_period < 1000 \
else (24*old_period if old_period<3600 else old_period)
allattrs = api.get_attributes() if hasattr(
api, 'get_attributes') else api.keys()
print('%s contains %d attributes' % (schema, len(allattrs)))
if attributes:
if fn.isString(attributes) and fn.isRegexp(attributes):
tattrs = [a for a in allattrs if clsearch(attributes, a)]
else:
attributes = map(fn.tango.get_normal_name, fn.toList(attributes))
tattrs = [
a for a in allattrs if fn.tango.get_normal_name(a) in allattrs
]
else:
tattrs = allattrs
excluded = [a for a in tattrs if any(fn.clmatch(e, a) for e in exclude)]
tattrs = [a for a in tattrs if a not in excluded]
print('%d attributes to check' % len(tattrs))
if not len(tattrs):
return
if excluded:
print('\t%d attributes excluded' % len(excluded))
archived = {}
for a in tattrs:
if hasattr(api, 'get_attribute_archiver'):
arch = api.get_attribute_archiver(a)
else:
arch = api[a].archiver
if arch:
archived[a] = arch
print('\t%d attributes are archived' % len(archived))
#Getting Tango devices currently not running
alldevs = set(t.rsplit('/', 1)[0] for t in tattrs)
#tdevs = filter(fn.check_device,alldevs)
#nodevs = [fn.tango.get_normal_name(d) for d in alldevs if d not in tdevs]
#if nodevs:
#print('\t%d devices are not running' % len(nodevs))
archs = sorted(set(archived.values()))
if loads:
astor = fn.Astor()
astor.load_from_devs_list(archs)
loads = fn.defaultdict(list)
for k, s in astor.items():
for d in s.get_device_list():
d = fn.tango.get_normal_name(d)
for a in archived:
if fn.tango.get_normal_name(archived[a]) == d:
loads[k].append(a)
for k, s in sorted(loads.items()):
print('\t%s archives %d attributes' % (k, len(s)))
noarchs = [
fn.tango.get_normal_name(d) for d in archs if not fn.check_device(d)
]
if noarchs:
print('\t%d archivers are not running: %s' % (len(noarchs), noarchs))
###########################################################################
if isString(values) and values.endswith('.pck'):
print('\nLoading last values from %s file\n' % values)
import pickle
values = pickle.load(open(values))
elif isString(values) and values.endswith('.json'):
print('\nLoading last values from %s file\n' % values)
values = fn.json2dict(values)
elif not use_index or is_hpp:
print('\nGetting last values ...\n')
for a in tattrs:
values[a] = api.load_last_values(a)
else:
print('\nGetting updated tables from database ...\n')
tups = pta.utils.get_table_updates(schema)
# Some tables do not update MySQL index tables
t0 = [a for a in tarch if a in tattrs and not tups[api[a].table]]
check.update((t, check_attribute(a, readable=True)) for t in t0
if not check.get(t))
t0 = [t for t in t0 if check[t]]
print('%d/%d archived attributes have indexes not updated ...' %
(len(t0), len(tarch)))
if t0 and len(t0) < 100:
vs = api.load_last_values(t0)
tups.update((api[t].table, api[t].last_date) for t in t0)
for a in tattrs:
if a in tups:
values[a] = [tups[api[a].table], 0]
for k, v in values.items():
if (len(v) if isSequence(v) else v):
if isinstance(v, dict):
v = v.values()[0]
if isSequence(v) and len(v) == 1:
v = v[0]
if v and not isNumber(v[0]):
v = [date2time(v[0]), v[1]]
values[k] = v
else:
values[k] = [] if isSequence(v) else None
print('%d values obtained' % len(values))
###########################################################################
now = fn.now()
result = fn.Struct()
times = [t[0] for t in values.values() if t]
futures = [t for t in times if t > now]
times = [t for t in times if t < now]
tmiss = []
tfutures = [k for k, v in values.items() if v and v[0] in futures]
tmin, tmax = min(times), max(times)
print('\toldest update was %s' % time2str(tmin))
print('\tnewest update was %s' % time2str(tmax))
if futures:
print('\t%d attributes have values in the future!' % len(futures))
tnovals = [a for a in archived if not values.get(a, None)]
if tnovals:
print('\t%d archived attributes have no values' % len(tnovals))
try:
tmiss = [
a for a, v in values.items()
if v and old_period < v[0] < ti - period and a not in archived
]
except:
print(values.items()[0])
if tmiss:
print('\t%d/%d attrs with values are not archived anymore' %
(len(tmiss), len(tattrs)))
result.Excluded = excluded
result.Schema = schema
result.All = tattrs
result.Archived = values
result.NoValues = tnovals
result.MissingOrRemoved = tmiss
result.TMin = tmin
result.TMax = tmax
result.Futures = tfutures
tup = sorted(a for a in values if values[a] and values[a][0] > ti - period)
tok = [a for a in tup if values[a][1] not in (None, [])]
print('\n%d/%d archived attributes are updated since %s - %s' %
(len(tup), len(archived), ti, period))
print('%d archived attributes are fully ok\n' % (len(tok)))
tnotup = sorted(a for a in values
if values[a] and values[a][0] < ti - period)
print('\t%d archived attrs are not updated' % len(tnotup))
tupnoread = [
a for a in tup if not values[a][1] and fn.read_attribute(a) is None
]
reads = dict((a, fn.read_attribute(a)) for a in tnotup)
tnotupread = [a for a in tnotup if reads[a] is not None]
print('\t%d not updated attrs are readable (Lost)' % len(tnotupread))
print('\t%d of them are not floats' %
len([t for t in tnotupread if not isinstance(reads[t], float)]))
print('\t%d of them are states' %
len([t for t in tnotupread if t.lower().endswith('/state')]))
print('\t%d of them seem motors' %
len([t for t in tnotupread if t.lower().endswith('/position')]))
tnotupevs = [a for a in tnotupread if fn.tango.check_attribute_events(a)]
print('\t%d not updated attrs are readable and have events (LostEvents)' %
len(tnotupevs))
tnotupnotread = [a for a in tnotup if a not in tnotupread]
print('\t%d not updated attrs are not readable' % len(tnotupnotread))
result.Lost = tnotupread
result.LostEvents = tnotupevs
losts = (tnotupevs if is_hpp else tnotupread)
diffs = dict()
for a in losts:
try:
v, vv = values.get(a, (None, ))[1], reads[a]
if fn.isSequence(v): v = fn.toList(v)
if fn.isSequence(vv): vv = fn.toList(vv)
diffs[a] = v != vv
if fn.isSequence(diffs[a]):
diffs[a] = any(diffs[a])
else:
diffs[a] = bool(diffs[a])
except:
diffs[a] = None
fams = fn.defaultdict(list)
for a in tnotupread:
fams['/'.join(a.split('/')[-4:-2])].append(a)
for f in sorted(fams):
print('\t%s: %d attrs not updated' % (f, len(fams[f])))
print()
differ = [a for a in losts if diffs[a]] #is True]
print('\t%d/%d not updated attrs have also wrong values!!!' %
(len(differ), len(losts)))
rd = pta.Reader()
only = [a for a in tnotupread if len(rd.is_attribute_archived(a)) == 1]
print('\t%d/%d not updated attrs are archived only in %s' %
(len(only), len(losts), schema))
result.LostDiff = differ
print()
archs = sorted(set(archived.values()))
astor = fn.Astor()
astor.load_from_devs_list(archs)
badloads = fn.defaultdict(list)
for k, s in astor.items():
for d in s.get_device_list():
d = fn.tango.get_normal_name(d)
for a in losts:
if fn.tango.get_normal_name(archived[a]) == d:
badloads[k].append(a)
for k, s in badloads.items():
if len(s):
print('\t%s archives %d lost attributes' % (k, len(s)))
print('\t%d updated attrs are not readable' % len(tupnoread))
result.ArchivedAndReadable = tok
result.Updated = tup
result.NotUpdated = tnotup
result.Unreadable = tnotupnotread
#result.DeviceNotRunning = nodevs
result.ArchiverNotRunning = noarchs
result.LostFamilies = fams
# Tnones is for readable attributes not being archived
tnones = [
a for a in archived
if (a not in values or values[a] and values[a][1] in (None, []))
and a not in tupnoread and a not in tnotupread
]
tupnones = [a for a in tnones if a in tup]
if tupnones:
print('\t%d archived readable attrs record empty values' %
len(tupnones))
result.Nones = tnones
if 0:
get_ratio = lambda a, b: float(len(a)) / float(len(b))
#result.ArchRatio = get_ratio([t for t in readarch if t not in tnotup],readarch)
#result.ReadRatio = get_ratio(result.Readable,tattrs)
#result.LostRatio = get_ratio([a for a in tread if a in tnotup],tread)
#result.MissRatio = get_ratio([a for a in tread if a not in tarch],tread)
#result.OkRatio = 1.0-result.LostRatio-result.MissRatio
#result.Summary = '\n'.join((
#('Checking archiving of %s attributes'%(len(attributes) if attributes else schema))
#,('%d attributes in %s, %d are currently active'%(len(api),schema,len(tarch)))
#,('%d devices with %d archived attributes are not running'%(len(nodevs),len([a for a in api if a.rsplit('/',1) in nodevs])))
#,('%d archived attributes (%2.1f %%) are unreadable! (check and remove)'%(len(tnoread),1e2*get_ratio(tnoread,tarch)))
#,('%d readable attributes are not archived'%(len(tmiss)))
#,('%d attributes (readable or not) are updated (%2.1f %% of all readables)'%(len(tok),1e2*result.OkRatio))
#,('-'*80)
#,('%d archived attributes (readable or not) are not updated!'%len(tnotup))
#,('%d archived and readable attributes are not updated! (check and restart?)'%len(treadnotup))
#,('-'*80)
#,('%d readable attributes have been removed in the last %d days!'%(len(removed),old_period/(24*3600)))
#,('%d readable scalar attributes are not being archived (not needed anymore?)'%len(tmscalar))
#,('%d readable array attributes are not being archived (Ok)'%len(tmarray))
#,('%d readable array attributes are archived (Expensive)'%len(tarray))
#,('')))
#if trace: print(result.Summary)
#print('%d readable lost,Ok = %2.1f%%, %2.1f %% over all Readables (%2.1f %% of total)'%\
#(len(treadnotup),1e2*result.ArchRatio,1e2*result.OkRatio,1e2*result.ReadRatio))
if action:
if action == 'start_devices':
print('Executing action %s' % action)
api.start_devices()
if action == 'restart_all':
print('Executing action %s' % action)
devs = api.get_archivers()
astor = fn.Astor()
print('Restarting %d devs:' % (len(devs), devs))
astor.load_from_devs_list(devs)
astor.stop_servers()
fn.wait(10.)
astor.start_servers()
#print('NO ACTIONS ARE GONNA BE EXECUTED, AS THESE ARE ONLY RECOMMENDATIONS')
#print("""
#api = PyTangoArchiving.HDBpp(schema)
#api.start_devices()
#or
#api = PyTangoArchiving.ArchivingAPI('%s')
#lostdevs = sorted(set(api[a].archiver for a in result.NotUpdated))
#print(lostdevs)
#if lostdevs < a_reasonable_number:
#astor = fn.Astor()
#astor.load_from_devs_list(lostdevs)
#astor.stop_servers()
#fn.time.sleep(10.)
#astor.start_servers()
#"""%schema)
print('\nfinished in %d seconds\n\n' % (fn.now() - ti))
if export is not None:
if export is True:
export = 'txt'
for x in (export.split(',') if isString(export) else export):
if x in ('json', 'pck', 'pickle', 'txt'):
x = '/tmp/%s.%s' % (schema, x)
print('Saving %s file with keys:\n%s' % (x, result.keys()))
if 'json' in x:
fn.dict2json(result.dict(), x)
else:
f = open(x, 'w')
if 'pck' in x or 'pickle' in x:
pickle.dump(result.dict(), f)
else:
f.write(fn.dict2str(result.dict()))
f.close()
return result
def UpdateValues(self):
# PROTECTED REGION ID(SchemaManager.UpdateValues) ENABLED START #
try:
t0 = t1 = fn.now()
self.info_stream('UpdateValues()')
if (self.ValuesFile or '').strip():
self.info_stream('Load values from: %s ...' % self.ValuesFile)
if self.ValuesFile.endswith('json'):
self.values = fn.json2dict(self.ValuesFile)
else:
with open(self.ValuesFile) as f:
self.values = pickle.load(f)
self.values = dict((a, self.get_last_value(a, v))
for a, v in self.values.items())
t1 = max(v[0] for v in self.values.values() if v)
t1 = min((t1, fn.now()))
self.info_stream('reference time is %s' % fn.time2str(t1))
elif self.Threaded:
self.info_stream('Loading values from thread cache ...')
self.values = dict((a, v) for a, v in self.threadDict.items()
if self.threadDict._updates.get(a, 0))
else:
self.info_stream('Loading values from db ...')
self.values = self.api.load_last_values(self.attr_on)
self.info_stream(
'Updating %d values: %s' %
(len(self.values),
str(len(self.values) and self.values.items()[0])))
self.attr_ok = []
self.attr_nok = []
self.attr_lost = []
self.attr_err = []
for a, v in sorted(self.values.items()):
try:
a = fn.tango.get_full_name(a)
if self.Threaded:
t1 = self.threadDict._updates.get(a, 0)
self.check_attribute_ok(a, v, t=t1)
except Exception as e:
self.attr_err.append(a)
traceback.print_exc()
m = str("%s: %s: %s" % (a, str(v), str(e)))
#self.error_stream(m)
print('*' * 80)
print(fn.time2str() + ' ' + self.get_name() + '.ERROR!:' +
m)
fn.wait(1e-6)
for a in [
'AttributeValues', 'AttributeOkList', 'AttributeNokList',
'AttributeWrongList', 'AttributeLostList',
'AttributeNoevList', 'AttributeStalledList'
]:
self.push_change_event(a, getattr(self, 'read_%s' % a)())
self.update_time = fn.now()
self.state_machine()
self.info_stream(self.get_status())
self.info_stream('UpdateValues() took %f seconds' %
(fn.now() - t0))
except Exception as e:
traceback.print_exc()
self.error_stream(fn.except2str())
raise e
def UpdateValues(self):
# PROTECTED REGION ID(SchemaManager.UpdateValues) ENABLED START #
try:
t0 = t1 = fn.now()
self.info_stream('UpdateValues()')
if (self.ValuesFile or '').strip():
self.info_stream('Load values from: %s ...' % self.ValuesFile)
if self.ValuesFile.endswith('json'):
self.values = fn.json2dict(self.ValuesFile)
else:
with open(self.ValuesFile) as f:
self.values = pickle.load(f)
self.values = dict((a,self.get_last_value(a,v))
for a,v in self.values.items())
t1 = max(v[0] for v in self.values.values() if v)
t1 = min((t1,fn.now()))
self.info_stream('reference time is %s' % fn.time2str(t1))
elif self.Threaded:
self.info_stream('Loading values from thread cache ...')
self.values = dict((a,v) for a,v in self.threadDict.items()
if self.threadDict._updates.get(a,0))
else:
self.info_stream('Loading values from db ...')
self.values = self.api.load_last_values(self.attr_on)
self.info_stream('Updating %d values: %s' % (
len(self.values),str(len(self.values)
and self.values.items()[0])))
self.attr_ok = []
self.attr_nok = []
self.attr_lost = []
self.attr_err = []
for a,v in sorted(self.values.items()):
try:
a = fn.tango.get_full_name(a)
if self.Threaded:
t1 = self.threadDict._updates.get(a,0)
self.check_attribute_ok(a,v,t=t1)
except Exception as e:
self.attr_err.append(a)
traceback.print_exc()
m = str("%s: %s: %s" % (a, str(v), str(e)))
#self.error_stream(m)
print('*'*80)
print(fn.time2str()+' '+self.get_name()+'.ERROR!:'+m)
fn.wait(1e-6)
for a in ['AttributeValues','AttributeOkList','AttributeNokList',
'AttributeWrongList','AttributeLostList',
'AttributeNoevList','AttributeStalledList']:
self.push_change_event(a,getattr(self,'read_%s'%a)())
self.update_time = fn.now()
self.state_machine()
self.info_stream(self.get_status())
self.info_stream('UpdateValues() took %f seconds' % (fn.now()-t0))
except Exception as e:
traceback.print_exc()
self.error_stream(fn.except2str())
raise e
