def start_devices(self,
regexp='*',
force=False,
do_init=False,
do_restart=False):
#devs = fn.tango.get_class_devices('HdbEventSubscriber')
devs = self.get_archivers()
if regexp:
devs = fn.filtersmart(devs, regexp)
off = sorted(set(d for d in devs if not fn.check_device(d)))
if off and do_restart:
print('Restarting %s Archiving Servers ...' % self.db_name)
astor = fn.Astor()
astor.load_from_devs_list(list(off))
astor.stop_servers()
fn.wait(3.)
astor.start_servers()
fn.wait(3.)
for d in devs:
try:
dp = fn.get_device(d)
if do_init:
dp.init()
if force or dp.attributenumber != dp.attributestartednumber:
off.append(d)
print('%s.Start()' % d)
dp.start()
except Exception, e:
self.warning('start_archivers(%s) failed: %s' % (d, e))
def start_devices(self,regexp = '*', force = False,
do_init = False, do_restart = False):
#devs = fn.tango.get_class_devices('HdbEventSubscriber')
devs = self.get_archivers()
if regexp:
devs = fn.filtersmart(devs,regexp)
off = sorted(set(d for d in devs if not fn.check_device(d)))
if off and do_restart:
print('Restarting %s Archiving Servers ...'%self.db_name)
astor = fn.Astor()
astor.load_from_devs_list(list(off))
astor.stop_servers()
fn.wait(3.)
astor.start_servers()
fn.wait(3.)
for d in devs:
try:
dp = fn.get_device(d, keep=True)
if do_init:
dp.init()
if force or dp.attributenumber != dp.attributestartednumber:
off.append(d)
print('%s.Start()' % d)
dp.start()
except Exception,e:
self.warning('start_archivers(%s) failed: %s' % (d,e))
def export_attributes_to_pck(filein='ui_exported_devices.txt',
fileout='ui_attribute_values.pck'):
print('export_attributes:' + str((filein, fileout)))
if fandango.isSequence(filein):
devs = filein
else:
devs = map(str.strip, open(filein).readlines())
proxies = dict((d, PyTango.DeviceProxy(d)) for d in devs)
devs = defaultdict(Struct)
for d, dp in sorted(proxies.items()):
print('%s (%d/%d)' % (d, 1 + len(devs), len(proxies)))
obj = devs[d]
obj.dev_class, obj.attrs, obj.comms = '', defaultdict(Struct), {}
obj.props = dict(
(k, v if not 'vector' in str(type(v)).lower() else list(v))
for k, v in fandango.tango.get_matching_device_properties(
d, '*').items() if 'dynamicattributes' not in k.lower())
if fandango.check_device(d):
devs[d].name = d
devs[d].dev_class = dp.info().dev_class
for c in dp.command_list_query():
if c.cmd_name.lower() not in ('state', 'status', 'init'):
obj.comms[c.cmd_name] = (str(c.in_type), str(c.out_type))
for a in dp.get_attribute_list():
if a.lower() == 'status':
continue
obj.attrs[a] = fandango.tango.export_attribute_to_dict(
d, a, as_struct=True)
pickle.dump(devs, open(fileout, 'w'))
return (fileout)
def setModel(self, model):
print '*' * 80
self.info('VaccaProfilePlot.setModel(%s)' % model)
print '*' * 80
try:
#if self._profile_loaded: return
if fandango.isSequence(
model
) or 'attributename' in fandango.tango.parse_tango_model(model):
self.info('setting an attribute model')
TaurusPlot.setModel(self, model) # model = model[0]# str(
# model).rsplit('/',1)[0]
else:
self.info('setting a composer model')
assert fandango.check_device(model)
dev = taurus.Device(model)
if all(a in map(str.lower, dev.get_attribute_list())
for a in ('ccgaxxis', 'ccgpressures', 'ipaxxis',
'ippressures', 'thermoaxxis', 'thermocouples',
'axxispositions', 'axxislabels')):
TaurusPlot.setModel(self, [])
setup_profile_plot(self, model)
else:
self.warning('%s has not all required attributes' % model)
if len(self._positions) and len(self._labels):
self.info('Setting CustomXLabels ...')
self.setAxisCustomLabels(Qwt5.QwtPlot.xBottom,
zip(self._positions, self._labels),
60)
except Exception, e:
self.warning('VaccaProfilePlot.setModel(%s) failed!: %s' %
(model, e))
def get_hdbpp_databases(active=True): #archivers=[],dbs={}):
"""
Method to obtain list of dbs/archivers; it allows to match any
archiver list against existing dbs.
This method can be used in cached mode executed like:
dbs = get_hdbpp_databases()
for a in archivers:
db = get_hdbpp_databases(a,dbs).keys()[0]
"""
schemas = pta.Schemas.load()
hdbpp = sorted(k for k in schemas if fn.clsearch('hdbpp', str(schemas[k])))
if active:
r = []
for h in hdbpp:
try:
if fn.check_device(pta.api(h).manager):
r.append(h)
except:
pass
return r
else:
return hdbpp
def restart_attributes(self, attributes=None, timewait=0.5):
if attributes is None:
attributes = self.get_attributes_not_updated()
todo = []
for a in attributes:
a = self.is_attribute_archived(a)
if a:
todo.append(a)
else:
self.warning('%s is not archived!' % a)
devs = dict(fn.kmap(self.get_attribute_archiver, todo))
for a, d in fn.randomize(sorted(devs.items())):
if not fn.check_device(d):
self.start_devices('(.*/)?' + d, do_restart=True)
else:
dp = fn.get_device(d, keep=True)
dp.AttributeStop(a)
fn.wait(timewait)
fn.wait(10. * timewait)
for a, d in devs.items():
dp = fn.get_device(d, keep=True)
dp.AttributeStart(a)
fn.wait(timewait)
print('%d attributes restarted' % len(attributes))
def get_idle_servers(api='hdb'):
idle = dict()
if fun.isString(api): api = pta.api(api)
for s, t in api.servers.items():
if 'archiver' not in s: continue
for d in t.get_device_list():
if not fun.check_device(d):
idle[s] = [d]
break
trace('\t%d servers have idle devices' % len(idle))
return idle
def get_idle_servers(api='hdb'):
idle = dict()
if fun.isString(api): api = pta.api(api)
for s,t in api.servers.items():
if 'archiver' not in s: continue
for d in t.get_device_list():
if not fun.check_device(d):
idle[s] = [d]
break
trace('\t%d servers have idle devices'%len(idle))
return idle
def attribute_name_check(attribute):
dev,attr = a.rsplit('/',1)
all_devs = fandango.get_all_devices()
if dev not in all_devs:
return False #Device does not exist
elif not fandango.check_device(dev):
return True #If we can't check the attribute we assume that exists.
elif attr.lower() in map(str.lower,PyTango.DeviceProxy(dev).get_attribute_list()):
return True
else:
return False
def check_schema_with_queries(schema):
api = pta.api(schema)
pending = []
done = []
trace('check_schema(%s)' % schema)
#Check IDLE devices
for s, t in api.servers.items():
if 'archiver' not in s: continue
for d in t.get_device_list():
if not fun.check_device(d):
pending.append(s)
break
trace('\t%d servers have idle devices' % len(pending))
for s, t in sorted(api.servers.items()):
if s in pending or s in done: continue
#Check current server attributes
now = time.time()
devs = map(str.lower, t.get_device_list())
attrs = [a for a in api if api[a].archiver.lower() in devs]
for a in attrs:
if 'errorcode' in a: continue
api.load_last_values(a)
if api[a].last_date < now - (MAX_ERROR +
api[a].modes['MODE_P'][0] / 1000.):
pending.append(s)
trace('\t%s marked to restart (%s not updated since %s)' %
(s, a, fun.time2str(api[a].last_date)))
break
#Then check servers pending to restart
now = time.time()
if pending and now > last_restart + WAIT_TIME:
done.append(restart_server(api, pending.pop(0)))
trace('\tAttribute check finished, %d/%d servers pending to restart' %
(len(pending), len(pending) + len(done)))
#Emptying the queue
while len(pending):
if pending and now > last_restart + WAIT_TIME:
done.append(restart_server(api, pending.pop(0)))
else:
time.sleep(1.)
trace('%s check finished, %d/%d servers have idle devices' %
(schema, len(pending), len(servers)))
#Now checking for attributes in .csvs that are not archived!
#or attributes with dedicated != '' but not archived
##@todo...
return done
def restart_attribute(self,attr, d=''):
try:
d = self.get_attribute_archiver(attr)
print('%s.restart_attribute(%s)' % (d,attr))
dp = fn.get_device(d, keep=True)
if not fn.check_device(dp):
self.start_devices('(.*/)?'+d,do_restart=True)
dp.AttributeStop(attr)
fn.wait(.1)
dp.AttributeStart(attr)
except:
print('%s.AttributeStart(%s) failed!'%(d,attr))
def check_schema_with_queries(schema):
api = pta.api(schema)
pending = []
done = []
trace('check_schema(%s)'%schema)
#Check IDLE devices
for s,t in api.servers.items():
if 'archiver' not in s: continue
for d in t.get_device_list():
if not fun.check_device(d):
pending.append(s)
break
trace('\t%d servers have idle devices'%len(pending))
for s,t in sorted(api.servers.items()):
if s in pending or s in done: continue
#Check current server attributes
now = time.time()
devs = map(str.lower,t.get_device_list())
attrs = [a for a in api if api[a].archiver.lower() in devs]
for a in attrs:
if 'errorcode' in a: continue
api.load_last_values(a)
if api[a].last_date<now-(MAX_ERROR+api[a].modes['MODE_P'][0]/1000.):
pending.append(s)
trace('\t%s marked to restart (%s not updated since %s)'%(s,a,fun.time2str(api[a].last_date)))
break
#Then check servers pending to restart
now = time.time()
if pending and now > last_restart+WAIT_TIME:
done.append(restart_server(api,pending.pop(0)))
trace('\tAttribute check finished, %d/%d servers pending to restart'%(
len(pending),len(pending)+len(done)))
#Emptying the queue
while len(pending):
if pending and now > last_restart+WAIT_TIME:
done.append(restart_server(api,pending.pop(0)))
else:
time.sleep(1.)
trace('%s check finished, %d/%d servers have idle devices'%(schema,len(pending),len(servers)))
#Now checking for attributes in .csvs that are not archived!
#or attributes with dedicated != '' but not archived
##@todo...
return done
def restart_attribute(self, attr, d=''):
try:
d = self.get_attribute_archiver(attr)
print('%s.restart_attribute(%s)' % (d, attr))
dp = fn.get_device(d, keep=True)
if not fn.check_device(dp):
self.start_devices('(.*/)?' + d, do_restart=True)
dp.AttributeStop(attr)
fn.wait(.1)
dp.AttributeStart(attr)
except:
print('%s.AttributeStart(%s) failed!' % (d, attr))
def UpdateArchivers(self):
# PROTECTED REGION ID(SchemaManager.UpdateArchivers) ENABLED START #
self.info_stream('UpdateArchivers()')
self.klass,self.is_hpp = ('HdbArchiver',0) if self.schema == 'hdb' \
else ('TdbArchiver',0) if self.schema == 'tdb' \
else ('HdbEventSubscriber',1)
self.arch_on, self.arch_off = [],[]
self.archivers = map(fn.tango.get_full_name,self.api.get_archivers())
for d in sorted(self.archivers):
if fn.check_device(d) not in (None,PyTango.DevState.FAULT):
self.arch_on.append(d)
else:
self.arch_off.append(d)
self.state_machine()
def UpdateArchivers(self):
# PROTECTED REGION ID(SchemaManager.UpdateArchivers) ENABLED START #
self.info_stream('UpdateArchivers()')
self.klass,self.is_hpp = ('HdbArchiver',0) if self.schema == 'hdb' \
else ('TdbArchiver',0) if self.schema == 'tdb' \
else ('HdbEventSubscriber',1)
self.arch_on, self.arch_off = [], []
self.archivers = map(fn.tango.get_full_name, self.api.get_archivers())
for d in sorted(self.archivers):
if fn.check_device(d) not in (None, PyTango.DevState.FAULT):
self.arch_on.append(d)
else:
self.arch_off.append(d)
self.state_machine()
def getAlarmTimestamp(alarm,attr_value=None,use_taurus=True):
"""
Returns alarm activation timestamp (or 0) of an alarm object
"""
trace('panic.gui.getAlarmTimestamp(%s(%s),%s,%s)'%(type(alarm),alarm,attr_value,use_taurus))
#Not using API method, reusing last Taurus polled attribute instead
try:
if attr_value is None and use_taurus:
attr_value = taurus.Attribute(alarm.device+'/ActiveAlarms').read()
attr_value = getAttrValue(attr_value)
return alarm.get_time(attr_value=attr_value)
except:
trace('getAlarmTimestamp(%s/%s): Failed!'%(alarm.device,alarm.tag))
trace(fandango.check_device(alarm.device) and traceback.format_exc())
return 0 #In case of error it must always return 0!!! (as it is used to set alarm.active)
def get_servers_status(regexp='*', exclude=['bpms', 'test', 'sr_vc_']):
servers = fandango.Astor()
servers.load_by_name('PyAlarm/*%s*' % regexp)
servers.load_by_name('Panic*/*%s*' % regexp)
print('%d servers loaded' % len(servers))
states = servers.states()
[states.pop(k) for k in states.keys() if any(e in k for e in exclude)]
exported = fandango.get_all_devices(exported=True)
exported = [s for s in states if 'dserver/' + s in exported]
zombies = sorted(d for d, s in states.items()
if d in exported and s is None)
off = sorted(d for d, s in states.items()
if d not in zombies and s is None)
on = sorted(s for s in states if states[s] is not None)
print('\n')
for s in off:
print('%s : %s : OFF' % (servers[s].host, s))
for s in zombies:
print('%s : %s : ZOMBIE!' % (servers[s].host, s))
print('\n')
failed = []
for s in on:
for d in sorted(servers[s].get_device_list()):
if not fandango.matchCl('(sys|dserver)/*', d):
ss = fandango.check_device(d)
p = fandango.tango.get_device_property(d, 'pollingperiod')
if not p: print('%s has no polling defined' % d)
elif float(p) > 1000:
print('%s has a wrong polling! %s' % (d, p))
if str(ss) not in ('ALARM', 'ON'):
failed.append(s)
print('%s : %s : %s : %s' %
(servers[s].host, s, d, str(ss)))
print('\n%d servers have failed devices' % len(failed))
restart = sorted(set(d for l in (off, zombies, failed) for d in l))
print('%d servers should be restarted' % len(restart))
print('')
return {
'off': off,
'on': on,
'zombies': zombies,
'failed': failed,
'restart': restart
}
def restart_attribute(self, attr, d=''):
"""
execute AttributeStop/Start on subscriber device
"""
try:
a = self.is_attribute_archived(attr)
if not a:
raise Exception('%s is not archived!' % attr)
attr = a
d = self.get_attribute_archiver(attr)
print('%s.restart_attribute(%s)' % (d, attr))
dp = fn.get_device(d, keep=True)
if not fn.check_device(dp):
self.start_devices('(.*/)?' + d, do_restart=True)
dp.AttributeStop(attr)
fn.wait(10.)
dp.AttributeStart(attr)
except:
print('%s.AttributeStart(%s) failed!' % (d, attr))
def export_attributes_to_pck(filein='ui_exported_devices.txt',
fileout='ui_attribute_values.pck'):
print('export_attributes from:' + str((filein, fileout)))
assert fileout.endswith('.pck'), 'output must be a pickle file!'
all_devs = fd.tango.get_all_devices()
filein = fd.toList(filein)
if all(d.lower() in all_devs for d in filein):
devs = filein
else:
devs = export_devices_from_sources(*filein, check=True)
print('devices to export: %s' % str(devs))
proxies = dict((d, PyTango.DeviceProxy(d)) for d in devs)
devs = defaultdict(Struct)
for d, dp in sorted(proxies.items()):
print('%s (%d/%d)' % (d, 1 + len(devs), len(proxies)))
obj = devs[d]
obj.dev_class, obj.attrs, obj.comms = '', defaultdict(Struct), {}
obj.props = dict(
(k, v if not 'vector' in str(type(v)).lower() else list(v))
for k, v in fd.tango.get_matching_device_properties(
d, '*').items() if 'dynamicattributes' not in k.lower())
if fd.check_device(d):
devs[d].name = d
devs[d].dev_class = dp.info().dev_class
for c in dp.command_list_query():
if c.cmd_name.lower() not in ('state', 'status', 'init'):
obj.comms[c.cmd_name] = (str(c.in_type), str(c.out_type))
for a in dp.get_attribute_list():
if a.lower() == 'status':
continue
obj.attrs[a] = fd.tango.export_attribute_to_dict(
d, a, as_struct=True)
pickle.dump(devs, open(fileout, 'w'))
return (fileout)
def add_event_subscriber(self, srv, dev, libname=''):
if not fn.check_device(self.manager):
raise Exception('%s not running!' % self.manager)
if '/' not in srv:
srv = 'hdb++es-srv/' + srv
libname = libname or self.get_hdbpp_libname()
dev = parse_tango_model(dev, fqdn=True).fullname
add_new_device(srv, 'HdbEventSubscriber', dev)
manager, dp = self.manager, self.get_manager()
props = Struct(get_matching_device_properties(manager, '*'))
prev = get_device_property(dev, 'AttributeList') or ''
put_device_property(dev, 'AttributeList', prev)
put_device_property(dev, 'DbHost', self.host)
put_device_property(dev, 'DbName', self.db_name)
#put_device_property(dev,'DbUser',self.user)
#put_device_property(dev,'DbPassword',self.passwd)
#put_device_property(dev,'DbPort','3306')
#put_device_property(dev,'DbStartArchivingAtStartup','true')
put_device_property(dev, 'LibConfiguration', [
'user='******'password='******'port=' + getattr(self, 'port', '3306'),
'host=' + self.host,
'dbname=' + self.db_name,
'libname=' + libname,
'lightschema=1',
])
if 'ArchiverList' not in props:
props.ArchiverList = []
#put_device_property(manager,'ArchiverList',
#list(set(list(props.ArchiverList)+[dev])))
print(dev)
dp.ArchiverAdd(dev)
return dev
def set_push_events(filein, period=3000, diff=1e-5):
print('set_push_events(%s,%s,%s)' % (filein, period, diff))
devs = fd.get_matching_devices(filein)
for d in devs[:]:
if not check_device(d):
q = raw_input('Unable to configure events for %s, '
'do you wish to continue?' % d).lower()
if 'y' not in q: return
devs.remove(d)
if devs:
devs = dict(
(d, fd.Struct({'attrs': fd.get_device(d).get_attribute_list()}))
for d in devs)
else:
devs = pickle.load(open(filein))
for d, t in sorted(devs.items()):
print('Setting events (%s,%s) for %s' % (period, diff, d))
try:
dp = PyTango.DeviceProxy(d)
for a in t.attrs:
dp.poll_attribute(a, int(period))
if period > 0:
ac = dp.get_attribute_config(a)
cei = PyTango.ChangeEventInfo()
cei.rel_change = str(diff)
ac.events.ch_event = cei
try:
dp.set_attribute_config(ac)
except:
pass
except:
q = raw_input('Unable to configure events for %s, '
'do you wish to continue?' % d)
if 'y' not in q.lower():
break
print('done')
def create_simulators(filein,
instance='',
path='',
domains={},
tango_host='controls02',
filters='',
override=True): #domains = {'wr/rf':'test/rf'}
path = path or os.path.abspath(os.path.dirname(filein)) + '/'
print('create_simulators:' + str(
(filein, instance, path, domains, tango_host)))
## THIS CHECK IS MANDATORY, YOU SHOULD EXPORT AND THEN LAUNCH IN DIFFERENT CALLS
assert tango_host in str(
fandango.tango.get_tango_host()), 'Use Controls02 for tests!!!'
devs, org = {}, pickle.load(
open(filein if '/' in filein else path + filein))
done = []
all_devs = fandango.get_all_devices()
print('>' * 80)
if not filters:
print('%d devices in %s: %s' % (len(org), filein, sorted(org.keys())))
filters = raw_input('Do you want to filter devices? [*/*/*]').lower()
for d, t in org.items():
k = ('/'.join(
d.split('/')[-3:])).lower() #Removing tango host from the name
for a, b in domains.items():
if k.startswith(a): k = k.replace(a, b)
if not filters or fandango.matchCl(filters, d) or fandango.matchCl(
filters, org[d].dev_class):
devs[k] = t
if override is not False:
dds = [
d for d in devs
if ('/'.join(d.split('/')[-3:])).lower() in all_devs
]
if dds:
print('%d devices already exist: %s' % (len(dds), sorted(dds)))
override = raw_input('Do you want to override existing properties?'
).lower().startswith('y')
else:
override = False
if not instance:
instance = raw_input(
'Enter your instance name for the simulated DynamicServer (use "instance-" to use multiple instances):'
)
print('>' * 80)
for d, t in sorted(devs.items()):
t.dev_class = t.dev_class or d.split('/')[-1]
klass = 'PyStateComposer' if t.dev_class == 'PyStateComposer' else 'SimulatorDS'
server = 'DynamicDS'
instance_temp = '%s%s' % (instance,
t.dev_class) if '-' in instance else instance
print('%s/%s:%s , "%s" => %s ' %
(server, instance_temp, d, t.dev_class, klass))
its_new = ('/'.join(('dserver', server, instance_temp))
).lower() not in all_devs or d.lower() not in all_devs
if its_new or override:
print('writing ... %s(%s)' % (type(t), d))
fandango.tango.add_new_device('%s/%s' % (server, instance_temp),
klass, d)
for p, v in t.props.items():
if not p.startswith(
'__'
): #p not in ('DynamicCommands','DynamicStates','LoadFromFile','DevicesList') and
fandango.tango.put_device_property(d, p, v)
#Overriding Dynamic* properties
try:
fandango.tango.put_device_property(
d, 'LoadFromFile',
path + '%s_attributes.txt' % t.dev_class)
fandango.tango.put_device_property(
d, 'DynamicAttributes',
filter(
bool,
map(
str.strip,
open(path + '%s_attributes.txt' %
t.dev_class).readlines())))
fandango.tango.put_device_property(
d, 'DynamicCommands',
filter(
bool,
map(
str.strip,
open(path + '%s_commands.txt' %
t.dev_class).readlines())))
fandango.tango.put_device_property(
d, 'DynamicStates',
filter(
bool,
map(
str.strip,
open(path +
'%s_states.txt' % t.dev_class).readlines())))
except:
print('Unable to configure %s(%s) properties ' %
(d, t.dev_class))
#traceback.print_exc()
fandango.tango.put_device_property(d, 'OFFSET',
random.randint(0, len(devs)))
done.append(d)
exported = fandango.get_all_devices(exported=True)
update = [d for d in done if d in exported]
print('Updating %d Devices ...' % len(update))
for d in update:
if fandango.check_device(d):
print('Updating %s ...' % d)
try:
fandango.get_device(d).updateDynamicAttributes()
except Exception, e:
print(e)
else:
print('%s failed!' % d)
time.sleep(.2)
def check_db_schema(schema, attributes = None, values = None,
tref = -12*3600, n = 1, filters = '*', export = 'json',
restart = False, subscribe = False):
"""
tref is the time that is considered updated (e.g. now()-86400)
n is used to consider multiple values
attrs: all attributes in db
on: archived
off: in db but not currently archived
ok: updated
known error causes (attrs not lost but not updated):
nok: attributes are not currently readable
noevs: attributes not sending events
novals: attributes never recorded a value
stall: not updated, but current value matches archiving
lost: not updated, and values doesn't match with current
"""
t0 = fn.now()
if hasattr(schema,'schema'):
api,schema = schema,api.schema
else:
api = pta.api(schema)
r = fn.Struct(api=api,schema=schema)
if isString(tref):
tref = fn.str2time(tref)
r.tref = fn.now()+tref if tref < 0 else tref
r.attrs = [a for a in (attributes or api.get_attributes())
if fn.clmatch(filters,a)]
print('check_db_schema(%s,attrs[%s],tref="%s",export as %s)'
% (schema,len(r.attrs),fn.time2str(r.tref),export))
if restart and schema!='hdbpc':
archs = [a for a in api.get_archivers() if not fn.check_device(a)]
if archs:
try:
print('Restarting archivers: %s' % str(archs))
astor = fn.Astor(archs)
astor.stop_servers()
astor.start_servers()
except:
traceback.print_exc()
stopped = api.get_stopped_attributes()
print('Restarting %d stopped attributes' % len(stopped))
api.restart_attributes(stopped)
r.on = [a for a in api.get_archived_attributes() if a in r.attrs]
r.off = [a for a in r.attrs if a not in r.on]
r.archs = fn.defaultdict(list)
r.pers = fn.defaultdict(list)
r.values = load_schema_values(api,r.on,values,n,tref=tref)
if schema in ('tdb','hdb'):
[r.archs[api[k].archiver].append(k) for k in r.on]
else:
r.rvals = r.values
r.freq, r.values = {}, {}
for k,v in r.rvals.items():
try:
if n > 1:
v = v[0] if isSequence(v) and len(v) else v
r.values[k] = v[0] if isSequence(v) and len(v) else v
r.freq[k] = v and float(len(v))/abs(v[0][0]-v[-1][0])
else:
r.values[k] = v
except Exception as e:
print(k,v)
print(fn.except2str())
for k in api.get_archivers():
r.archs[k] = api.get_archiver_attributes(k)
for k in api.get_periodic_archivers():
r.pers[k] = api.get_periodic_archivers_attributes(k)
# Get all updated attributes
r.ok = [a for a,v in r.values.items() if v and v[0] > r.tref]
# Try to read not-updated attributes
r.check = dict((a,fn.check_attribute(a)
) for a in r.on if a not in r.ok)
#r.novals = [a for a,v in r.values.items() if not v]
r.nok, r.stall, r.noevs, r.lost, r.novals, r.evs, r.rem = [],[],[],[],[],{},[]
# Method to compare numpy values
for a,v in r.check.items():
state = check_archived_attribute(a, v, default=CheckState.LOST,
cache=r, tref=r.tref,
check_events = subscribe and not api.is_periodic_archived(a))
{
#CheckState.ON : r.on,
#CheckState.OFF : r.off,
CheckState.OK : r.ok, #Shouldn't be any ok in check list
CheckState.NO_READ : r.nok,
CheckState.STALL : r.stall,
CheckState.NO_EVENTS : r.noevs,
CheckState.LOST : r.lost,
CheckState.UNK : r.novals,
}[state].append(a)
# SUMMARY
r.summary = schema +'\n'
r.summary += ','.join(
"""on: archived
off: not archived
ok: updated
nok: not readable
noevs: no events
novals: no values
stall: not changing
lost: not updated
""".split('\n'))+'\n'
getline = lambda k,v,l: '\t%s:\t:%d\t(%s)' % (k,len(v),l)
r.summary += '\n\t%s:\t:%d\tok+stall: %2.1f %%' % (
'attrs',len(r.attrs),
(100.*(len(r.ok)+len(r.stall))/(len(r.on) or 1e12)))
r.summary += '\n\t%s/%s:\t:%d/%d' % (
'on','off',len(r.on),len(r.off))
#if r.off > 20: r.summary+=' !!!'
r.summary += '\n\t%s/%s:\t:%d/%d' % (
'ok','nok',len(r.ok),len(r.nok))
if len(r.nok) > 10:
r.summary+=' !!!'
r.summary += '\n\t%s/%s:\t:%d/%d' % (
'noevs','novals',len(r.noevs),len(r.novals))
if len(r.novals) > 1:
r.summary+=' !!!'
r.summary += '\n\t%s/%s:\t:%d/%d' % (
'lost','stall',len(r.lost),len(r.stall))
if len(r.lost) > 1:
r.summary+=' !!!'
r.summary += '\n'
r.archivers = dict.fromkeys(api.get_archivers())
for d in sorted(r.archivers):
r.archivers[d] = api.get_archiver_attributes(d)
novals = [a for a in r.archivers[d] if a in r.novals]
lost = [a for a in r.archivers[d] if a in r.lost]
if (len(novals)+len(lost)) > 2:
r.summary += ('\n%s (all/novals/lost): %s/%s/%s'
% (d,len(r.archivers[d]),len(novals),len(lost)))
if hasattr(api,'get_periodic_archivers'):
r.periodics = dict.fromkeys(api.get_periodic_archivers())
for d in sorted(r.periodics):
r.periodics[d] = api.get_periodic_archiver_attributes(d)
novals = [a for a in r.periodics[d] if a in r.novals]
lost = [a for a in r.periodics[d] if a in r.lost]
if len(novals)+len(lost) > 2:
r.summary += ('\n%s (all/novals/lost): %s/%s/%s' %
(d,len(r.periodics[d]),len(novals),len(lost)))
r.perattrs = [a for a in r.on if a in api.get_periodic_attributes()]
r.notper = [a for a in r.on if a not in r.perattrs]
r.summary += '\nfinished in %d seconds\n\n'%(fn.now()-t0)
print(r.summary)
if restart:
try:
retries = r.lost+r.novals+r.nok
print('restarting %d attributes' % len(retries))
api.restart_attributes(retries)
except:
traceback.print_exc()
if export is not None:
if export is True:
export = 'txt'
for x in (export.split(',') if isString(export) else export):
if x in ('json','pck','pickle','txt'):
x = '/tmp/%s.%s' % (schema,x)
print('Saving %s file with keys:\n%s' % (x,r.keys()))
if 'json' in x:
fn.dict2json(r.dict(),x)
else:
f = open(x,'w')
if 'pck' in x or 'pickle' in x:
pickle.dump(r.dict(),f)
else:
f.write(fn.dict2str(r.dict()))
f.close()
for k,v in r.items():
if fn.isSequence(v):
r[k] = sorted(v)
return r
def check_archiving_schema(
schema='hdb',
attributes=[],values={},
ti = None,
period = 7200,
old_period=24*3600*90,\
exclude=['*/waveid','*/wavename','*/elotech-*'],
use_index = True,
loads = True,
action=False,
trace=True,
export=None):
ti = fn.now() if ti is None else str2time(ti) if isString(ti) else ti
api = pta.api(schema)
is_hpp = isinstance(api, pta.HDBpp)
check = dict()
old_period = 24*3600*old_period if old_period < 1000 \
else (24*old_period if old_period<3600 else old_period)
allattrs = api.get_attributes() if hasattr(
api, 'get_attributes') else api.keys()
print('%s contains %d attributes' % (schema, len(allattrs)))
if attributes:
if fn.isString(attributes) and fn.isRegexp(attributes):
tattrs = [a for a in allattrs if clsearch(attributes, a)]
else:
attributes = map(fn.tango.get_normal_name, fn.toList(attributes))
tattrs = [
a for a in allattrs if fn.tango.get_normal_name(a) in allattrs
]
else:
tattrs = allattrs
excluded = [a for a in tattrs if any(fn.clmatch(e, a) for e in exclude)]
tattrs = [a for a in tattrs if a not in excluded]
print('%d attributes to check' % len(tattrs))
if not len(tattrs):
return
if excluded:
print('\t%d attributes excluded' % len(excluded))
archived = {}
for a in tattrs:
if hasattr(api, 'get_attribute_archiver'):
arch = api.get_attribute_archiver(a)
else:
arch = api[a].archiver
if arch:
archived[a] = arch
print('\t%d attributes are archived' % len(archived))
#Getting Tango devices currently not running
alldevs = set(t.rsplit('/', 1)[0] for t in tattrs)
#tdevs = filter(fn.check_device,alldevs)
#nodevs = [fn.tango.get_normal_name(d) for d in alldevs if d not in tdevs]
#if nodevs:
#print('\t%d devices are not running' % len(nodevs))
archs = sorted(set(archived.values()))
if loads:
astor = fn.Astor()
astor.load_from_devs_list(archs)
loads = fn.defaultdict(list)
for k, s in astor.items():
for d in s.get_device_list():
d = fn.tango.get_normal_name(d)
for a in archived:
if fn.tango.get_normal_name(archived[a]) == d:
loads[k].append(a)
for k, s in sorted(loads.items()):
print('\t%s archives %d attributes' % (k, len(s)))
noarchs = [
fn.tango.get_normal_name(d) for d in archs if not fn.check_device(d)
]
if noarchs:
print('\t%d archivers are not running: %s' % (len(noarchs), noarchs))
###########################################################################
if isString(values) and values.endswith('.pck'):
print('\nLoading last values from %s file\n' % values)
import pickle
values = pickle.load(open(values))
elif isString(values) and values.endswith('.json'):
print('\nLoading last values from %s file\n' % values)
values = fn.json2dict(values)
elif not use_index or is_hpp:
print('\nGetting last values ...\n')
for a in tattrs:
values[a] = api.load_last_values(a)
else:
print('\nGetting updated tables from database ...\n')
tups = pta.utils.get_table_updates(schema)
# Some tables do not update MySQL index tables
t0 = [a for a in tarch if a in tattrs and not tups[api[a].table]]
check.update((t, check_attribute(a, readable=True)) for t in t0
if not check.get(t))
t0 = [t for t in t0 if check[t]]
print('%d/%d archived attributes have indexes not updated ...' %
(len(t0), len(tarch)))
if t0 and len(t0) < 100:
vs = api.load_last_values(t0)
tups.update((api[t].table, api[t].last_date) for t in t0)
for a in tattrs:
if a in tups:
values[a] = [tups[api[a].table], 0]
for k, v in values.items():
if (len(v) if isSequence(v) else v):
if isinstance(v, dict):
v = v.values()[0]
if isSequence(v) and len(v) == 1:
v = v[0]
if v and not isNumber(v[0]):
v = [date2time(v[0]), v[1]]
values[k] = v
else:
values[k] = [] if isSequence(v) else None
print('%d values obtained' % len(values))
###########################################################################
now = fn.now()
result = fn.Struct()
times = [t[0] for t in values.values() if t]
futures = [t for t in times if t > now]
times = [t for t in times if t < now]
tmiss = []
tfutures = [k for k, v in values.items() if v and v[0] in futures]
tmin, tmax = min(times), max(times)
print('\toldest update was %s' % time2str(tmin))
print('\tnewest update was %s' % time2str(tmax))
if futures:
print('\t%d attributes have values in the future!' % len(futures))
tnovals = [a for a in archived if not values.get(a, None)]
if tnovals:
print('\t%d archived attributes have no values' % len(tnovals))
try:
tmiss = [
a for a, v in values.items()
if v and old_period < v[0] < ti - period and a not in archived
]
except:
print(values.items()[0])
if tmiss:
print('\t%d/%d attrs with values are not archived anymore' %
(len(tmiss), len(tattrs)))
result.Excluded = excluded
result.Schema = schema
result.All = tattrs
result.Archived = values
result.NoValues = tnovals
result.MissingOrRemoved = tmiss
result.TMin = tmin
result.TMax = tmax
result.Futures = tfutures
tup = sorted(a for a in values if values[a] and values[a][0] > ti - period)
tok = [a for a in tup if values[a][1] not in (None, [])]
print('\n%d/%d archived attributes are updated since %s - %s' %
(len(tup), len(archived), ti, period))
print('%d archived attributes are fully ok\n' % (len(tok)))
tnotup = sorted(a for a in values
if values[a] and values[a][0] < ti - period)
print('\t%d archived attrs are not updated' % len(tnotup))
tupnoread = [
a for a in tup if not values[a][1] and fn.read_attribute(a) is None
]
reads = dict((a, fn.read_attribute(a)) for a in tnotup)
tnotupread = [a for a in tnotup if reads[a] is not None]
print('\t%d not updated attrs are readable (Lost)' % len(tnotupread))
print('\t%d of them are not floats' %
len([t for t in tnotupread if not isinstance(reads[t], float)]))
print('\t%d of them are states' %
len([t for t in tnotupread if t.lower().endswith('/state')]))
print('\t%d of them seem motors' %
len([t for t in tnotupread if t.lower().endswith('/position')]))
tnotupevs = [a for a in tnotupread if fn.tango.check_attribute_events(a)]
print('\t%d not updated attrs are readable and have events (LostEvents)' %
len(tnotupevs))
tnotupnotread = [a for a in tnotup if a not in tnotupread]
print('\t%d not updated attrs are not readable' % len(tnotupnotread))
result.Lost = tnotupread
result.LostEvents = tnotupevs
losts = (tnotupevs if is_hpp else tnotupread)
diffs = dict()
for a in losts:
try:
v, vv = values.get(a, (None, ))[1], reads[a]
if fn.isSequence(v): v = fn.toList(v)
if fn.isSequence(vv): vv = fn.toList(vv)
diffs[a] = v != vv
if fn.isSequence(diffs[a]):
diffs[a] = any(diffs[a])
else:
diffs[a] = bool(diffs[a])
except:
diffs[a] = None
fams = fn.defaultdict(list)
for a in tnotupread:
fams['/'.join(a.split('/')[-4:-2])].append(a)
for f in sorted(fams):
print('\t%s: %d attrs not updated' % (f, len(fams[f])))
print()
differ = [a for a in losts if diffs[a]] #is True]
print('\t%d/%d not updated attrs have also wrong values!!!' %
(len(differ), len(losts)))
rd = pta.Reader()
only = [a for a in tnotupread if len(rd.is_attribute_archived(a)) == 1]
print('\t%d/%d not updated attrs are archived only in %s' %
(len(only), len(losts), schema))
result.LostDiff = differ
print()
archs = sorted(set(archived.values()))
astor = fn.Astor()
astor.load_from_devs_list(archs)
badloads = fn.defaultdict(list)
for k, s in astor.items():
for d in s.get_device_list():
d = fn.tango.get_normal_name(d)
for a in losts:
if fn.tango.get_normal_name(archived[a]) == d:
badloads[k].append(a)
for k, s in badloads.items():
if len(s):
print('\t%s archives %d lost attributes' % (k, len(s)))
print('\t%d updated attrs are not readable' % len(tupnoread))
result.ArchivedAndReadable = tok
result.Updated = tup
result.NotUpdated = tnotup
result.Unreadable = tnotupnotread
#result.DeviceNotRunning = nodevs
result.ArchiverNotRunning = noarchs
result.LostFamilies = fams
# Tnones is for readable attributes not being archived
tnones = [
a for a in archived
if (a not in values or values[a] and values[a][1] in (None, []))
and a not in tupnoread and a not in tnotupread
]
tupnones = [a for a in tnones if a in tup]
if tupnones:
print('\t%d archived readable attrs record empty values' %
len(tupnones))
result.Nones = tnones
if 0:
get_ratio = lambda a, b: float(len(a)) / float(len(b))
#result.ArchRatio = get_ratio([t for t in readarch if t not in tnotup],readarch)
#result.ReadRatio = get_ratio(result.Readable,tattrs)
#result.LostRatio = get_ratio([a for a in tread if a in tnotup],tread)
#result.MissRatio = get_ratio([a for a in tread if a not in tarch],tread)
#result.OkRatio = 1.0-result.LostRatio-result.MissRatio
#result.Summary = '\n'.join((
#('Checking archiving of %s attributes'%(len(attributes) if attributes else schema))
#,('%d attributes in %s, %d are currently active'%(len(api),schema,len(tarch)))
#,('%d devices with %d archived attributes are not running'%(len(nodevs),len([a for a in api if a.rsplit('/',1) in nodevs])))
#,('%d archived attributes (%2.1f %%) are unreadable! (check and remove)'%(len(tnoread),1e2*get_ratio(tnoread,tarch)))
#,('%d readable attributes are not archived'%(len(tmiss)))
#,('%d attributes (readable or not) are updated (%2.1f %% of all readables)'%(len(tok),1e2*result.OkRatio))
#,('-'*80)
#,('%d archived attributes (readable or not) are not updated!'%len(tnotup))
#,('%d archived and readable attributes are not updated! (check and restart?)'%len(treadnotup))
#,('-'*80)
#,('%d readable attributes have been removed in the last %d days!'%(len(removed),old_period/(24*3600)))
#,('%d readable scalar attributes are not being archived (not needed anymore?)'%len(tmscalar))
#,('%d readable array attributes are not being archived (Ok)'%len(tmarray))
#,('%d readable array attributes are archived (Expensive)'%len(tarray))
#,('')))
#if trace: print(result.Summary)
#print('%d readable lost,Ok = %2.1f%%, %2.1f %% over all Readables (%2.1f %% of total)'%\
#(len(treadnotup),1e2*result.ArchRatio,1e2*result.OkRatio,1e2*result.ReadRatio))
if action:
if action == 'start_devices':
print('Executing action %s' % action)
api.start_devices()
if action == 'restart_all':
print('Executing action %s' % action)
devs = api.get_archivers()
astor = fn.Astor()
print('Restarting %d devs:' % (len(devs), devs))
astor.load_from_devs_list(devs)
astor.stop_servers()
fn.wait(10.)
astor.start_servers()
#print('NO ACTIONS ARE GONNA BE EXECUTED, AS THESE ARE ONLY RECOMMENDATIONS')
#print("""
#api = PyTangoArchiving.HDBpp(schema)
#api.start_devices()
#or
#api = PyTangoArchiving.ArchivingAPI('%s')
#lostdevs = sorted(set(api[a].archiver for a in result.NotUpdated))
#print(lostdevs)
#if lostdevs < a_reasonable_number:
#astor = fn.Astor()
#astor.load_from_devs_list(lostdevs)
#astor.stop_servers()
#fn.time.sleep(10.)
#astor.start_servers()
#"""%schema)
print('\nfinished in %d seconds\n\n' % (fn.now() - ti))
if export is not None:
if export is True:
export = 'txt'
for x in (export.split(',') if isString(export) else export):
if x in ('json', 'pck', 'pickle', 'txt'):
x = '/tmp/%s.%s' % (schema, x)
print('Saving %s file with keys:\n%s' % (x, result.keys()))
if 'json' in x:
fn.dict2json(result.dict(), x)
else:
f = open(x, 'w')
if 'pck' in x or 'pickle' in x:
pickle.dump(result.dict(), f)
else:
f.write(fn.dict2str(result.dict()))
f.close()
return result
#!/usr/bin/env python
import fandango as fn
devs = fn.tango.get_class_devices('HdbEventSubscriber')
for d in devs:
try:
if not fn.check_device(d):
fn.Astor(d).stop_servers()
fn.Astor(d).start_servers()
else:
# Wait to next iteration before setting polling
dp = fn.get_device(d)
dp.poll_command('start',1200000)
print(d,'done')
except:
print(fn.getLastException())
#!/usr/bin/env python
import fandango as fn
devs = fn.tango.get_class_devices('HdbEventSubscriber')
for d in devs:
try:
if not fn.check_device(d):
fn.Astor(d).stop_servers()
fn.Astor(d).start_servers()
else:
# Wait to next iteration before setting polling
dp = fn.get_device(d)
dp.poll_command('start', 1200000)
print(d, 'done')
except:
print(fn.getLastException())
tree = PanelDescription('Tree',
classname = 'vacca.VaccaTree',#'vacca.VaccaTree',#'TaurusDevTree',
model = CUSTOM_TREE or ','.join(DEVICES),
sharedDataRead={'LoadItems':'addModels',
##DISABLED BECAUSE TRIGGERED RECURSIVE SELECTION, TO BE AVOIDED IN TREE
#'SelectedInstrument':'findInTree',
}, #It will load devices from synoptic
sharedDataWrite={'SelectedInstrument':'deviceSelected(QString)'}
)
#: USE_DEVICE_PANEL: True or False, To Show by default the DevicePanel
USE_DEVICE_PANEL = USE_DEVICE_PANEL
if USE_DEVICE_PANEL:
print('\t>>> Loading Device panel (%s)...' % DEVICE)
from vacca.panel import VaccaPanel
m = DEVICE if DEVICE and fn.check_device(DEVICE) else None
panel = VaccaPanel.getPanelDescription('Device',model=m)
from vacca.panel import VaccaPanel
if AttributeFilters: VaccaPanel.setAttributeFilters(AttributeFilters)
if IconMap: VaccaPanel.setIconMap(IconMap)
if CommandFilters: VaccaPanel.setCommandFilters(CommandFilters)
#: JDRAW_FILE: The JDRAW file to create the Synoptic,
#: it can be .jdw or .svg but the second option will require the svgsynoptic module from maxlab.
JDRAW_FILE = JDRAW_FILE
if JDRAW_FILE:
print('\t>>> Loading Synoptic panel new ... %s, %s, %s'%(JDRAW_FILE,
JDRAW_HOOK, JDRAW_TREE))
from vacca.synoptics import VaccaSynoptic
def create_simulators(filein,
instance='',
path='',
domains={},
server='',
tango_host='',
filters='',
override=True):
#domains = {'wr/rf':'test/rf'}
path = path or os.path.abspath(os.path.dirname(filein)) + '/'
print('create_simulators:' + str(
(filein, instance, path, domains, tango_host)))
files = fd.listdir(path)
if not any(f.endswith('_attributes.txt') for f in files):
q = raw_input('Property files do not exist yet,\n'
'Do you want to generate them? (y/n)')
if q.lower().startswith('y'):
cur = os.path.abspath(os.curdir)
os.chdir(path)
generate_class_properties(filein)
os.chdir(cur)
## CHECK IS MANDATORY, YOU SHOULD EXPORT AND SIMULATE IN DIFFERENT HOSTS
assert tango_host and tango_host in str(fd.tango.get_tango_host()),\
'Tango Host (%s!=%s) does not match!'%(tango_host,fd.tango.get_tango_host())
devs, org = {}, pickle.load(
open(filein if '/' in filein else path + filein))
done = []
all_devs = fd.get_all_devices()
print('>' * 80)
if not filters:
print('%d devices in %s: %s' % (len(org), filein, sorted(org.keys())))
filters = raw_input('Enter a filter for device names: [*/*/*]').lower()
for d, t in org.items():
k = ('/'.join(
d.split('/')[-3:])).lower() #Removing tango host from the name
for a, b in domains.items():
if k.startswith(a): k = k.replace(a, b)
if not filters or fd.matchCl(filters, d) or fd.matchCl(
filters, org[d].dev_class):
devs[k] = t
if override is not False:
dds = [
d for d in devs
if ('/'.join(d.split('/')[-3:])).lower() in all_devs
]
if dds:
print('%d devices already exist: %s' % (len(dds), sorted(dds)))
override = raw_input(
'Do you want to override existing properties? (y/n)').lower(
).startswith('y')
else:
override = False
if not instance:
instance = raw_input(
'Enter your instance name for the simulated server (use "instance-" to use multiple instances):'
)
elif '/' in instance:
server, instance = instance.split('/')
keepclass = 'y' in raw_input('Keep original Class names?').lower()
if keepclass:
server = 'SimulatorDS'
elif not server:
server = raw_input(
'Enter your server name (SimulatorDS/DynamicDS): [SimulatorDS]') \
or 'SimulatorDS'
print('>' * 80)
for d, t in sorted(devs.items()):
t.dev_class = t.dev_class or d.split('/')[-1]
if t.dev_class == 'PyStateComposer':
klass = t.dev_class
elif keepclass:
klass = t.dev_class + '_sim'
else:
klass = 'SimulatorDS'
instance_temp = '%s%s' % (instance,
t.dev_class) if '-' in instance else instance
print('%s/%s:%s , "%s" => %s ' %
(server, instance_temp, d, t.dev_class, klass))
its_new = ('/'.join(('dserver', server, instance_temp))
).lower() not in all_devs or d.lower() not in all_devs
if its_new or override:
print('writing ... %s(%s)' % (type(t), d))
fd.tango.add_new_device('%s/%s' % (server, instance_temp), klass,
d)
for p, v in t.props.items():
if not p.startswith(
'__'
): #p not in ('DynamicCommands','DynamicStates','LoadFromFile','DevicesList') and
fd.tango.put_device_property(d, p, v)
#Overriding Dynamic* properties
try:
fd.tango.put_device_property(
d, 'LoadFromFile',
path + '%s_attributes.txt' % t.dev_class)
fd.tango.put_device_property(
d, 'DynamicAttributes',
filter(
bool,
map(
str.strip,
open(path + '%s_attributes.txt' %
t.dev_class).readlines())))
fd.tango.put_device_property(
d, 'DynamicCommands',
filter(
bool,
map(
str.strip,
open(path + '%s_commands.txt' %
t.dev_class).readlines())))
fd.tango.put_device_property(
d, 'DynamicStates',
filter(
bool,
map(
str.strip,
open(path +
'%s_states.txt' % t.dev_class).readlines())))
except:
print('Unable to configure %s(%s) properties ' %
(d, t.dev_class))
#traceback.print_exc()
fd.tango.put_device_property(d, 'OFFSET', random.randint(0, len(devs)))
done.append(d)
exported = fd.get_all_devices(exported=True)
update = [d for d in done if d in exported]
print('Updating %d Devices ...' % len(update))
for d in update:
if fd.check_device(d):
print('Updating %s ...' % d)
try:
fd.get_device(d).updateDynamicAttributes()
except Exception, e:
print(e)
else:
print('%s failed!' % d)
time.sleep(.2)
