def export_attributes_to_pck(filein='ui_exported_devices.txt',
fileout='ui_attribute_values.pck'):
print('export_attributes:' + str((filein, fileout)))
if fandango.isSequence(filein):
devs = filein
else:
devs = map(str.strip, open(filein).readlines())
proxies = dict((d, PyTango.DeviceProxy(d)) for d in devs)
devs = defaultdict(Struct)
for d, dp in sorted(proxies.items()):
print('%s (%d/%d)' % (d, 1 + len(devs), len(proxies)))
obj = devs[d]
obj.dev_class, obj.attrs, obj.comms = '', defaultdict(Struct), {}
obj.props = dict(
(k, v if not 'vector' in str(type(v)).lower() else list(v))
for k, v in fandango.tango.get_matching_device_properties(
d, '*').items() if 'dynamicattributes' not in k.lower())
if fandango.check_device(d):
devs[d].name = d
devs[d].dev_class = dp.info().dev_class
for c in dp.command_list_query():
if c.cmd_name.lower() not in ('state', 'status', 'init'):
obj.comms[c.cmd_name] = (str(c.in_type), str(c.out_type))
for a in dp.get_attribute_list():
if a.lower() == 'status':
continue
obj.attrs[a] = fandango.tango.export_attribute_to_dict(
d, a, as_struct=True)
pickle.dump(devs, open(fileout, 'w'))
return (fileout)
def tango2table(filters,opts=[]):
"""
New method to generate .csv using export_device_to_dict and dict2array
Valid options are:
--hosts, --text, --print, --skip-attributes
Calling tango2table(filters,['--hosts','--skip-attributes']) will
return same output that the old tango2csv scripts
"""
import fandango as fd
import fandango.tango as ft
devices = []
[devices.extend(ft.find_devices(f)) for f in filters]
output = fd.defaultdict(lambda :fd.defaultdict(dict))
attr_info = ('display_unit','standard_unit','label','unit','min_alarm',
'events','description','format','max_alarm','polling','alarms',)
excluded = ('None','No standard unit','No display unit','Not specified'
'nada',)
for d in devices:
v = ft.export_device_to_dict(d)
if '--hosts' in opts:
d = output[v['host']][v['server']]
else:
d = output[v['server']]
if '--skip-attributes' not in opts:
d[v['name']] = {'properties':v['properties']}
da = d[v['name']]['attributes'] = fd.defaultdict(dict)
for a,t in v['attributes'].items():
da[a] = dict((i,t.get(i)) for i in attr_info)
else:
d[v['name']] = v['properties']
table = fd.arrays.dict2array(output,
branch_filter = (lambda x:
str(x)!='extensions'),
leave_filter = (lambda y:
y and str(y).split()[0] not in excluded),
empty='')
if '--text' in opts or '--print' in opts:
text = '\n'.join('\t'.join(map(str,r)) for r in table)
if '--print' in opts:
print(text)
return text if '--text' in opts else table
def __init__(self,db_name='',host='',user='',
passwd='', manager='',
other=None, port = '3306'):
"""
Configuration can be loaded from PyTangoArchiving.Schemas,
an HdbConfigurationManager or another DB object.
api = pta.HDBpp(db_name,host,user,passwd,manager)
"""
assert db_name or manager, 'db_name/manager argument is required!'
self.tango = get_database()
if not all((db_name,host,user,passwd)):
if other:
print('HDBpp(): Loading from DB object')
db_name,host,user,passwd = \
other.db_name,other.host,other.user,other.passwd
elif manager:
print('HDBpp(): Loading from manager')
d,h,u,p = HDBpp.get_db_config(manager=manager,db_name=db_name)
db_name = db_name or d
host = host or h
user = user or u
passwd = passwd or p
else:
sch = Schemas.getSchema(db_name)
if sch:
print('HDBpp(): Loading from Schemas')
db_name = sch.get('dbname',sch.get('db_name'))
host = host or sch.get('host')
user = user or sch.get('user')
passwd = passwd or sch.get('passwd')
port = port or sch.get('port')
elif not manager:
print('HDBpp(): Searching for manager')
m = self.get_manager(db_name)
t = HDBpp.get_db_config(manager=m,db_name=db_name)
host,user,passwd = t[1],t[2],t[3]
self.port = port
self.archivers = []
self.attributes = fn.defaultdict(fn.Struct)
self.dedicated = {}
self.status = fn.defaultdict(list)
ArchivingDB.__init__(self,db_name,host,user,passwd,)
try:
self.get_manager()
self.get_attributes()
except:
traceback.print_exc()
print('Unable to get manager')
def add_periodic_attributes(self, attributes, periods, wait=3.):
"""
attributes must be a list, periods a number, list or dict
"""
attributes = sorted(
parse_tango_model(a, fqdn=True).fullname.lower()
for a in attributes)
if fn.isNumber(periods):
periods = dict((a, periods) for a in attributes)
elif fn.isSequence(periods):
periods = dict(zip(attributes, periods))
devs = fn.defaultdict(list)
[devs[fn.tango.get_dev_name(a)].append(a) for a in attributes]
done = []
for dev, attrs in devs.items():
archiver = self.get_next_periodic_archiver(attrexp=dev + '/*')
for attribute in attrs:
try:
period = periods[attribute]
self.info('add_periodic_attribute(%s,%s,%s)' %
(attribute, period, archiver))
self.add_periodic_attribute(attribute,
period=period,
archiver=archiver,
wait=wait)
done.append((attribute, period, archiver))
except:
self.warning(fn.except2str())
return done
def __init__(self,
db_name='',
host='',
user='',
passwd='',
manager='',
other=None,
port='3306'):
"""
Configuration can be loaded from PyTangoArchiving.Schemas,
an HdbConfigurationManager or another DB object.
"""
assert db_name or manager, 'db_name/manager argument is required!'
self.tango = get_database()
if not all((db_name, host, user, passwd)):
if other:
print('HDBpp(): Loading from DB object')
db_name,host,user,passwd = \
other.db_name,other.host,other.user,other.passwd
elif manager:
print('HDBpp(): Loading from manager')
d, h, u, p = self.get_db_config(manager=manager,
db_name=db_name)
db_name = db_name or d
host = host or h
user = user or u
passwd = passwd or p
else:
sch = Schemas.getSchema(db_name)
if sch:
print('HDBpp(): Loading from Schemas')
db_name = sch.get('dbname', sch.get('db_name'))
host = host or sch.get('host')
user = user or sch.get('user')
passwd = passwd or sch.get('passwd')
port = port or sch.get('port')
elif not manager:
print('HDBpp(): Searching for manager')
m = self.get_manager(db_name)
t = self.get_db_config(manager=m, db_name=db_name)
host, user, passwd = t[1], t[2], t[3]
self.port = port
self.archivers = []
self.attributes = fn.defaultdict(fn.Struct)
self.dedicated = {}
ArchivingDB.__init__(
self,
db_name,
host,
user,
passwd,
)
try:
self.get_manager()
self.get_attributes()
except:
traceback.print_exc()
print('Unable to get manager')
def get_hdbpp_for_attributes(attrlist):
filters = get_hdbpp_filters()
r = fn.defaultdict(list)
for a in attrlist:
for d, f in filters.items():
if fn.clmatch(f, a, extend=True):
r[d].append(a)
return r
def export_attributes_to_pck(filein='ui_exported_devices.txt',
fileout='ui_attribute_values.pck'):
print('export_attributes from:' + str((filein, fileout)))
assert fileout.endswith('.pck'), 'output must be a pickle file!'
all_devs = fd.tango.get_all_devices()
filein = fd.toList(filein)
if all(d.lower() in all_devs for d in filein):
devs = filein
else:
devs = export_devices_from_sources(*filein, check=True)
print('devices to export: %s' % str(devs))
proxies = dict((d, PyTango.DeviceProxy(d)) for d in devs)
devs = defaultdict(Struct)
for d, dp in sorted(proxies.items()):
print('%s (%d/%d)' % (d, 1 + len(devs), len(proxies)))
obj = devs[d]
obj.dev_class, obj.attrs, obj.comms = '', defaultdict(Struct), {}
obj.props = dict(
(k, v if not 'vector' in str(type(v)).lower() else list(v))
for k, v in fd.tango.get_matching_device_properties(
d, '*').items() if 'dynamicattributes' not in k.lower())
if fd.check_device(d):
devs[d].name = d
devs[d].dev_class = dp.info().dev_class
for c in dp.command_list_query():
if c.cmd_name.lower() not in ('state', 'status', 'init'):
obj.comms[c.cmd_name] = (str(c.in_type), str(c.out_type))
for a in dp.get_attribute_list():
if a.lower() == 'status':
continue
obj.attrs[a] = fd.tango.export_attribute_to_dict(
d, a, as_struct=True)
pickle.dump(devs, open(fileout, 'w'))
return (fileout)
def __init__(self, mask=None):
#mask = mask or 'FolderDS/*'
self.mask = mask
ProxiesDict.__init__(self)
devs = tango.get_class_devices('FolderDS')
extra = fn.get_database().get_class_property('FolderDS',
['ExtraDevices'])
devs.extend(extra.get('ExtraDevices', []))
if mask: devs = fn.filtersmart(devs, mask)
self.hosts = fn.defaultdict(list)
for d in devs:
self[d]
def main(args):
import sys, re, traceback, taurus
assert len(
args
) > 1, '\n\nUsage:\n\t> python panel.py [a/device/name or synoptic.jdw] [--attrs] [attr_regexps] --comms [comm_regexps]'
model = args[1]
taurus.setLogLevel(taurus.core.util.Logger.Debug)
app = Qt.QApplication(args)
form = None
if re.match('[\w-]+/[\w-]+/[\w-]+.*', model):
print 'loading a device panel'
k, filters = '--attrs', fandango.defaultdict(list)
for f in args[2:]:
if f.startswith('--'): k = f.strip('-')
else: filters[k].append(f) #(f,()) if k=='comms' else f)
form = VaccaPanel(filters=filters) #palette=get_fullWhite_palette()
form.setModel(model)
elif model.lower().endswith('.jdw'):
print 'loading a synoptic'
form = taurus.qt.qtgui.graphic.TauJDrawSynopticsView(
designMode=False,
updateMode=taurus.qt.qtgui.graphic.TauJDrawSynopticsView.
NoViewportUpdate)
#FullViewportUpdate, : AWFUL CPU USAGE!!!!!!!!
#MinimalViewportUpdate, : Qt Defaults
#SmartViewportUpdate, : ?
#BoundingRectViewportUpdate, : ?
#NoViewportUpdate : Tau defaults
form.setModel(model)
models = form.get_item_list()
for m in models:
m = str(m)
if m.count('/') == 2: m += '/state'
period = 120000.
try:
taurus.Attribute(m).changePollingPeriod(period)
except:
print '(%s).changePollingPeriod(%s): Failed: %s' % (
m, period, traceback.format_exc())
print 'showing ...'
form.show()
sys.exit(app.exec_())
def restart_attributes_archivers(schema,attributes,action=False):
import PyTangoArchiving
api = PyTangoArchiving.api(schema)
devs = fandango.defaultdict(list)
[devs[api[a].archiver].append(a) for a in attributes]
if not action:
print('%d archivers to restart, call with action=True to execute it'%len(devs))
else:
print('Restarting %d archivers'%len(devs))
astor = fandango.Astor()
astor.load_from_devs_list(devs.keys())
astor.stop_servers()
time.sleep(10.)
astor.start_servers()
return dict((k,len(v)) for k,v in devs.items())
def restart_attributes_archivers(schema,attributes,action=False):
import PyTangoArchiving
api = PyTangoArchiving.api(schema)
devs = fn.defaultdict(list)
[devs[api[a].archiver].append(a) for a in attributes]
if not action:
print('%d archivers to restart, call with action=True to execute it'%len(devs))
else:
print('Restarting %d archivers'%len(devs))
astor = fn.Astor()
astor.load_from_devs_list(devs.keys())
astor.stop_servers()
time.sleep(10.)
astor.start_servers()
return dict((k,len(v)) for k,v in devs.items())
def merge_csv_attrs(exported = True, currents = True, check_dups = True):
"""
OJU! Correctors are not exported but should be archived anyway!
"""
folder = fn.tango.get_free_property('PyTangoArchiving','CSVFolder')
csvs = [f for f in fn.listdir(folder) if f.endswith('csv')]
print('Parsing %d files from %s' % (len(csvs),folder))
archattrs = fn.defaultdict(dict)
alldevs = fn.tango.get_all_devices(exported = exported)
sources = dict()
for f in csvs:
try:
sources[f] = pta.ParseCSV(folder+f)
except Exception,e:
print('%s failed: %s\n'%(f,e))
def get_panic_status(*args):
if args and isinstance(args[0], panic.AlarmAPI):
api = args[0]
else:
api = panic.api(*args)
txt = ['Panic(%s) Status:' % str(api.filters)]
txt.append('\t%d alarms in db' % len(api))
txt.append('\t%d devices in db' % len(api.devices))
txt.append('')
states = defaultdict(list)
[states[check_device(d)].append(d) for d in api.devices]
for s, v in sorted(states.items()):
txt.append('%d devices in %s state' % (len(v), s))
ds = ['%s (%d)' % (d, len(api.devices[d].alarms)) for d in sorted(v)]
txt.append('\t%s' % ', '.join(ds))
return '\n'.join(txt)
def get_archivers_attributes(self, archs=None, from_db=True, full=False):
archs = archs or self.get_archivers()
dedicated = fn.defaultdict(list)
if from_db:
for a in archs:
dedicated[a] = [
str(l) for l in get_device_property(a, 'AttributeList')
]
if not full:
dedicated[a] = [str(l).split(';')[0] for l in dedicated[a]]
else:
for a in archs:
try:
dedicated[a].extend(get_device(a).AttributeList)
except:
dedicated[a] = []
self.dedicated.update(dedicated)
return dedicated
def get_archivers_attributes(self,archs=None,from_db=True,full=False):
"""
If not got from_db, the manager may limit the list available
"""
archs = archs or self.get_archivers()
dedicated = fn.defaultdict(list)
if from_db:
for a in archs:
dedicated[a] = [str(l) for l in
get_device_property(a,'AttributeList')]
if not full:
dedicated[a] = [str(l).split(';')[0] for l in
dedicated[a]]
else:
for a in archs:
try:
dedicated[a].extend(get_device(a, keep=True).AttributeList)
except:
dedicated[a] = []
self.dedicated.update(dedicated)
return dedicated
def add_attributes(self, attributes, *args, **kwargs):
"""
Call add_attribute sequentially with a 1s pause between calls
:param start: True by default, will force Start() in related archivers
See add_attribute? for more help on arguments
"""
try:
attributes = sorted(attributes)
start = kwargs.get('start', True)
devs = fn.defaultdict(list)
[devs[fn.tango.get_dev_name(a)].append(a) for a in attributes]
for dev, attrs in devs.items():
arch = self.get_next_archiver(attrexp=dev + '/*')
for a in attrs:
kwargs['start'] = False #Avoid recursive start
try:
kwargs['clear'] = False
self.add_attribute(a, archiver=arch, *args, **kwargs)
except:
self.warning('add_attribute(%s) failed!\n%s' %
(a, traceback.format_exc()))
time.sleep(3.)
self.clear_caches()
if start:
self.get_archivers_attributes()
archs = set(map(self.get_attribute_archiver, attributes))
for h in archs:
try:
if h:
self.info('%s.Start()' % h)
fn.get_device(h, keep=True).Start()
except:
traceback.print_exc()
except Exception, e:
self.error('add_attributes(%s) failed!: %s' %
(attributes, traceback.print_exc()))
def get_attributes_errors(self,
regexp='*',
timeout=3 * 3600,
from_db=False,
extend=False):
"""
Returns a dictionary {attribute, error/last value}
If from_db=True and extend=True, it performs a full attribute check
"""
if regexp == '*':
self.status = fn.defaultdict(list)
if from_db or extend:
timeout = fn.now() - timeout
attrs = self.get_attributes(True)
attrs = fn.filtersmart(attrs, regexp)
print('get_attributes_errors([%d/%d])' %
(len(attrs), len(self.attributes)))
vals = self.load_last_values(attrs)
for a, v in vals.items():
if v and v[0] > timeout:
self.status['Updated'].append(a)
if v[1] is not None:
self.status['Readable'].append(a)
else:
rv = fn.read_attribute(a)
if rv is not None:
self.status['WrongNone'].append(a)
else:
self.status['None'].append(a)
vals.pop(a)
if not extend:
self.status['NotUpdated'] = vals.keys()
else:
for a, v in vals.items():
c = fn.check_attribute(a)
if c is None:
vals[a] = 'Unreadable'
self.status['Unreadable'].append(a)
elif isinstance(c, Exception):
vals[a] = str(c)
self.status['Exception'].append(a)
else:
ev = fn.tango.check_attribute_events(a)
if not ev:
vals[a] = 'NoEvents'
self.status['NoEvents'].append(a)
else:
d = self.get_attribute_archiver(a)
e = self.get_archiver_errors(d)
if a in e:
vals[a] = e[a]
self.status['ArchiverError'].append(a)
else:
rv = fn.read_attribute(a)
if v and str(rv) == str(v[1]):
vals[a] = 'NotChanged'
self.status['NotChanged'].append(a)
else:
self.status['NotUpdated'].append(a)
if regexp == '*':
for k, v in self.status.items():
print('%s: %s' % (k, len(v)))
return vals
else:
# Should inspect the Subscribers Error Lists
vals = dict()
for d in self.get_archivers():
err = self.get_archiver_errors(d)
for a, e in err.items():
if fn.clmatch(regexp, a):
vals[a] = e
return vals
def start_attributes_for_archivers(target,attr_regexp='',event_conf={},
load=False, by_class=False, min_polling = 100, overwrite = False,
check = True):
"""
Target may be an attribute list or a device regular expression
if by_class = True, config will be loaded from Tango class properties
"""
import PyTangoArchiving.hdbpp as ptah
if fn.isSequence(target):
if attr_regexp:
attrs = [a for a in target if fn.clmatch(attr_regexp,a.rsplit('/')[-1])]
else:
attrs = target
else:
dev_regexp = target
attrs = fn.find_attributes(dev_regexp+'/'+(attr_regexp or '*'))
if by_class:
classes = fn.defaultdict(dict)
devs = fn.defaultdict(list)
[devs[a.rsplit('/',1)[0]].append(a) for a in attrs]
for d,v in devs.items():
classes[fn.tango.get_device_class(d)][d] = v
attrs = {}
for c,devs in classes.items():
cfg = get_class_archiving(devs.keys()[0])
for d in devs:
raw = devs[d]
for a,v in cfg.items():
for aa in raw:
if fn.clmatch(a,aa.split('/')[-1],terminate=True):
if not attr_regexp or fn.clmatch(attr_regexp,aa):
attrs[aa] = v
elif event_conf:
attrs = dict((a,event_conf) for a in attrs)
else:
attrs = dict((a,get_current_conf(a)) for a in attrs)
print('Starting %d attributes' % (len(attrs)))
archs = ptah.multi.match_attributes_and_archivers(attrs.keys())
rd = PyTangoArchiving.Reader()
#print(archs)
alldbs = ptah.multi.get_hdbpp_databases()
dbs = ptah.multi.get_hdbpp_databases(archs,alldbs)
#return dbs,archs,attrs
for db,rcs in dbs.items():
api = PyTangoArchiving.Schemas.getApi(db)
dbs[db] = dict.fromkeys(rcs)
for d in rcs:
dbs[db][d] = ts = dict.fromkeys(archs[d])
#return ts
for a in ts:
try:
m = fn.parse_tango_model(a,fqdn=True)
dbs[db][d][a] = mode = attrs[a]
if not overwrite and db in rd.is_attribute_archived(a):
print('%s already archived in %s' % (a,db))
continue
events = ft.check_attribute_events(a,ft.EventType.ARCHIVE_EVENT)
ep = events.get(ft.EventType.ARCHIVE_EVENT,False)
if ep is True:
if 'polling' in mode:
mode.pop('polling')
elif isinstance(events.get(ep,(int,float))):
mode['polling'] = min((ep,mode.get('polling',10000)))
mode['polling'] = max((mode['polling'],min_polling))
if not events.get(ft.EventType.CHANGE_EVENT,False):
if mode.get('archive_abs_change',0):
mode['abs_event'] = mode['archive_abs_change']
if mode.get('archive_rel_change',0):
mode['rel_event'] = mode['archive_rel_change']
if mode.get('arch_per_event',0):
mode['per_event'] = mode['archive_per_event']
print('%s.start_archiving(%s,%s,%s): %s' % (db,d,m.fullname,mode,load))
if load:
fn.tango.set_attribute_events(a,**mode)
r = api.start_archiving(m.fullname,d,code_event=True)
assert not check or r
except:
print('%s failed!'%a)
traceback.print_exc()
return dbs
def main(args):
import sys, re, traceback, taurus
print('args: %s' % str(args))
assert len(args)>1, '\n\nUsage:\n\t'\
'> python panel.py [a/device/name or synoptic.jdw] [--attrs] '\
'[attr_regexps] --comms [comm_regexps]'
model = args[1]
taurus.setLogLevel(taurus.core.util.Logger.Debug)
app = Qt.QApplication(args)
form = None
VACCA_CONFIG = os.getenv('VACCA_CONFIG')
if not VACCA_CONFIG:
VACCA_CONFIG = fandango.objects.find_module('vaccagui')
print('VACCA_CONFIG: %s' % VACCA_CONFIG)
if VACCA_CONFIG:
import vacca.utils as vu
PROPS = vu.get_config_properties(VACCA_CONFIG)
VACCA_DIR = WDIR = PROPS.get('VACCA_DIR', vu.VACCA_DIR)
try:
import default
except:
try:
default = get_config_file(
imp.find_module('vacca')[1] + '/default.py')
except:
traceback.print_exc()
CONFIG = vu.get_config_file()
if re.match(fandango.tango.retango, model):
print 'loading a device panel'
k, filters = '--attrs', fandango.defaultdict(list)
for f in args[2:]:
if f.startswith('--'): k = f.strip('-')
else: filters[k].append(f) #(f,()) if k=='comms' else f)
form = VaccaPanel(filters=filters) #palette=get_fullWhite_palette()
form.setModel(model)
elif model.lower().endswith('.jdw'):
print 'loading a synoptic'
import taurus.qt.qtgui.graphic as tqqg
form = tqqg.TauJDrawSynopticsView(
designMode=False,
updateMode=tqqg.TauJDrawSynopticsView.NoViewportUpdate)
#FullViewportUpdate, : AWFUL CPU USAGE!!!!!!!!
#MinimalViewportUpdate, : Qt Defaults
#SmartViewportUpdate, : ?
#BoundingRectViewportUpdate, : ?
#NoViewportUpdate : Tau defaults
form.setModel(model)
models = form.get_item_list()
for m in models:
m = str(m)
if m.count('/') == 2: m += '/state'
period = 120000.
try:
taurus.Attribute(m).changePollingPeriod(period)
except:
print '(%s).changePollingPeriod(%s): Failed: %s' % (
m, period, traceback.format_exc())
else:
print('Unknown model: %s' % model)
print 'showing ...'
form.show()
sys.exit(app.exec_())
def check_archiving_schema(
schema='hdb',
attributes=[],values={},
ti = None,
period = 7200,
old_period=24*3600*90,\
exclude=['*/waveid','*/wavename','*/elotech-*'],
use_index = True,
loads = True,
action=False,
trace=True,
export=None):
ti = fn.now() if ti is None else str2time(ti) if isString(ti) else ti
api = pta.api(schema)
is_hpp = isinstance(api, pta.HDBpp)
check = dict()
old_period = 24*3600*old_period if old_period < 1000 \
else (24*old_period if old_period<3600 else old_period)
allattrs = api.get_attributes() if hasattr(
api, 'get_attributes') else api.keys()
print('%s contains %d attributes' % (schema, len(allattrs)))
if attributes:
if fn.isString(attributes) and fn.isRegexp(attributes):
tattrs = [a for a in allattrs if clsearch(attributes, a)]
else:
attributes = map(fn.tango.get_normal_name, fn.toList(attributes))
tattrs = [
a for a in allattrs if fn.tango.get_normal_name(a) in allattrs
]
else:
tattrs = allattrs
excluded = [a for a in tattrs if any(fn.clmatch(e, a) for e in exclude)]
tattrs = [a for a in tattrs if a not in excluded]
print('%d attributes to check' % len(tattrs))
if not len(tattrs):
return
if excluded:
print('\t%d attributes excluded' % len(excluded))
archived = {}
for a in tattrs:
if hasattr(api, 'get_attribute_archiver'):
arch = api.get_attribute_archiver(a)
else:
arch = api[a].archiver
if arch:
archived[a] = arch
print('\t%d attributes are archived' % len(archived))
#Getting Tango devices currently not running
alldevs = set(t.rsplit('/', 1)[0] for t in tattrs)
#tdevs = filter(fn.check_device,alldevs)
#nodevs = [fn.tango.get_normal_name(d) for d in alldevs if d not in tdevs]
#if nodevs:
#print('\t%d devices are not running' % len(nodevs))
archs = sorted(set(archived.values()))
if loads:
astor = fn.Astor()
astor.load_from_devs_list(archs)
loads = fn.defaultdict(list)
for k, s in astor.items():
for d in s.get_device_list():
d = fn.tango.get_normal_name(d)
for a in archived:
if fn.tango.get_normal_name(archived[a]) == d:
loads[k].append(a)
for k, s in sorted(loads.items()):
print('\t%s archives %d attributes' % (k, len(s)))
noarchs = [
fn.tango.get_normal_name(d) for d in archs if not fn.check_device(d)
]
if noarchs:
print('\t%d archivers are not running: %s' % (len(noarchs), noarchs))
###########################################################################
if isString(values) and values.endswith('.pck'):
print('\nLoading last values from %s file\n' % values)
import pickle
values = pickle.load(open(values))
elif isString(values) and values.endswith('.json'):
print('\nLoading last values from %s file\n' % values)
values = fn.json2dict(values)
elif not use_index or is_hpp:
print('\nGetting last values ...\n')
for a in tattrs:
values[a] = api.load_last_values(a)
else:
print('\nGetting updated tables from database ...\n')
tups = pta.utils.get_table_updates(schema)
# Some tables do not update MySQL index tables
t0 = [a for a in tarch if a in tattrs and not tups[api[a].table]]
check.update((t, check_attribute(a, readable=True)) for t in t0
if not check.get(t))
t0 = [t for t in t0 if check[t]]
print('%d/%d archived attributes have indexes not updated ...' %
(len(t0), len(tarch)))
if t0 and len(t0) < 100:
vs = api.load_last_values(t0)
tups.update((api[t].table, api[t].last_date) for t in t0)
for a in tattrs:
if a in tups:
values[a] = [tups[api[a].table], 0]
for k, v in values.items():
if (len(v) if isSequence(v) else v):
if isinstance(v, dict):
v = v.values()[0]
if isSequence(v) and len(v) == 1:
v = v[0]
if v and not isNumber(v[0]):
v = [date2time(v[0]), v[1]]
values[k] = v
else:
values[k] = [] if isSequence(v) else None
print('%d values obtained' % len(values))
###########################################################################
now = fn.now()
result = fn.Struct()
times = [t[0] for t in values.values() if t]
futures = [t for t in times if t > now]
times = [t for t in times if t < now]
tmiss = []
tfutures = [k for k, v in values.items() if v and v[0] in futures]
tmin, tmax = min(times), max(times)
print('\toldest update was %s' % time2str(tmin))
print('\tnewest update was %s' % time2str(tmax))
if futures:
print('\t%d attributes have values in the future!' % len(futures))
tnovals = [a for a in archived if not values.get(a, None)]
if tnovals:
print('\t%d archived attributes have no values' % len(tnovals))
try:
tmiss = [
a for a, v in values.items()
if v and old_period < v[0] < ti - period and a not in archived
]
except:
print(values.items()[0])
if tmiss:
print('\t%d/%d attrs with values are not archived anymore' %
(len(tmiss), len(tattrs)))
result.Excluded = excluded
result.Schema = schema
result.All = tattrs
result.Archived = values
result.NoValues = tnovals
result.MissingOrRemoved = tmiss
result.TMin = tmin
result.TMax = tmax
result.Futures = tfutures
tup = sorted(a for a in values if values[a] and values[a][0] > ti - period)
tok = [a for a in tup if values[a][1] not in (None, [])]
print('\n%d/%d archived attributes are updated since %s - %s' %
(len(tup), len(archived), ti, period))
print('%d archived attributes are fully ok\n' % (len(tok)))
tnotup = sorted(a for a in values
if values[a] and values[a][0] < ti - period)
print('\t%d archived attrs are not updated' % len(tnotup))
tupnoread = [
a for a in tup if not values[a][1] and fn.read_attribute(a) is None
]
reads = dict((a, fn.read_attribute(a)) for a in tnotup)
tnotupread = [a for a in tnotup if reads[a] is not None]
print('\t%d not updated attrs are readable (Lost)' % len(tnotupread))
print('\t%d of them are not floats' %
len([t for t in tnotupread if not isinstance(reads[t], float)]))
print('\t%d of them are states' %
len([t for t in tnotupread if t.lower().endswith('/state')]))
print('\t%d of them seem motors' %
len([t for t in tnotupread if t.lower().endswith('/position')]))
tnotupevs = [a for a in tnotupread if fn.tango.check_attribute_events(a)]
print('\t%d not updated attrs are readable and have events (LostEvents)' %
len(tnotupevs))
tnotupnotread = [a for a in tnotup if a not in tnotupread]
print('\t%d not updated attrs are not readable' % len(tnotupnotread))
result.Lost = tnotupread
result.LostEvents = tnotupevs
losts = (tnotupevs if is_hpp else tnotupread)
diffs = dict()
for a in losts:
try:
v, vv = values.get(a, (None, ))[1], reads[a]
if fn.isSequence(v): v = fn.toList(v)
if fn.isSequence(vv): vv = fn.toList(vv)
diffs[a] = v != vv
if fn.isSequence(diffs[a]):
diffs[a] = any(diffs[a])
else:
diffs[a] = bool(diffs[a])
except:
diffs[a] = None
fams = fn.defaultdict(list)
for a in tnotupread:
fams['/'.join(a.split('/')[-4:-2])].append(a)
for f in sorted(fams):
print('\t%s: %d attrs not updated' % (f, len(fams[f])))
print()
differ = [a for a in losts if diffs[a]] #is True]
print('\t%d/%d not updated attrs have also wrong values!!!' %
(len(differ), len(losts)))
rd = pta.Reader()
only = [a for a in tnotupread if len(rd.is_attribute_archived(a)) == 1]
print('\t%d/%d not updated attrs are archived only in %s' %
(len(only), len(losts), schema))
result.LostDiff = differ
print()
archs = sorted(set(archived.values()))
astor = fn.Astor()
astor.load_from_devs_list(archs)
badloads = fn.defaultdict(list)
for k, s in astor.items():
for d in s.get_device_list():
d = fn.tango.get_normal_name(d)
for a in losts:
if fn.tango.get_normal_name(archived[a]) == d:
badloads[k].append(a)
for k, s in badloads.items():
if len(s):
print('\t%s archives %d lost attributes' % (k, len(s)))
print('\t%d updated attrs are not readable' % len(tupnoread))
result.ArchivedAndReadable = tok
result.Updated = tup
result.NotUpdated = tnotup
result.Unreadable = tnotupnotread
#result.DeviceNotRunning = nodevs
result.ArchiverNotRunning = noarchs
result.LostFamilies = fams
# Tnones is for readable attributes not being archived
tnones = [
a for a in archived
if (a not in values or values[a] and values[a][1] in (None, []))
and a not in tupnoread and a not in tnotupread
]
tupnones = [a for a in tnones if a in tup]
if tupnones:
print('\t%d archived readable attrs record empty values' %
len(tupnones))
result.Nones = tnones
if 0:
get_ratio = lambda a, b: float(len(a)) / float(len(b))
#result.ArchRatio = get_ratio([t for t in readarch if t not in tnotup],readarch)
#result.ReadRatio = get_ratio(result.Readable,tattrs)
#result.LostRatio = get_ratio([a for a in tread if a in tnotup],tread)
#result.MissRatio = get_ratio([a for a in tread if a not in tarch],tread)
#result.OkRatio = 1.0-result.LostRatio-result.MissRatio
#result.Summary = '\n'.join((
#('Checking archiving of %s attributes'%(len(attributes) if attributes else schema))
#,('%d attributes in %s, %d are currently active'%(len(api),schema,len(tarch)))
#,('%d devices with %d archived attributes are not running'%(len(nodevs),len([a for a in api if a.rsplit('/',1) in nodevs])))
#,('%d archived attributes (%2.1f %%) are unreadable! (check and remove)'%(len(tnoread),1e2*get_ratio(tnoread,tarch)))
#,('%d readable attributes are not archived'%(len(tmiss)))
#,('%d attributes (readable or not) are updated (%2.1f %% of all readables)'%(len(tok),1e2*result.OkRatio))
#,('-'*80)
#,('%d archived attributes (readable or not) are not updated!'%len(tnotup))
#,('%d archived and readable attributes are not updated! (check and restart?)'%len(treadnotup))
#,('-'*80)
#,('%d readable attributes have been removed in the last %d days!'%(len(removed),old_period/(24*3600)))
#,('%d readable scalar attributes are not being archived (not needed anymore?)'%len(tmscalar))
#,('%d readable array attributes are not being archived (Ok)'%len(tmarray))
#,('%d readable array attributes are archived (Expensive)'%len(tarray))
#,('')))
#if trace: print(result.Summary)
#print('%d readable lost,Ok = %2.1f%%, %2.1f %% over all Readables (%2.1f %% of total)'%\
#(len(treadnotup),1e2*result.ArchRatio,1e2*result.OkRatio,1e2*result.ReadRatio))
if action:
if action == 'start_devices':
print('Executing action %s' % action)
api.start_devices()
if action == 'restart_all':
print('Executing action %s' % action)
devs = api.get_archivers()
astor = fn.Astor()
print('Restarting %d devs:' % (len(devs), devs))
astor.load_from_devs_list(devs)
astor.stop_servers()
fn.wait(10.)
astor.start_servers()
#print('NO ACTIONS ARE GONNA BE EXECUTED, AS THESE ARE ONLY RECOMMENDATIONS')
#print("""
#api = PyTangoArchiving.HDBpp(schema)
#api.start_devices()
#or
#api = PyTangoArchiving.ArchivingAPI('%s')
#lostdevs = sorted(set(api[a].archiver for a in result.NotUpdated))
#print(lostdevs)
#if lostdevs < a_reasonable_number:
#astor = fn.Astor()
#astor.load_from_devs_list(lostdevs)
#astor.stop_servers()
#fn.time.sleep(10.)
#astor.start_servers()
#"""%schema)
print('\nfinished in %d seconds\n\n' % (fn.now() - ti))
if export is not None:
if export is True:
export = 'txt'
for x in (export.split(',') if isString(export) else export):
if x in ('json', 'pck', 'pickle', 'txt'):
x = '/tmp/%s.%s' % (schema, x)
print('Saving %s file with keys:\n%s' % (x, result.keys()))
if 'json' in x:
fn.dict2json(result.dict(), x)
else:
f = open(x, 'w')
if 'pck' in x or 'pickle' in x:
pickle.dump(result.dict(), f)
else:
f.write(fn.dict2str(result.dict()))
f.close()
return result
def get_attributes_errors(self, regexp='*', timeout=3*3600,
from_db=False, extend = False):
"""
Returns a dictionary {attribute, error/last value}
If from_db=True and extend=True, it performs a full attribute check
"""
if regexp == '*':
self.status = fn.defaultdict(list)
if from_db or extend:
timeout = fn.now()-timeout
attrs = self.get_attributes(True)
attrs = fn.filtersmart(attrs,regexp)
print('get_attributes_errors([%d/%d])'
% (len(attrs),len(self.attributes)))
vals = self.load_last_values(attrs)
for a,v in vals.items():
if v and v[0] > timeout:
self.status['Updated'].append(a)
if v[1] is not None:
self.status['Readable'].append(a)
else:
rv = fn.read_attribute(a)
if rv is not None:
self.status['WrongNone'].append(a)
else:
self.status['None'].append(a)
vals.pop(a)
if not extend:
self.status['NotUpdated'] = vals.keys()
else:
for a,v in vals.items():
c = fn.check_attribute(a)
if c is None:
vals[a] = 'Unreadable'
self.status['Unreadable'].append(a)
elif isinstance(c,Exception):
vals[a] = str(c)
self.status['Exception'].append(a)
else:
ev = fn.tango.check_attribute_events(a)
if not ev:
vals[a] = 'NoEvents'
self.status['NoEvents'].append(a)
else:
d = self.get_attribute_archiver(a)
e = self.get_archiver_errors(d)
if a in e:
vals[a] = e[a]
self.status['ArchiverError'].append(a)
else:
rv = fn.read_attribute(a)
if v and str(rv) == str(v[1]):
vals[a] = 'NotChanged'
self.status['NotChanged'].append(a)
else:
self.status['NotUpdated'].append(a)
if regexp == '*':
for k,v in self.status.items():
print('%s: %s' % (k,len(v)))
return vals
else:
# Should inspect the Subscribers Error Lists
vals = dict()
for d in self.get_archivers():
err = self.get_archiver_errors(d)
for a,e in err.items():
if fn.clmatch(regexp,a):
vals[a] = e
return vals
def decimate(db_name, keys, tstart, tend, period=10, dry=False):
"""
time arguments are strings
BUT!, this method seems to not work anymore to free space in TDB
Maybe a mysqld restart is needed, I don't know;
but up to now space is not freed
"""
api = pta.api(db_name)
if '/' in keys[0]:
print('Decimating by attribute names')
tables = fn.defaultdict(list)
for a in keys:
api.get_attr_id_type_table(a)
tables[api[a].table].append(a)
print('tables: %s' % (tables.keys()))
for table, attrs in tables.items():
for a in attrs:
pta.dbs.decimate_db_table_by_time(api,
table,
api[a].id,
tstart,
tend,
period,
optimize=(a == attrs[-1]))
if not '/' in keys[0]:
print('Decimating by data_type')
data_types = keys
if not data_types:
data_types = [
r[0]
for r in api.Query('select data_type from att_conf_data_type')
]
else:
data_types = [d.replace('att_', '') for d in data_types]
print('Decimating %s types between %s and %s: %s' %
(db_name, tstart, tend, data_types))
for data_type in data_types:
attrs = api.Query(
'select att_conf_id from att_conf,att_conf_data_type '
'where att_conf.att_conf_data_type_id = att_conf_data_type.att_conf_data_type_id '
'and data_type = "%s"' % data_type)
attrs = [r[0] for r in attrs]
q = ("select partition_name,table_name"
" from information_schema.partitions where"
" partition_name is not NULL"
" and table_schema = '%s'" % db_name +
" and table_name like '%" + data_type + "'")
print(q)
partitions = api.Query(q)
if partitions:
table = partitions[0][1]
else:
table = 'att_' + data_type
print('%s has %d attributes in %d partitions' %
(table, len(attrs), len(partitions)))
c0 = api.Query('select count(*) from %s ' % table)
import re
intervals = []
for p in partitions:
p = p[0]
r = '(?P<year>[0-9][0-9][0-9][0-9])(?P<month>[0-9][0-9])'
md = re.search(r, p).groupdict()
t0 = '%s-%s-01 00:00:00' % (md['year'], md['month'])
m, y = int(md['month']), int(md['year'])
if m == 12:
m, y = 1, y + 1
else:
m += 1
t1 = '%04d-%02d-01 00:00:00' % (y, m)
if fn.str2time(t0)<fn.str2time(tend) and \
fn.str2time(t1)>fn.str2time(tstart):
intervals.append((t0, t1, p))
if not partitions:
ts, te = fn.str2time(tstart), fn.str2time(tend)
tinc = (te - ts) / 10.
for i in range(1, 11):
intervals.append((fn.time2str(ts + (i - 1) * tinc),
fn.time2str(ts + i * tinc), None))
print('%d intervals in %s' % (len(intervals), table))
for t0, t1, p in intervals:
print((t0, t1))
if dry: continue
for a in attrs:
c0 = api.getTableSize(table)
pta.dbs.decimate_db_table(
db=api,
table=table,
start=fn.str2time(t0),
end=fn.str2time(t1),
period=600 if 'string' in table else 300,
condition=' att_conf_id = %s ' % a,
iteration=2000,
cols=['data_time', 'value_r'],
us=True,
repeated=True)
if p:
api.Query('alter table %s optimize partition %s' %
(table, p))
if not dry:
q = 'repair table %s;' % table
print('\n' + q)
api.Query(q)
c1 = api.getTableSize(table)
print('\n\n%s size reduced from %s to %s' % (table, c0, c1))
print('ellapsed %d seconds' % (time.time() - tt0))
def check_db_schema(schema, attributes = None, values = None,
tref = -12*3600, n = 1, filters = '*', export = 'json',
restart = False, subscribe = False):
"""
tref is the time that is considered updated (e.g. now()-86400)
n is used to consider multiple values
attrs: all attributes in db
on: archived
off: in db but not currently archived
ok: updated
known error causes (attrs not lost but not updated):
nok: attributes are not currently readable
noevs: attributes not sending events
novals: attributes never recorded a value
stall: not updated, but current value matches archiving
lost: not updated, and values doesn't match with current
"""
t0 = fn.now()
if hasattr(schema,'schema'):
api,schema = schema,api.schema
else:
api = pta.api(schema)
r = fn.Struct(api=api,schema=schema)
if isString(tref):
tref = fn.str2time(tref)
r.tref = fn.now()+tref if tref < 0 else tref
r.attrs = [a for a in (attributes or api.get_attributes())
if fn.clmatch(filters,a)]
print('check_db_schema(%s,attrs[%s],tref="%s",export as %s)'
% (schema,len(r.attrs),fn.time2str(r.tref),export))
if restart and schema!='hdbpc':
archs = [a for a in api.get_archivers() if not fn.check_device(a)]
if archs:
try:
print('Restarting archivers: %s' % str(archs))
astor = fn.Astor(archs)
astor.stop_servers()
astor.start_servers()
except:
traceback.print_exc()
stopped = api.get_stopped_attributes()
print('Restarting %d stopped attributes' % len(stopped))
api.restart_attributes(stopped)
r.on = [a for a in api.get_archived_attributes() if a in r.attrs]
r.off = [a for a in r.attrs if a not in r.on]
r.archs = fn.defaultdict(list)
r.pers = fn.defaultdict(list)
r.values = load_schema_values(api,r.on,values,n,tref=tref)
if schema in ('tdb','hdb'):
[r.archs[api[k].archiver].append(k) for k in r.on]
else:
r.rvals = r.values
r.freq, r.values = {}, {}
for k,v in r.rvals.items():
try:
if n > 1:
v = v[0] if isSequence(v) and len(v) else v
r.values[k] = v[0] if isSequence(v) and len(v) else v
r.freq[k] = v and float(len(v))/abs(v[0][0]-v[-1][0])
else:
r.values[k] = v
except Exception as e:
print(k,v)
print(fn.except2str())
for k in api.get_archivers():
r.archs[k] = api.get_archiver_attributes(k)
for k in api.get_periodic_archivers():
r.pers[k] = api.get_periodic_archivers_attributes(k)
# Get all updated attributes
r.ok = [a for a,v in r.values.items() if v and v[0] > r.tref]
# Try to read not-updated attributes
r.check = dict((a,fn.check_attribute(a)
) for a in r.on if a not in r.ok)
#r.novals = [a for a,v in r.values.items() if not v]
r.nok, r.stall, r.noevs, r.lost, r.novals, r.evs, r.rem = [],[],[],[],[],{},[]
# Method to compare numpy values
for a,v in r.check.items():
state = check_archived_attribute(a, v, default=CheckState.LOST,
cache=r, tref=r.tref,
check_events = subscribe and not api.is_periodic_archived(a))
{
#CheckState.ON : r.on,
#CheckState.OFF : r.off,
CheckState.OK : r.ok, #Shouldn't be any ok in check list
CheckState.NO_READ : r.nok,
CheckState.STALL : r.stall,
CheckState.NO_EVENTS : r.noevs,
CheckState.LOST : r.lost,
CheckState.UNK : r.novals,
}[state].append(a)
# SUMMARY
r.summary = schema +'\n'
r.summary += ','.join(
"""on: archived
off: not archived
ok: updated
nok: not readable
noevs: no events
novals: no values
stall: not changing
lost: not updated
""".split('\n'))+'\n'
getline = lambda k,v,l: '\t%s:\t:%d\t(%s)' % (k,len(v),l)
r.summary += '\n\t%s:\t:%d\tok+stall: %2.1f %%' % (
'attrs',len(r.attrs),
(100.*(len(r.ok)+len(r.stall))/(len(r.on) or 1e12)))
r.summary += '\n\t%s/%s:\t:%d/%d' % (
'on','off',len(r.on),len(r.off))
#if r.off > 20: r.summary+=' !!!'
r.summary += '\n\t%s/%s:\t:%d/%d' % (
'ok','nok',len(r.ok),len(r.nok))
if len(r.nok) > 10:
r.summary+=' !!!'
r.summary += '\n\t%s/%s:\t:%d/%d' % (
'noevs','novals',len(r.noevs),len(r.novals))
if len(r.novals) > 1:
r.summary+=' !!!'
r.summary += '\n\t%s/%s:\t:%d/%d' % (
'lost','stall',len(r.lost),len(r.stall))
if len(r.lost) > 1:
r.summary+=' !!!'
r.summary += '\n'
r.archivers = dict.fromkeys(api.get_archivers())
for d in sorted(r.archivers):
r.archivers[d] = api.get_archiver_attributes(d)
novals = [a for a in r.archivers[d] if a in r.novals]
lost = [a for a in r.archivers[d] if a in r.lost]
if (len(novals)+len(lost)) > 2:
r.summary += ('\n%s (all/novals/lost): %s/%s/%s'
% (d,len(r.archivers[d]),len(novals),len(lost)))
if hasattr(api,'get_periodic_archivers'):
r.periodics = dict.fromkeys(api.get_periodic_archivers())
for d in sorted(r.periodics):
r.periodics[d] = api.get_periodic_archiver_attributes(d)
novals = [a for a in r.periodics[d] if a in r.novals]
lost = [a for a in r.periodics[d] if a in r.lost]
if len(novals)+len(lost) > 2:
r.summary += ('\n%s (all/novals/lost): %s/%s/%s' %
(d,len(r.periodics[d]),len(novals),len(lost)))
r.perattrs = [a for a in r.on if a in api.get_periodic_attributes()]
r.notper = [a for a in r.on if a not in r.perattrs]
r.summary += '\nfinished in %d seconds\n\n'%(fn.now()-t0)
print(r.summary)
if restart:
try:
retries = r.lost+r.novals+r.nok
print('restarting %d attributes' % len(retries))
api.restart_attributes(retries)
except:
traceback.print_exc()
if export is not None:
if export is True:
export = 'txt'
for x in (export.split(',') if isString(export) else export):
if x in ('json','pck','pickle','txt'):
x = '/tmp/%s.%s' % (schema,x)
print('Saving %s file with keys:\n%s' % (x,r.keys()))
if 'json' in x:
fn.dict2json(r.dict(),x)
else:
f = open(x,'w')
if 'pck' in x or 'pickle' in x:
pickle.dump(r.dict(),f)
else:
f.write(fn.dict2str(r.dict()))
f.close()
for k,v in r.items():
if fn.isSequence(v):
r[k] = sorted(v)
return r
__doc__ = """
script to generate all test_method or test_class functions for a module
if a module is passed, docs and test cases are added to the resulting file
Usage:
gen_test_file.py [generate|check] [src/module] [dest]
"""
import sys, os, traceback, fandango
try:
outs = []
docs = fandango.defaultdict(str)
tests = fandango.defaultdict(str)
action = sys.argv[1]
fname = sys.argv[2]
modname = ''
if not os.path.exists(fname):
print('loading %s module' % fname)
mod = fandango.objects.loadModule(sys.argv[1])
fname = fandango.objects.findModule(sys.argv[1])
modname = mod.__name__
docs.update(
(k, getattr(v, '__doc__', '')) for k, v in mod.__dict__.items())
tests = getattr(mod, '__test__', {})
else:
SUMMARY_FIELDS = 'tag', 'state', 'priority', 'time', 'formula', 'message'
# Must be lists, not tuples
DATA_FIELDS = ('tag', 'device', 'priority', 'formula', 'message',
'annunciators')
STATE_FIELDS = ('state', 'time', 'counter', 'active', 'disabled',
'acknowledged', 'updated', 'last_sent', 'last_error')
#ALARM_ROW = ['tag','get_state','get_time','device','description']
#DEFAULT_COLUMNS = ['tag','get_state','active','get_time','severity']
VIEW_FIELDS = ['tag', 'device', 'state', 'priority', 'time']
CSV_FIELDS = 'tag,device,description,severity,receivers,formula'.split(',')
FORMATTERS = fd.defaultdict(lambda: str)
FORMATTERS.update({
'tag':
lambda s, l=10: ('{0:<%d}' % (l or 4)).format(s),
#'time' : lambda s,l=25: ('{:^%d}'%l).format(s),
'device':
lambda s, l=25: ('{0:^%d}' % (l or 4)).format(s),
'description':
lambda s, l=50: ('{0:<}').format(s),
'message':
lambda s, l=50: ('{0:<}').format(s),
'severity':
lambda s, l=10: ('{0:^%d}' % (l or 4)).format(s),
'priority':
lambda s, l=10: ('{0:^%d}' % (l or 4)).format(s),
'get_state':
def generate_class_properties(filein='ui_attribute_values.pck', all_rw=False):
print('generate_class_properties:' + str(filein))
devs = pickle.load(open(filein))
classes = defaultdict(Struct)
print('classes in %s are: %s' %
(filein, sorted(set(s.dev_class for s in devs.values()))))
filters = raw_input(
'Do you want to filter out some classes? [PyStateComposer]'
) or 'PyStateComposer'
for d, s in devs.items():
if s.dev_class in filters: continue
classes[s.dev_class].attrs = {}
classes[s.dev_class].comms = {}
classes[s.dev_class].values = defaultdict(set)
for d, s in devs.items():
if s.dev_class in filters:
continue
for a, t in s.attrs.items():
t['datatype'] = t.get('data_type', 'DevDouble')
if not isinstance(t, Struct): t = Struct(t)
if t.value is not None and not any(x in t.datatype.lower()
for x in ('array', )):
try:
classes[s.dev_class].values[a].add(t.value)
except:
print d, s.dev_class, a, t
for d, s in devs.items():
if s.dev_class in filters:
continue
#Iterate attributes
for a, t in s.attrs.items():
if a.lower() in ('state', 'status'):
continue
if t.value is None and a in classes[s.dev_class].attrs:
continue
else:
if t.value is None:
datatype, formula = 'DevDouble', 'NaN'
else:
datatype = t.datatype if t.data_format == 'SCALAR' else t.datatype.replace(
'Dev', 'DevVar') + 'Array'
if 'bool' in datatype.lower(): formula = DEFAULT_BOOL()
elif 'state' in datatype.lower():
formula = DEFAULT_STATE(
f='choice(%s or [0])' %
list(classes[s.dev_class].values[a]))
elif 'string' in datatype.lower():
formula = DEFAULT_STRING(
d=d,
a=a,
f='choice(%s or [0])' %
list(classes[s.dev_class].values[a]))
elif 'double' in datatype.lower(
) or 'float' in datatype.lower():
formula = DEFAULT_DOUBLE(f=random.choice(
list(classes[s.dev_class].values[a]) or [0]))
else:
formula = DEFAULT_INT(
f='choice(%s or [0])' %
list(classes[s.dev_class].values[a]))
if 'Array' in datatype:
formula = "[%s for i in range(10)]" % formula
if all_rw or 'WRITE' in t.writable:
formula = DEFAULT_WRITE(a=a, f=formula)
classes[s.dev_class].attrs[a] = '%s = %s(%s)' % (
a, datatype, formula)
#Iterate commands
for c, t in s.comms.items():
if fd.isMapping(t):
t = t['in_type'], t['out_type']
datatype = t[1] if t[1] != 'DevVoid' else 'DevString'
if 'bool' in datatype.lower(): formula = DEFAULT_BOOL()
elif 'state' in datatype.lower(): formula = DEFAULT_STATE()
elif 'string' in datatype.lower():
formula = DEFAULT_STRING(d=d, a=c)
elif 'double' in datatype.lower() or 'float' in datatype.lower():
formula = DEFAULT_DOUBLE()
else:
formula = DEFAULT_INT()
if 'Array' in datatype:
formula = "[%s for i in range(10)]" % formula
if 'DevVoid' not in t[0]: formula = DEFAULT_ARGS(f=formula)
classes[s.dev_class].comms[c] = '%s = %s(%s)' % (c, datatype,
formula)
classes[s.dev_class].states = DEFAULT_STATES
for k, t in classes.items():
print('\nWriting %s attributes ([%d])\n' % (k, len(t.attrs)))
f = open('%s_attributes.txt' % k, 'w')
for a in sorted(t.attrs.values()):
print('%s' % a)
f.write('%s\n' % a)
f.close()
print('\nWriting %s commands ([%d])\n' % (k, len(t.comms)))
f = open('%s_commands.txt' % k, 'w')
for a in sorted(t.comms.values()):
print('%s' % a)
f.write('%s\n' % a)
f.close()
print('\nWriting %s states ([%d])\n' % (k, len(t.states)))
f = open('%s_states.txt' % k, 'w')
for a in t.states:
print('%s' % a)
f.write('%s\n' % a)
f.close()
return (filein)
def decimate(db_name,keys,tstart,tend,period=10,dry=False):
"""
time arguments are strings
BUT!, this method seems to not work anymore to free space in TDB
Maybe a mysqld restart is needed, I don't know;
but up to now space is not freed
"""
api = pta.api(db_name)
if '/' in keys[0]:
print('Decimating by attribute names')
tables = fn.defaultdict(list)
for a in keys:
api.get_attr_id_type_table(a)
tables[api[a].table].append(a)
print('tables: %s' % (tables.keys()))
for table,attrs in tables.items():
for a in attrs:
pta.dbs.decimate_db_table_by_time(api,
table,api[a].id,tstart,tend,period,
optimize=(a==attrs[-1]))
if not '/' in keys[0]:
print('Decimating by data_type')
data_types = keys
if not data_types:
data_types = [r[0] for r in api.Query('select data_type from att_conf_data_type')]
else:
data_types = [d.replace('att_','') for d in data_types]
print('Decimating %s types between %s and %s: %s'%(db_name,tstart,tend,data_types))
for data_type in data_types:
attrs = api.Query('select att_conf_id from att_conf,att_conf_data_type '
'where att_conf.att_conf_data_type_id = att_conf_data_type.att_conf_data_type_id '
'and data_type = "%s"'%data_type)
attrs = [r[0] for r in attrs]
q = ("select partition_name,table_name"
" from information_schema.partitions where"
" partition_name is not NULL"
" and table_schema = '%s'"%db_name +
" and table_name like '%"+data_type+"'" )
print(q)
partitions = api.Query(q)
if partitions:
table = partitions[0][1]
else:
table = 'att_'+data_type
print('%s has %d attributes in %d partitions'%(table,len(attrs),len(partitions)))
c0 = api.Query('select count(*) from %s '%table)
import re
intervals = []
for p in partitions:
p = p[0]
r = '(?P<year>[0-9][0-9][0-9][0-9])(?P<month>[0-9][0-9])'
md = re.search(r,p).groupdict()
t0 = '%s-%s-01 00:00:00'%(md['year'],md['month'])
m,y = int(md['month']),int(md['year'])
if m == 12:
m,y = 1, y+1
else:
m+=1
t1 = '%04d-%02d-01 00:00:00'%(y,m)
if fn.str2time(t0)<fn.str2time(tend) and \
fn.str2time(t1)>fn.str2time(tstart):
intervals.append((t0,t1,p))
if not partitions:
ts,te = fn.str2time(tstart),fn.str2time(tend)
tinc = (te-ts)/10.
for i in range(1,11):
intervals.append((fn.time2str(ts+(i-1)*tinc),
fn.time2str(ts+i*tinc),None))
print('%d intervals in %s'%(len(intervals),table))
for t0,t1,p in intervals:
print((t0,t1))
if dry: continue
for a in attrs:
c0 = api.getTableSize(table)
pta.dbs.decimate_db_table(db=api,table=table,
start=fn.str2time(t0),end=fn.str2time(t1),
period=600 if 'string' in table else 300,
condition=' att_conf_id = %s '%a,
iteration=2000,cols=['data_time','value_r'],
us=True, repeated=True)
if p: api.Query('alter table %s optimize partition %s'%(table,p))
if not dry:
q = 'repair table %s;'%table
print('\n'+q)
api.Query(q)
c1 = api.getTableSize(table)
print('\n\n%s size reduced from %s to %s'%(table,c0,c1))
print('ellapsed %d seconds'%(time.time()-tt0))
def check_db_schema(schema, tref=None):
r = fn.Struct()
r.api = api = pta.api(schema)
r.tref = fn.notNone(tref, fn.now() - 3600)
r.attrs = api.keys()
r.on = api.get_archived_attributes()
r.off = [a for a in r.attrs if a not in r.on]
if schema in ('tdb', 'hdb'):
ups = api.db.get_table_updates()
r.vals = dict((k, (ups[api[k].table], None)) for k in r.on)
else:
r.vals = dict(fn.kmap(api.load_last_values, r.on))
r.vals = dict((k, v and v.values()[0]) for k, v in r.vals.items())
dups = fn.defaultdict(list)
if getattr(api, 'dedicated', None):
[
dups[a].append(k) for a in r.on for k, v in api.dedicated.items()
if a in v
]
nups = [a for a, v in dups.items() if len(v) <= 1]
[dups.pop(a) for a in nups]
r.dups = dict(dups)
# Get all updated attributes
r.ok = [a for a, v in r.vals.items() if v and v[0] > r.tref]
# Try to read not-updated attributes
r.check = dict((a, fn.check_attribute(a)) for a in r.on if a not in r.ok)
r.nok, r.stall, r.noev, r.lost, r.evs = [], [], [], [], {}
# Method to compare numpy values
fbool = lambda x: all(x) if fn.isSequence(x) else bool(x)
for a, v in r.check.items():
# Get current value/timestamp
vv, t = getattr(v, 'value', v), getattr(v, 'time', 0)
t = t and fn.ctime2time(t)
if isinstance(vv, (type(None), Exception)):
# attribute is not readable
r.nok.append(a)
elif r.vals[a] and 0 < t <= r.vals[a][0]:
# attribute timestamp doesnt change
r.stall.append(a)
elif r.vals[a] and fbool(vv == r.vals[a][1]):
# attribute value doesnt change
r.stall.append(a)
else:
r.evs[a] = fn.tango.check_attribute_events(a)
if not r.evs[a]:
# attribute doesnt send events
r.noev.append(a)
else:
# archiving failure (events or polling)
r.lost.append(a)
# SUMMARY
print(schema)
for k in 'attrs on off dups ok nok noev stall lost'.split():
print('\t%s:\t:%d' % (k, len(r.get(k))))
return r
