def get_attributes_row_counts(db, attrs='*', start=-86400, stop=-1, limit=0):
"""
DUPLICATED BY HDBPP.get_attribute_rows !!!
It will return matching $attrs that recorded more than $limit values in
the $start-$stop period::
countsrf = get_attributes_row_counts('hdbrf',start=-3*86400,limit=20000)
"""
db = pta.api(db) if fn.isString(db) else db
start = start if fn.isString(start) else fn.time2str(start)
stop = stop if fn.isString(stop) else fn.time2str(stop)
if fn.isString(attrs):
attrs = [a for a in db.get_attributes() if fn.clmatch(attrs, a)]
r = {}
for a in attrs:
i, t, b = db.get_attr_id_type_table(a)
l = db.Query("select count(*) from %s where att_conf_id = %d"
" and data_time between '%s' and '%s'" %
(b, i, start, stop))
c = l[0][0] if len(l) else 0
if c >= limit:
r[a] = c
return r
def get_attributes_row_counts(db,attrs='*',start=-86400,stop=-1,limit=0):
"""
DUPLICATED BY HDBPP.get_attribute_rows !!!
It will return matching $attrs that recorded more than $limit values in
the $start-$stop period::
countsrf = get_attributes_row_counts('hdbrf',start=-3*86400,limit=20000)
"""
db = pta.api(db) if fn.isString(db) else db
start = start if fn.isString(start) else fn.time2str(start)
stop = stop if fn.isString(stop) else fn.time2str(stop)
if fn.isString(attrs):
attrs = [a for a in db.get_attributes() if fn.clmatch(attrs,a)]
r = {}
for a in attrs:
i,t,b = db.get_attr_id_type_table(a)
l = db.Query("select count(*) from %s where att_conf_id = %d"
" and data_time between '%s' and '%s'" % (b,i,start,stop))
c = l[0][0] if len(l) else 0
if c >= limit:
r[a] = c
return r
def getApi(k, schema):
if schema == '*':
import PyTangoArchiving.reader
return PyTangoArchiving.reader.Reader()
elif fn.isString(schema):
schema = k.getSchema(schema)
if schema is not None:
api = schema.get('api', 'PyTangoArchiving.ArchivingAPI')
if fn.isString(api):
api = k._load_object(api, schema)
return api(schema['schema']) if isinstance(api, type) else api
else:
return schema
def mysqldump_by_date(schema, user, passwd, folder, start, stop,
compress = True, delete = True):
"""
This method creates a backup between selected dates for each table
of the selected database.
All dump files are exported to the same folder, and a compressed file
is created at the end.
Deleting of temporary files created (folder/*dmp) must be done manually.
"""
print('mysqldump_by_date(%s,,,folder=%s,%s,%s,compress=%s,delete=%s)'
% (schema, folder, start, stop, compress, delete))
db = FriendlyDB(schema,user=user,passwd=passwd)
t,e = start,stop
print(t,e)
start = start if fn.isString(start) else fn.time2str(start)
stop = stop if fn.isString(stop) else fn.time2str(stop)
tables = db.getTables()
print('mysqldump_by_date(%s): %d tables to backup between %s and %s'
% (schema,len(tables),start,stop))
if not os.path.isdir(folder):
print('mkdir %s' % folder)
os.mkdir(folder)
for t in tables:
filename = ('%s/%s-%s-%s-%s.dmp'
% (folder,schema,t,start.split()[0],stop.split()[0]))
cols = db.getTableCols(t)
col = [c for c in ('time','data_time') if c in cols]
if col:
where = " %s >= '%s' and %s < '%s' " % (col[0],start,col[0],stop)
else:
where = ""
mysqldump(schema,user,passwd,filename,t,where)
ext = ('part.' if fn.str2time(stop) > fn.now() else '') + 'tgz'
if compress:
filename = ('%s/%s-%s-%s.%s'
% (folder,schema,start.split()[0],stop.split()[0],ext))
cmd = 'tar zcvf %s %s/*.dmp' % (filename,folder)
print(cmd)
fn.linos.shell_command(cmd)
if compress and delete:
cmd = 'rm -rf %s/*.dmp' % folder
print(cmd)
fn.linos.shell_command(cmd)
return filename
def getReader(k, schema, dct=None):
# This method initializes a reader object from Schema config
# It does not update the Schema object, just returns a reader
dct = dct if dct is not None else k.getSchema(
schema if fn.isString(schema) else schema.get('schema'))
rd = dct.get('reader', dct.get('api'))
if rd and isinstance(rd, str):
try:
#print('Schemas.getReader(%s): instantiating reader' % schema)
rd = k._load_object(rd, dct)
#print('getReader(%s): %s' % (schema,type(rd)))
if not hasattr(rd, 'is_attribute_archived'):
rd.is_attribute_archived = lambda *a, **k: True
if not hasattr(rd, 'get_attributes'):
rd.get_attributes = lambda *a, **k: []
if not hasattr(rd, 'get_attribute_values'):
if dct['method']:
rd.get_attribute_values = getattr(rd, dct['method'])
if not hasattr(rd, 'schema'):
rd.schema = schema
except:
print('getReader(%s) failed!' % schema)
#traceback.print_exc()
rd = None
return rd
def get_average_read_time(api='hdb',period=10*3600*24,N=100):
if fn.isString(api):
import PyTangoArchiving
api = PyTangoArchiving.ArchivingAPI(api)
reader = api.get_reader()
active = [a for a in api.get_archived_attributes() if api[a].data_type not in (1,8)]
target = [active[i] for i in fn.randomize(range(len(active)))][:int(2*N)]
stats = []
navg,tavg,count = 0,0,0
print('testing %s %s attributes'%(len(target),api.schema))
for t in target:
if count == N:
break
t0 = time.time()
try:
vs = reader.get_attribute_values(t,time.time()-period-3600,time.time()-3600)
if not len(vs):
continue
except:
continue
t1 = time.time()-t0
if not count%10:
print(count,':',t,len(vs),t1)
navg += len(vs)
tavg += t1
count += 1
stats.append((t1,len(vs),t))
N = float(count)
print('Worst tread were: \n%s'%'\n'.join(map(str,sorted(stats)[-10:])))
return (N, (N>0 and navg/N),(N>0 and tavg/N))
def get_submodules(self, module=None, nr=0):
print('get_submodules(%s)' % module)
try:
module = module or self.module
if fn.isString(module):
m = self.load_module(module)
else:
m, module = module, module.__name__
result = set()
l = getattr(m, '__test__', dir(m))
print m, l
l = list(l)
for o in l:
o = o.split('.')[-1]
n = getattr(m, o)
if self.is_module(n) and module == n.__package__:
o = module + '.' + o
result.add(o)
if nr < 10:
result = result.union(self.get_submodules(n,
nr=nr + 1))
except Exception, e:
print('get_submodules(%s)' % module)
traceback.print_exc()
raise e
def call(args=None,locals_=None):
"""
Calls a method from local scope parsing a pipe-like argument list
"""
if args is None:
import sys
args = sys.argv[1:]
f,args = args[0],args[1:]
print(f,args)
if not isCallable(f):
locals_ = locals_ or globals()
if f=='help':
if args and args[0] in locals_:
n,o = args[0],locals_[args[0]]
if hasattr(o,'func_code'):
n = n+str(o.func_code.co_varnames)
return '%s:\n%s' % (n,o.__doc__)
else:
m = [k for k,v in locals_.items() if isCallable(v)]
return ('\n'.join(sorted(m,key=str.lower)))
f = locals_.get(f,None)
if all(isString(a) for a in args):
args = map(str2type,args)
return f(*args)
def call(args=None, locals_=None):
"""
Calls a method from local scope parsing a pipe-like argument list
"""
if args is None:
import sys
args = sys.argv[1:]
f, args = args[0], args[1:]
print(f, args)
if not isCallable(f):
locals_ = locals_ or globals()
if f == 'help':
if args and args[0] in locals_:
n, o = args[0], locals_[args[0]]
if hasattr(o, 'func_code'):
n = n + str(o.func_code.co_varnames)
return '%s:\n%s' % (n, o.__doc__)
else:
m = [k for k, v in locals_.items() if isCallable(v)]
return ('\n'.join(sorted(m, key=str.lower)))
f = locals_.get(f, None)
if all(isString(a) for a in args):
args = map(str2type, args)
return f(*args)
def get_average_read_time(api='hdb',period=10*3600*24,N=100):
if fandango.isString(api):
import PyTangoArchiving
api = PyTangoArchiving.ArchivingAPI(api)
reader = api.get_reader()
active = [a for a in api.get_archived_attributes() if api[a].data_type not in (1,8)]
target = [active[i] for i in fandango.randomize(range(len(active)))][:int(2*N)]
stats = []
navg,tavg,count = 0,0,0
print('testing %s %s attributes'%(len(target),api.schema))
for t in target:
if count == N:
break
t0 = time.time()
try:
vs = reader.get_attribute_values(t,time.time()-period-3600,time.time()-3600)
if not len(vs):
continue
except:
continue
t1 = time.time()-t0
if not count%10:
print(count,':',t,len(vs),t1)
navg += len(vs)
tavg += t1
count += 1
stats.append((t1,len(vs),t))
N = float(count)
print('Worst tread were: \n%s'%'\n'.join(map(str,sorted(stats)[-10:])))
return (N, (N>0 and navg/N),(N>0 and tavg/N))
def load_from_device(self, device):
"""
This method allows to load ModbusMap values from a running device server
"""
if fn.isString(device): device = fn.tango.get_device(device)
self.load(fn.tango.get_device_property(device.name(), 'Mapping'))
[self[k].set(list(getattr(device, k))) for k in self]
def load_last_values(self,
attributes=None,
n=1,
epoch=None,
tref=90 * 86400):
"""
load_last_values provided to comply with Reader API
get_last_attribute_values provided to comply with CommonAPI
load_last_values returns a dictionary {attr:(last_time,last_value)}
attributes: attribute name or list
n: the number of last values to be retorned
tref: time from which start searching values (-1d by default)
epoch: end of window to search values (now by default)
"""
if attributes is None:
attributes = self.get_archived_attributes()
if epoch is not None:
epoch = fn.str2time(epoch) if fn.isString(epoch) else epoch
kwargs = {
'epoch': epoch,
'period': (epoch - tref) if tref > 1e9 else abs(tref)
}
else:
kwargs = {}
vals = dict((a, self.get_last_attribute_values(a, n=n, **kwargs))
for a in fn.toList(attributes))
return vals
def get_deactivated_attributes(api='hdb',
updates=None,
period=6 * 30 * 24 * 3600):
# Returns the list of attributes that are not archived despite readable and having data from the last months
if fun.isString(api): api = pta.api(api)
if updates is None: updates = get_table_updates(api)
now = fun.time.time()
return sorted(
a for a, t in updates.items()
if (now - period) < t < (now - 24 * 3600) and fun.check_attribute(a))
def checkSchema(k, schema, attribute='', start=None, stop=None):
schema = k.getSchema(schema)
if not schema:
return False
f = schema.get('check')
if not f:
print('%s has no check function' % str(schema))
return True
try:
now = time.time()
start = (str2time(start) if fn.isString(start)
else fn.notNone(start,now-1))
stop = (str2time(stop) if fn.isString(stop)
else fn.notNone(stop,now))
k.LOCALS.update({'attribute':attribute.lower(),
'match':clmatch,'clmatch':clmatch,
'overlap':overlap,
'time2str':time2str,'str2time':str2time,
't2s':time2str,'s2t':str2time,
'start':start,'stop':stop,'now':now,
'begin':start,'end':stop,'NOW':now,
'reader':schema.get('reader',schema.get('api')),
'schema':schema.get('schema'),
'dbname':schema.get('dbname',schema.get('schema','')),
})
if 'reader' in f:
k.getReader(schema)
if 'api' in f:
k.getApi(schema)
#print('In reader.Schemas.checkSchema(%s,%s,%s,%s): %s'
#% (schema,attribute,start,stop,f))
#print('(%s)%%(%s)'%(f,[t for t in k.LOCALS.items() if t[0] in f]))
v =fn.evalX(f,k.LOCALS,k.MODULES)
except:
print('checkSchema(%s,%s) failed!' % (schema,attribute))
traceback.print_exc()
v = False
#print('checkSchema(%s): %s'%(schema,v))
return v
def generate_partition_name_for_date(self, table, date):
"""
generates the matching partition name for the date given
"""
if not fn.isString(date):
date = fn.time2str(date)
p = partition_prefixes.get(table, None)
if p:
p += ''.join(date.split('-')[0:2]) + '01'
return p
def get_idle_servers(api='hdb'):
idle = dict()
if fun.isString(api): api = pta.api(api)
for s, t in api.servers.items():
if 'archiver' not in s: continue
for d in t.get_device_list():
if not fun.check_device(d):
idle[s] = [d]
break
trace('\t%d servers have idle devices' % len(idle))
return idle
def getWindow(klass,window):
if not fandango.isString(window):
if window in klass.WINDOWS:
return window
else:
for w in klass.WINDOWS:
try:
if w.windowTitle()==window:
return w
except:pass
return None
def restart_server(ss,api='hdb',wait=WAIT_TIME):
if fun.isString(api): api = pta.api('hdb')
trace('restart_server(%s)'%ss)
api.servers.stop_servers(ss)
time.sleep(STOP_WAIT)
api.servers.start_servers(ss,wait=0.1)
last_restart = time.time()
if wait:
print '\tWaiting %s seconds' %wait
time.sleep(wait)
return ss
def getWindow(klass,window):
if not fandango.isString(window):
if window in klass.WINDOWS:
return window
else:
for w in klass.WINDOWS:
try:
if w.windowTitle()==window:
return w
except:pass
return None
def restart_server(ss, api='hdb', wait=WAIT_TIME):
if fun.isString(api): api = pta.api('hdb')
trace('restart_server(%s)' % ss)
api.servers.stop_servers(ss)
time.sleep(STOP_WAIT)
api.servers.start_servers(ss, wait=0.1)
last_restart = time.time()
if wait:
print '\tWaiting %s seconds' % wait
time.sleep(wait)
return ss
def get_idle_servers(api='hdb'):
idle = dict()
if fun.isString(api): api = pta.api(api)
for s,t in api.servers.items():
if 'archiver' not in s: continue
for d in t.get_device_list():
if not fun.check_device(d):
idle[s] = [d]
break
trace('\t%d servers have idle devices'%len(idle))
return idle
def get_attr_id_type_table(self, attr):
if fn.isString(attr):
attr = fn.tango.get_full_name(attr, True).lower()
try:
s = self.attributes[attr]
return s.id, s.type, s.table
except:
self.get_att_conf_table.cache.clear()
self.get_att_conf_table()
if attr not in self.attributes:
return None, None, ''
else:
s = self.attributes[attr]
return s.id, s.type, s.table
def setModels(self, args):
#self.clear()
if len(args) == 1:
args = args[0]
if fn.isString(args):
args = fn.find_attributes(args)
print('setModels(%s)' % args)
for m in args:
w = TaurusSingleValue()
w.setModel(m)
self.main.layout().addWidget(w)
self.widgets.append(w)
#self.main.show()
self.setWidget(self.main)
#self.resize(400,600)
self.show()
def get_table_partitions(api, table, description=''):
"""
DUPLICATED BY pta.dbs.get_partitions_from_query !!!
"""
if fn.isString(api): api = pta.api(api)
if not description: description = get_table_description(api, table)
rows = [l for l in description.split('\n') if 'partition' in l.lower()]
f = rows[0].split()[-1]
data = (f, [])
for i, l in enumerate(rows[1:]):
try:
l, n = l.split(), i and rows[i].split() or [0] * 6
data[-1].append((l[1], n[5], l[5]))
except:
print(fn.except2str())
print(i, l, n)
return (data)
def get_table_partitions(api,table,description=''):
"""
DUPLICATED BY pta.dbs.get_partitions_from_query !!!
"""
if fn.isString(api): api = pta.api(api)
if not description: description = get_table_description(api,table)
rows = [l for l in description.split('\n') if 'partition' in l.lower()]
f = rows[0].split()[-1]
data = (f,[])
for i,l in enumerate(rows[1:]):
try:
l,n = l.split(),i and rows[i].split() or [0]*6
data[-1].append((l[1],n[5],l[5]))
except:
print(fn.except2str())
print(i,l,n)
return(data)
def get_last_attribute_values(self,table,n,check_table=False,epoch=fn.END_OF_TIME):
"""
Check table set to False as sometimes order of insertion is not the same as expected, BE CAREFUL WITH THIS ARGUMENT!
"""
query,where = table,''
if check_table:
table_size = self.getTableSize(table)
if table_size>1e3:
x = max((2*n,20))
query = '(select * from %s limit %d,%d)'%(table,table_size-x,x)
epoch = fn.str2time(epoch) if fn.isString(epoch) else epoch
if epoch not in (None, fn.END_OF_TIME):
where = " where T.time < '%s' " % (fn.time2str(epoch))
what = 'SELECT time'
what += (',value',',read_value')['read_value' in self.getTableCols(table)]
return self.Query('%s from %s T %s order by T.time desc limit %d' % (
what, query, where, n))
def get_attributes_servers(attr_list, api='hdb'):
if fun.isString(api): api = pta.api(api)
api.load_dedicated_archivers(check=False)
from collections import defaultdict
devices = defaultdict(set)
servers = defaultdict(set)
[
devices[x].add(l) for l in attr_list
for x in (api[l].archiver, api[l].dedicated) if x
]
#assigned = set(x for l in attr_list for x in (api[l].archiver,api[l].dedicated) if x)
[
servers[api.servers.get_device_server(d).lower()].update(devices[d])
for d in devices
]
#servers = sorted(set(api.servers.get_device_server(a) for a in set(x for l in attr_list for x in (api[l].archiver,api[l].dedicated) if x)))
return servers
def get_attributes_servers(attr_list,api='hdb',dedicated=False):
if fun.isString(api):
api = pta.api(api)
if dedicated:
api.load_dedicated_archivers(check=False);
devices = defaultdict(set)
servers = defaultdict(set)
[devices[x].add(l) for l in attr_list
for x in (api[l].archiver,api[l].dedicated) if x]
#assigned = set(x for l in attr_list
# for x in (api[l].archiver,api[l].dedicated) if x)
[servers[api.servers.get_device_server(d).lower()].update(devices[d])
for d in devices]
#servers = sorted(set(api.servers.get_device_server(a)
# for a in set(x for l in attr_list
# for x in (api[l].archiver,api[l].dedicated) if x)))
return servers
def launch_app(self, app=None):
args = self.args
try:
print('launch_app(%s)' % app)
if not fn.isString(app):
if hasattr(app, 'text'):
app = str(app.text()).strip()
else:
app = str(app)
print('launch_app(%s)' % app)
args = str(args.text()).strip()
#-hold
app = "xterm -e %s" % app
if check.isChecked():
args = TF + " " + args
c = "%s %s &" % (app, args)
print('launch_app(%s)' % c)
print('>' * 80)
fn.shell_command(c)
except:
traceback.print_exc()
def generate_rest_files(module,path='source'):
print '\n'*5
print 'Writing documentation settings to %s/*rst' % (path)
if fandango.isString(module): module = fandango.loadModule(module)
submodules = [(o,v) for o,v in vars(module).items()
if inspect.ismodule(v) and v.__name__.startswith(module.__name__)]
for o,v in submodules:
filename = path+'/'+o+'.rst'
if not os.path.isfile(filename):
print('writing %s'%filename)
open(filename,'w').write(DEFAULT_MODULE%(v.__name__,'='*len(v.__name__),v.__name__))
print('\nWrite this into index.rst:\n')
print("""
.. toctree::
:maxdepth: 2
"""+
'\n '.join([t[0] for t in submodules]))
def generate_rest_files(module,path='source'):
import fandango
print '\n'*5
print 'Writing documentation settings to %s/*rst' % (path)
if fandango.isString(module): module = fandango.loadModule(module)
submodules = [(o,v) for o,v in vars(module).items()
if inspect.ismodule(v) and v.__name__.startswith(module.__name__)]
for o,v in submodules:
filename = path+'/'+o+'.rst'
if not os.path.isfile(filename):
print('writing %s'%filename)
open(filename,'w').write(DEFAULT_MODULE%(v.__name__,'='*len(v.__name__),v.__name__))
print('\nWrite this into index.rst:\n')
print("""
.. toctree::
:maxdepth: 2
"""+
'\n '.join([t[0] for t in submodules]))
def check_attributes(self, attrs='', load=False, t0=0):
db, t0, result, vals = self, t0 or fn.now(), {}, {}
print('Checking %s' % str(db))
if fn.isDictionary(attrs):
attrs, vals = attrs.keys(), attrs
if isinstance(vals.values()[0], dict):
vals = dict((k, v.values()[0]) for k, v in vals.items())
else:
if fn.isString(attrs):
attrs = fn.filtersmart(db.get_attributes(), attrs)
load = True
if load:
[vals.update(db.load_last_values(a)) for a in attrs]
print('\t%d attributes' % len(attrs))
result['attrs'] = attrs
result['vals'] = vals
result['novals'] = [a for a, v in vals.items() if not v]
result['nones'], result['down'], result['lost'] = [], [], []
for a, v in vals.items():
if not v or [1] is None:
if not fn.read_attribute(a): #USE read not check!!
result['down'].append(a)
else:
result['novals' if not v else 'nones'].append(a)
elif v[0] < (t0 - 7200):
result['lost'].append(a)
print('\t%d attributes have no values' % len(result['novals']))
print('\t%d attributes are not readable' % len(result['down']))
print('\t%d attributes are not updated' % len(result['lost']))
print('\t%d attributes have None values' % len(result['nones']))
return result
def check_attributes(self,attrs = '', load = False, t0 = 0):
db,t0,result,vals = self,t0 or fn.now(),{},{}
print('Checking %s' % str(db))
if fn.isDictionary(attrs):
attrs,vals = attrs.keys(),attrs
if isinstance(vals.values()[0],dict):
vals = dict((k,v.values()[0]) for k,v in vals.items())
else:
if fn.isString(attrs):
attrs = fn.filtersmart(db.get_attributes(),attrs)
load = True
if load:
[vals.update(db.load_last_values(a)) for a in attrs]
print('\t%d attributes'%len(attrs))
result['attrs'] = attrs
result['vals'] = vals
result['novals'] = [a for a,v in vals.items() if not v]
result['nones'],result['down'],result['lost'] = [],[],[]
for a,v in vals.items():
if not v or [1] is None:
if not fn.read_attribute(a): #USE read not check!!
result['down'].append(a)
else:
result['novals' if not v else 'nones'].append(a)
elif v[0] < (t0 - 7200):
result['lost'].append(a)
print('\t%d attributes have no values'%len(result['novals']))
print('\t%d attributes are not readable'%len(result['down']))
print('\t%d attributes are not updated'%len(result['lost']))
print('\t%d attributes have None values'%len(result['nones']))
return result
def get_submodules(self,module=None,nr=0):
print('get_submodules(%s)'%module)
try:
module = module or self.module
if fn.isString(module):
m = self.load_module(module)
else:
m,module = module,module.__name__
result = set()
l = getattr(m,'__test__',dir(m))
print m,l
l = list(l)
for o in l:
o = o.split('.')[-1]
n = getattr(m,o)
if self.is_module(n) and module == n.__package__:
o = module+'.'+o
result.add(o)
if nr<10:
result = result.union(self.get_submodules(n,nr=nr+1))
except Exception,e:
print('get_submodules(%s)'%module)
traceback.print_exc()
raise e
def check_archiving_performance(schema='hdb',attributes=[],period=24*3600*90,\
exclude=['*/waveid','*/wavename','*/elotech-*'],action=False,trace=True):
import PyTangoArchiving as pta
import fandango as fn
ti = fn.now()
api = pta.api(schema)
check = dict()
period = 24*3600*period if period < 1000 else (24*period if period<3600 else period)
attributes = fn.get_matching_attributes(attributes) if fn.isString(attributes) else map(str.lower,attributes)
tattrs = [a for a in api if not attributes or a in attributes]
excluded = [a for a in tattrs if any(fn.clmatch(e,a) for e in exclude)]
tattrs = [a for a in tattrs if a not in excluded]
#Getting Tango devices currently not running
alldevs = set(t.rsplit('/',1)[0] for t in tattrs if api[t].archiver)
tdevs = filter(fn.check_device,alldevs)
nodevs = [d for d in alldevs if d not in tdevs]
#Updating data from archiving config tables
if not attributes:
tattrs = sorted(a for a in api if a.rsplit('/',1)[0] in tdevs)
tattrs = [a for a in tattrs if not any(fn.clmatch(e,a) for e in exclude)]
print('%d attributes will not be checked (excluded or device not running)'%(len(api)-len(tattrs)))
tarch = sorted(a for a in api if api[a].archiver)
tnoread = sorted(t for t in tarch if t not in tattrs)
check.update((t,None) for t in tnoread)
#Getting attributes archived in the past and not currently active
tmiss = [t for t in tattrs if not api[t].archiver]
check.update((t,fn.check_attribute(t,readable=True)) for t in tmiss)
tmiss = [t for t in tmiss if check[t]]
tmarray = [t for t in tmiss if fn.isString(check[t].value) or fn.isSequence(check[t].value)]
tmscalar = [t for t in tmiss if t not in tmarray]
#Getting updated tables from database
tups = pta.utils.get_table_updates(schema)
# Some tables do not update MySQL index tables
t0 = [a for a in tarch if a in tattrs and not tups[api[a].table]]
check.update((t,check_attribute(a,readable=True)) for t in t0 if not check.get(t))
t0 = [t for t in t0 if check[t]]
print('%d/%d archived attributes have indexes not updated ...'%(len(t0),len(tarch)))
if t0 and len(t0)<100:
vs = api.load_last_values(t0);
tups.update((api[t].table,api[t].last_date) for t in t0)
tnotup = [a for a in tarch if tups[api[a].table]<fn.now()-1800]
check.update((t,1) for t in tarch if t not in tnotup)
#Updating readable attributes (all updated are considered as readable)
tread = sorted(t for t in tattrs if t not in tnoread)
for t in tattrs:
if t not in check:
check[t] = fn.check_attribute(t,readable=True)
tread = sorted(t for t in tattrs if check[t])
tnoread.extend(t for t in tread if not check[t])
tnoread = sorted(set(tnoread))
#tread contains all readable attributes from devices with some attribute archived
#tnoread contains all unreadable attributes from already archived
#Calcullating all final stats
#tok will be all archivable attributes that are archived
#tnotup = [a for a in tnotup if check[a]]
#tok = [t for t in tread if t in tarch and t not in tnotup]
tok = [t for t in tarch if t not in tnotup]
readarch = [a for a in tread if a in tarch]
treadnotup = [t for t in readarch if t in tnotup] #tnotup contains only data from tarch
tokread = [t for t in readarch if t not in tnotup] #Useless, all archived are considered readable
tarray = [t for t in tarch if check[t] and get_attribute_pytype(t) in (str,list)]
removed = [a for a in tattrs if not api[a].archiver and tups[api[a].table]>fn.now()-period]
result = fn.Struct()
result.Excluded = excluded
result.Schema = schema
result.All = api.keys()
result.Archived = tarch
result.Readable = tread
result.ArchivedAndReadable = readarch
result.Updated = tok #tokread
result.Lost = treadnotup
result.Removed = removed
result.TableUpdates = tups
result.NotUpdated = tnotup
result.Missing = tmiss
result.MissingScalars = tmscalar
result.MissingArrays = tmarray
result.ArchivedArray = tarray
result.Unreadable = tnoread
result.DeviceNotRunning = nodevs
get_ratio = lambda a,b:float(len(a))/float(len(b))
result.ArchRatio = get_ratio([t for t in readarch if t not in tnotup],readarch)
result.ReadRatio = get_ratio(result.Readable,tattrs)
result.LostRatio = get_ratio([a for a in tread if a in tnotup],tread)
result.MissRatio = get_ratio([a for a in tread if a not in tarch],tread)
result.OkRatio = 1.0-result.LostRatio-result.MissRatio
result.Summary = '\n'.join((
('Checking archiving of %s attributes'%(len(attributes) if attributes else schema))
,('%d attributes in %s, %d are currently active'%(len(api),schema,len(tarch)))
,('%d devices with %d archived attributes are not running'%(len(nodevs),len([a for a in api if a.rsplit('/',1) in nodevs])))
,('%d archived attributes (%2.1f %%) are unreadable! (check and remove)'%(len(tnoread),1e2*get_ratio(tnoread,tarch)))
,('%d readable attributes are not archived'%(len(tmiss)))
,('%d attributes (readable or not) are updated (%2.1f %% of all readables)'%(len(tok),1e2*result.OkRatio))
,('-'*80)
,('%d archived attributes (readable or not) are not updated!'%len(tnotup))
,('%d archived and readable attributes are not updated! (check and restart?)'%len(treadnotup))
,('-'*80)
,('%d readable attributes have been removed in the last %d days!'%(len(removed),period/(24*3600)))
,('%d readable scalar attributes are not being archived (not needed anymore?)'%len(tmscalar))
,('%d readable array attributes are not being archived (Ok)'%len(tmarray))
,('%d readable array attributes are archived (Expensive)'%len(tarray))
,('')))
if trace: print(result.Summary)
print('%d readable lost,Ok = %2.1f%%, %2.1f %% over all Readables (%2.1f %% of total)'%\
(len(treadnotup),1e2*result.ArchRatio,1e2*result.OkRatio,1e2*result.ReadRatio))
if action:
print('NO ACTIONS ARE GONNA BE EXECUTED, AS THESE ARE ONLY RECOMMENDATIONS')
print("""
api = PyTangoArchiving.ArchivingAPI('%s')
lostdevs = sorted(set(api[a].archiver for a in result.NotUpdated))
print(lostdevs)
if lostdevs < a_reasonable_number:
astor = fn.Astor()
astor.load_from_devs_list(lostdevs)
astor.stop_servers()
fn.time.sleep(10.)
astor.start_servers()
"""%schema)
if trace: print('finished in %d seconds'%(fn.now()-ti))
return result
def check(self, method=None):
method = method or self.get_data_types
if fn.isString(method):
return self.Query(method)
else:
return method()
def check_archiving_schema(
schema='hdb',
attributes=[],values={},
ti = None,
period = 7200,
old_period=24*3600*90,\
exclude=['*/waveid','*/wavename','*/elotech-*'],
use_index = True,
loads = True,
action=False,
trace=True,
export=None):
ti = fn.now() if ti is None else str2time(ti) if isString(ti) else ti
api = pta.api(schema)
is_hpp = isinstance(api, pta.HDBpp)
check = dict()
old_period = 24*3600*old_period if old_period < 1000 \
else (24*old_period if old_period<3600 else old_period)
allattrs = api.get_attributes() if hasattr(
api, 'get_attributes') else api.keys()
print('%s contains %d attributes' % (schema, len(allattrs)))
if attributes:
if fn.isString(attributes) and fn.isRegexp(attributes):
tattrs = [a for a in allattrs if clsearch(attributes, a)]
else:
attributes = map(fn.tango.get_normal_name, fn.toList(attributes))
tattrs = [
a for a in allattrs if fn.tango.get_normal_name(a) in allattrs
]
else:
tattrs = allattrs
excluded = [a for a in tattrs if any(fn.clmatch(e, a) for e in exclude)]
tattrs = [a for a in tattrs if a not in excluded]
print('%d attributes to check' % len(tattrs))
if not len(tattrs):
return
if excluded:
print('\t%d attributes excluded' % len(excluded))
archived = {}
for a in tattrs:
if hasattr(api, 'get_attribute_archiver'):
arch = api.get_attribute_archiver(a)
else:
arch = api[a].archiver
if arch:
archived[a] = arch
print('\t%d attributes are archived' % len(archived))
#Getting Tango devices currently not running
alldevs = set(t.rsplit('/', 1)[0] for t in tattrs)
#tdevs = filter(fn.check_device,alldevs)
#nodevs = [fn.tango.get_normal_name(d) for d in alldevs if d not in tdevs]
#if nodevs:
#print('\t%d devices are not running' % len(nodevs))
archs = sorted(set(archived.values()))
if loads:
astor = fn.Astor()
astor.load_from_devs_list(archs)
loads = fn.defaultdict(list)
for k, s in astor.items():
for d in s.get_device_list():
d = fn.tango.get_normal_name(d)
for a in archived:
if fn.tango.get_normal_name(archived[a]) == d:
loads[k].append(a)
for k, s in sorted(loads.items()):
print('\t%s archives %d attributes' % (k, len(s)))
noarchs = [
fn.tango.get_normal_name(d) for d in archs if not fn.check_device(d)
]
if noarchs:
print('\t%d archivers are not running: %s' % (len(noarchs), noarchs))
###########################################################################
if isString(values) and values.endswith('.pck'):
print('\nLoading last values from %s file\n' % values)
import pickle
values = pickle.load(open(values))
elif isString(values) and values.endswith('.json'):
print('\nLoading last values from %s file\n' % values)
values = fn.json2dict(values)
elif not use_index or is_hpp:
print('\nGetting last values ...\n')
for a in tattrs:
values[a] = api.load_last_values(a)
else:
print('\nGetting updated tables from database ...\n')
tups = pta.utils.get_table_updates(schema)
# Some tables do not update MySQL index tables
t0 = [a for a in tarch if a in tattrs and not tups[api[a].table]]
check.update((t, check_attribute(a, readable=True)) for t in t0
if not check.get(t))
t0 = [t for t in t0 if check[t]]
print('%d/%d archived attributes have indexes not updated ...' %
(len(t0), len(tarch)))
if t0 and len(t0) < 100:
vs = api.load_last_values(t0)
tups.update((api[t].table, api[t].last_date) for t in t0)
for a in tattrs:
if a in tups:
values[a] = [tups[api[a].table], 0]
for k, v in values.items():
if (len(v) if isSequence(v) else v):
if isinstance(v, dict):
v = v.values()[0]
if isSequence(v) and len(v) == 1:
v = v[0]
if v and not isNumber(v[0]):
v = [date2time(v[0]), v[1]]
values[k] = v
else:
values[k] = [] if isSequence(v) else None
print('%d values obtained' % len(values))
###########################################################################
now = fn.now()
result = fn.Struct()
times = [t[0] for t in values.values() if t]
futures = [t for t in times if t > now]
times = [t for t in times if t < now]
tmiss = []
tfutures = [k for k, v in values.items() if v and v[0] in futures]
tmin, tmax = min(times), max(times)
print('\toldest update was %s' % time2str(tmin))
print('\tnewest update was %s' % time2str(tmax))
if futures:
print('\t%d attributes have values in the future!' % len(futures))
tnovals = [a for a in archived if not values.get(a, None)]
if tnovals:
print('\t%d archived attributes have no values' % len(tnovals))
try:
tmiss = [
a for a, v in values.items()
if v and old_period < v[0] < ti - period and a not in archived
]
except:
print(values.items()[0])
if tmiss:
print('\t%d/%d attrs with values are not archived anymore' %
(len(tmiss), len(tattrs)))
result.Excluded = excluded
result.Schema = schema
result.All = tattrs
result.Archived = values
result.NoValues = tnovals
result.MissingOrRemoved = tmiss
result.TMin = tmin
result.TMax = tmax
result.Futures = tfutures
tup = sorted(a for a in values if values[a] and values[a][0] > ti - period)
tok = [a for a in tup if values[a][1] not in (None, [])]
print('\n%d/%d archived attributes are updated since %s - %s' %
(len(tup), len(archived), ti, period))
print('%d archived attributes are fully ok\n' % (len(tok)))
tnotup = sorted(a for a in values
if values[a] and values[a][0] < ti - period)
print('\t%d archived attrs are not updated' % len(tnotup))
tupnoread = [
a for a in tup if not values[a][1] and fn.read_attribute(a) is None
]
reads = dict((a, fn.read_attribute(a)) for a in tnotup)
tnotupread = [a for a in tnotup if reads[a] is not None]
print('\t%d not updated attrs are readable (Lost)' % len(tnotupread))
print('\t%d of them are not floats' %
len([t for t in tnotupread if not isinstance(reads[t], float)]))
print('\t%d of them are states' %
len([t for t in tnotupread if t.lower().endswith('/state')]))
print('\t%d of them seem motors' %
len([t for t in tnotupread if t.lower().endswith('/position')]))
tnotupevs = [a for a in tnotupread if fn.tango.check_attribute_events(a)]
print('\t%d not updated attrs are readable and have events (LostEvents)' %
len(tnotupevs))
tnotupnotread = [a for a in tnotup if a not in tnotupread]
print('\t%d not updated attrs are not readable' % len(tnotupnotread))
result.Lost = tnotupread
result.LostEvents = tnotupevs
losts = (tnotupevs if is_hpp else tnotupread)
diffs = dict()
for a in losts:
try:
v, vv = values.get(a, (None, ))[1], reads[a]
if fn.isSequence(v): v = fn.toList(v)
if fn.isSequence(vv): vv = fn.toList(vv)
diffs[a] = v != vv
if fn.isSequence(diffs[a]):
diffs[a] = any(diffs[a])
else:
diffs[a] = bool(diffs[a])
except:
diffs[a] = None
fams = fn.defaultdict(list)
for a in tnotupread:
fams['/'.join(a.split('/')[-4:-2])].append(a)
for f in sorted(fams):
print('\t%s: %d attrs not updated' % (f, len(fams[f])))
print()
differ = [a for a in losts if diffs[a]] #is True]
print('\t%d/%d not updated attrs have also wrong values!!!' %
(len(differ), len(losts)))
rd = pta.Reader()
only = [a for a in tnotupread if len(rd.is_attribute_archived(a)) == 1]
print('\t%d/%d not updated attrs are archived only in %s' %
(len(only), len(losts), schema))
result.LostDiff = differ
print()
archs = sorted(set(archived.values()))
astor = fn.Astor()
astor.load_from_devs_list(archs)
badloads = fn.defaultdict(list)
for k, s in astor.items():
for d in s.get_device_list():
d = fn.tango.get_normal_name(d)
for a in losts:
if fn.tango.get_normal_name(archived[a]) == d:
badloads[k].append(a)
for k, s in badloads.items():
if len(s):
print('\t%s archives %d lost attributes' % (k, len(s)))
print('\t%d updated attrs are not readable' % len(tupnoread))
result.ArchivedAndReadable = tok
result.Updated = tup
result.NotUpdated = tnotup
result.Unreadable = tnotupnotread
#result.DeviceNotRunning = nodevs
result.ArchiverNotRunning = noarchs
result.LostFamilies = fams
# Tnones is for readable attributes not being archived
tnones = [
a for a in archived
if (a not in values or values[a] and values[a][1] in (None, []))
and a not in tupnoread and a not in tnotupread
]
tupnones = [a for a in tnones if a in tup]
if tupnones:
print('\t%d archived readable attrs record empty values' %
len(tupnones))
result.Nones = tnones
if 0:
get_ratio = lambda a, b: float(len(a)) / float(len(b))
#result.ArchRatio = get_ratio([t for t in readarch if t not in tnotup],readarch)
#result.ReadRatio = get_ratio(result.Readable,tattrs)
#result.LostRatio = get_ratio([a for a in tread if a in tnotup],tread)
#result.MissRatio = get_ratio([a for a in tread if a not in tarch],tread)
#result.OkRatio = 1.0-result.LostRatio-result.MissRatio
#result.Summary = '\n'.join((
#('Checking archiving of %s attributes'%(len(attributes) if attributes else schema))
#,('%d attributes in %s, %d are currently active'%(len(api),schema,len(tarch)))
#,('%d devices with %d archived attributes are not running'%(len(nodevs),len([a for a in api if a.rsplit('/',1) in nodevs])))
#,('%d archived attributes (%2.1f %%) are unreadable! (check and remove)'%(len(tnoread),1e2*get_ratio(tnoread,tarch)))
#,('%d readable attributes are not archived'%(len(tmiss)))
#,('%d attributes (readable or not) are updated (%2.1f %% of all readables)'%(len(tok),1e2*result.OkRatio))
#,('-'*80)
#,('%d archived attributes (readable or not) are not updated!'%len(tnotup))
#,('%d archived and readable attributes are not updated! (check and restart?)'%len(treadnotup))
#,('-'*80)
#,('%d readable attributes have been removed in the last %d days!'%(len(removed),old_period/(24*3600)))
#,('%d readable scalar attributes are not being archived (not needed anymore?)'%len(tmscalar))
#,('%d readable array attributes are not being archived (Ok)'%len(tmarray))
#,('%d readable array attributes are archived (Expensive)'%len(tarray))
#,('')))
#if trace: print(result.Summary)
#print('%d readable lost,Ok = %2.1f%%, %2.1f %% over all Readables (%2.1f %% of total)'%\
#(len(treadnotup),1e2*result.ArchRatio,1e2*result.OkRatio,1e2*result.ReadRatio))
if action:
if action == 'start_devices':
print('Executing action %s' % action)
api.start_devices()
if action == 'restart_all':
print('Executing action %s' % action)
devs = api.get_archivers()
astor = fn.Astor()
print('Restarting %d devs:' % (len(devs), devs))
astor.load_from_devs_list(devs)
astor.stop_servers()
fn.wait(10.)
astor.start_servers()
#print('NO ACTIONS ARE GONNA BE EXECUTED, AS THESE ARE ONLY RECOMMENDATIONS')
#print("""
#api = PyTangoArchiving.HDBpp(schema)
#api.start_devices()
#or
#api = PyTangoArchiving.ArchivingAPI('%s')
#lostdevs = sorted(set(api[a].archiver for a in result.NotUpdated))
#print(lostdevs)
#if lostdevs < a_reasonable_number:
#astor = fn.Astor()
#astor.load_from_devs_list(lostdevs)
#astor.stop_servers()
#fn.time.sleep(10.)
#astor.start_servers()
#"""%schema)
print('\nfinished in %d seconds\n\n' % (fn.now() - ti))
if export is not None:
if export is True:
export = 'txt'
for x in (export.split(',') if isString(export) else export):
if x in ('json', 'pck', 'pickle', 'txt'):
x = '/tmp/%s.%s' % (schema, x)
print('Saving %s file with keys:\n%s' % (x, result.keys()))
if 'json' in x:
fn.dict2json(result.dict(), x)
else:
f = open(x, 'w')
if 'pck' in x or 'pickle' in x:
pickle.dump(result.dict(), f)
else:
f.write(fn.dict2str(result.dict()))
f.close()
return result
def get_table_description(api,table):
if fn.isString(api): api = pta.api(api)
return api.Query('show create table %s'%table)[-1][-1]
def mysqldump_by_date(schema, user, passwd, folder, start, stop, options = '',
tables = None, compress = True, delete = True):
"""
This method creates a backup between selected dates for each table
of the selected database.
All dump files are exported to the same folder, and a compressed file
is created at the end.
Deleting of temporary files created (folder/*dmp) must be done manually.
"""
print('mysqldump_by_date(%s,,,folder=%s,%s,%s,compress=%s,delete=%s)'
% (schema, folder, start, stop, compress, delete))
db = FriendlyDB(schema,user=user,passwd=passwd)
t,e = start,stop
print(t,e)
start = start if fn.isString(start) else fn.time2str(start)
stop = stop if fn.isString(stop) else fn.time2str(stop)
tables = tables or db.getTables()
print('mysqldump_by_date(%s): %d tables to backup between %s and %s'
% (schema,len(tables),start,stop))
if not os.path.isdir(folder):
print('mkdir %s' % folder)
os.mkdir(folder)
t0 = fn.now()
filenames = []
for t in sorted(tables):
currops = options
filename = ('%s/%s-%s-%s-%s.dmp'
% (folder,schema,t,start.split()[0],stop.split()[0]))
cols = db.getTableCols(t)
col = [c for c in ('int_time','time','data_time') if c in cols]
if col and col[0] == 'int_time':
where = " %s >= %s and %s < %s " % (
col[0],fn.str2time(start),col[0],fn.str2time(stop))
elif col:
where = " %s >= '%s' and %s < '%s' " % (col[0],start,col[0],stop)
else:
where = ""
if t in CONFIG_TABLES:
currops += " --add-drop-table "
else:
currops += ""
mysqldump(schema,user,passwd,filename,t,where,options=currops)
filenames.append(filename)
t1 = fn.now()
ext = ('part.' if fn.str2time(stop) > fn.now() else '') + 'tgz'
dext = '.dmp'
if compress:
# doing it on separate files ...
#for f in filenames:
#cmd = 'tar zcvf %s.tgz %s' % (f,f)
#print(cmd)
#fn.linos.shell_command(cmd)
#dext+='.tgz'
filename = ('%s/%s-%s-%s.%s'
% (folder,schema,start.split()[0],stop.split()[0],ext))
cmd = 'tar zcvf %s %s/*%s' % (filename,folder,dext)
print(cmd)
fn.linos.shell_command(cmd)
if compress and delete:
cmd = 'rm -rf %s/*.dmp*' % folder
print(cmd)
fn.linos.shell_command(cmd)
t2 = fn.now()
print('Backup took %d seconds' % int(t1-t0))
print('Compression took %d seconds' % int(t2-t1))
return filename
def getApi(k,schema):
schema = k.getSchema(schema)
if schema is not None:
api = schema.get('api','PyTangoArchiving.ArchivingAPI')
if fn.isString(api): api = k._load_object(api,schema)
return api(schema['schema']) if isinstance(api,type) else api
def get_assigned_attributes(api='hdb',dedicated=False): if fun.isString(api): api = pta.api(api) if dedicated: api.load_dedicated_archivers(check=False); return sorted(set(a for a in api if api[a].archiver or dedicated and api[a].dedicated))
def get_assigned_attributes(api='hdb', dedicated=False):
if fun.isString(api): api = pta.api(api)
api.load_dedicated_archivers(check=False)
return sorted(
set(a for a in api
if api[a].archiver or dedicated and api[a].dedicated))
def get_table_description(api, table):
if fn.isString(api): api = pta.api(api)
return api.Query('show create table %s' % table)[-1][-1]
def get_deactivated_attributes(api='hdb',updates=None,period=6*30*24*3600): # Returns the list of attributes that are not archived despite readable and having data from the last months if fun.isString(api): api = pta.api(api) if updates is None: updates = get_table_updates(api) now = fun.time.time() return sorted(a for a,t in updates.items() if (now-period)<t<(now-24*3600) and fun.check_attribute(a))
def checkSchema(k, schema, attribute='', start=None, stop=None):
if not isinstance(schema, SchemaDict):
schema = k.getSchema(schema)
if not schema:
return False
f = schema.get('check')
if not f:
print('%s has no check function' % str(schema))
return True
try:
now = time.time()
start = (str2time(start) if fn.isString(start) else fn.notNone(
start, now - 1))
stop = (str2time(stop) if fn.isString(stop) else fn.notNone(
stop, now))
xmatch = lambda e, a: clmatch(e, a, extend=True)
k.LOCALS.update({
'attr':
attribute.lower(),
'attribute':
attribute.lower(),
'device':
attribute.lower().rsplit('/', 1)[0],
'match':
lambda r: xmatch(r, attribute),
'clmatch':
xmatch,
'overlap':
overlap,
'time2str':
time2str,
'str2time':
str2time,
't2s':
time2str,
's2t':
str2time,
'start':
start,
'stop':
stop,
'now':
now,
'begin':
start,
'end':
stop,
'NOW':
now,
'reader':
schema.get('reader', schema.get('api')),
'schema':
schema.get('schema'),
'dbname':
schema.get('dbname',
schema.get('db_name', schema.get('schema', ''))),
})
if 'reader' in f:
k.getReader(schema.get('schema'))
if 'api' in f:
k.getApi(schema.get('schema'))
#print('In reader.Schemas.checkSchema(%s,%s,%s,%s): %s'
#% (schema,attribute,start,stop,f))
#print('(%s)%%(%s)'%(f,[t for t in k.LOCALS.items() if t[0] in f]))
v = fn.evalX(f, k.LOCALS, k.MODULES)
except:
print('checkSchema(%s,%s) failed!' % (schema, attribute))
traceback.print_exc()
v = False
#print('checkSchema(%s): %s'%(schema,v))
return v
