def addXYModels(self, attrs, t0=None, t1=None):
"""
Convert model, dates to
'tgarch://alba03.cells.es:10000/sr/id/scw01/pressure?db=*;t0=2019-11-11T11:41:59;t1=2020-01-03T09:03:03;ts',
"""
c = self.cursor()
self.setCursor(Qt.Qt.WaitCursor)
attrs = fn.toList(attrs)
if not t0 and not t1 and not self.t0 and not self.t1:
t0, t1 = self.tc.getTimes()
if t0 and t1:
t0 = t0 if fn.isNumber(t0) else fn.str2time(t0, relative=True)
t1 = t1 if fn.isNumber(t1) else fn.str2time(t1, relative=True)
self.t0, self.t1 = fn.time2str(t0, iso=1), fn.time2str(t1, iso=1)
self.t0 = self.t0.replace(' ', 'T')
self.t1 = self.t1.replace(' ', 'T')
ms = []
for attr in attrs:
attr = fn.tango.get_full_name(attr, fqdn=True)
attr = attr.replace('tango://', '')
q = 'tgarch://%s?db=*;t0=%s;t1=%s' % (attr, self.t0, self.t1)
m = (q + ';ts', q)
ms.append(m)
self.plot.onAddXYModel(ms)
self.setCursor(c)
def add_periodic_attributes(self, attributes, periods, wait=3.):
"""
attributes must be a list, periods a number, list or dict
"""
attributes = sorted(
parse_tango_model(a, fqdn=True).fullname.lower()
for a in attributes)
if fn.isNumber(periods):
periods = dict((a, periods) for a in attributes)
elif fn.isSequence(periods):
periods = dict(zip(attributes, periods))
devs = fn.defaultdict(list)
[devs[fn.tango.get_dev_name(a)].append(a) for a in attributes]
done = []
for dev, attrs in devs.items():
archiver = self.get_next_periodic_archiver(attrexp=dev + '/*')
for attribute in attrs:
try:
period = periods[attribute]
self.info('add_periodic_attribute(%s,%s,%s)' %
(attribute, period, archiver))
self.add_periodic_attribute(attribute,
period=period,
archiver=archiver,
wait=wait)
done.append((attribute, period, archiver))
except:
self.warning(fn.except2str())
return done
def __contains__(self, key):
if key in self.mappings:
return True
elif fn.isNumber(key):
#Return a mapped addresses
return any(m.has_address(key) for m in self.mappings.values())
else:
return False
def decimate_db_table_by_time(db,table,att_id,tstart,tend,period=1,
id_column="att_conf_id",time_column='data_time',min_to_delete=3,
optimize = False):
"""
This simplified method will remove all values in a table that are nearer than a given period
It doesnt analyze values, it just gets the last value within the interval
It is the most suitable for hdb++ and arrays
Partition optimization and repair should be called afterwards
https://dev.mysql.com/doc/refman/5.6/en/partitioning-maintenance.html
ALTER TABLE t1 REBUILD PARTITION p0, p1;
ALTER TABLE t1 OPTIMIZE PARTITION p0, p1;
ALTER TABLE t1 REPAIR PARTITION p0,p1;
"""
t0 = fn.now()
s0 = db.getTableSize(table)
if fn.isNumber(tstart):
tstart,tend = fn.time2str(tstart),fn.time2str(tend)
q = "select distinct CAST(UNIX_TIMESTAMP(%s) AS DOUBLE) from %s where %s = %s and %s between '%s' and '%s'" % (
time_column, table, id_column, att_id, time_column, tstart, tend)
partitions = get_partitions_from_query(db,q)
print('Query: '+q)
print('table size is %s, partitions affected: %s' % (s0, partitions))
vals = db.Query(q)
t1 = fn.now()
print('query took %d seconds, %d rows returned' % ((t1-t0), len(vals)))
if not vals:
return
goods,p = [vals[0][0]],vals[0][0]
for i,v in enumerate(vals):
v = v[0]
if v > period+goods[-1] and p!=goods[-1]:
goods.append(p)
p = v
print(fn.now()-t1)
print('%d rows to delete, %d to preserve' % (len(vals)-len(goods), len(goods)))
for i in range(len(goods)-1):
s,e = goods[i],goods[i+1]
s,e = fn.time2str(s,us=True),fn.time2str(e,us=True)
dq = "delete from %s where %s = %s and %s > '%s' and %s < '%s'" % (
table, id_column, att_id, time_column, s, time_column, e)
if not i%1000: print(dq)
db.Query(dq)
t2 = fn.now()
s1 = db.getTableSize(table)
print('deleting %d rows took %d seconds' % (s0-s1, t2-t1))
if optimize:# or (goods[-1] - goods[0]) > 86400*5:
rq = 'alter table %s optimize partition %s' % (table,partitions)
print(rq)
db.Query(rq)
print('Optimizing took %d seconds' % (fn.now()-t2))
return s1-s0
def __getitem__(self, key):
if fn.isNumber(key):
key = int(key)
for v in self.mappings.values(): #Return a mapped addresses
if v.has_address(key):
return v.get_address(key)
if self.default is not Exception:
return self.default
raise Exception("AdressNotMapped: '%s'" % key)
elif key in self.mappings:
return self.mappings[key] #Return a Map
else:
raise Exception("KeyError: '%s'" % key)
def get_attr_id_type_table(self, attr):
if fn.isNumber(attr):
where = 'att_conf_id = %s' % attr
else:
where = "att_name like '%s'" % get_search_model(attr)
q = "select att_conf_id,att_conf_data_type_id from att_conf where %s"\
%where
ids = self.Query(q)
self.debug(str((q, ids)))
if not ids:
return []
aid, tid = ids[0]
table = self.Query("select data_type from att_conf_data_type "\
+"where att_conf_data_type_id = %s"%tid)[0][0]
return aid, tid, 'att_' + table
def get_attr_id_type_table(self,attr):
if fn.isNumber(attr):
where = 'att_conf_id = %s'%attr
else:
where = "att_name like '%s'"%get_search_model(attr)
q = "select att_name,att_conf_id,att_conf_data_type_id from att_conf"\
" where %s"%where
ids = self.Query(q)
self.debug(str((q,ids)))
if not ids:
return None,None,''
attr,aid,tid = ids[0]
table = self.Query("select data_type from att_conf_data_type "\
+"where att_conf_data_type_id = %s"%tid)[0][0]
self.attributes[attr].id = aid
self.attributes[attr].type = table
self.attributes[attr].table = 'att_'+table
self.attributes[attr].modes = {'MODE_E':True}
return aid,tid,'att_'+table
def get_attr_id_type_table(self, attr):
if fn.isNumber(attr):
where = 'att_conf_id = %s' % attr
else:
where = "att_name like '%s'" % get_search_model(attr)
q = "select att_name,att_conf_id,att_conf_data_type_id from att_conf"\
" where %s"%where
ids = self.Query(q)
self.debug(str((q, ids)))
if not ids:
return None, None, ''
attr, aid, tid = ids[0]
table = self.Query("select data_type from att_conf_data_type "\
+"where att_conf_data_type_id = %s"%tid)[0][0]
self.attributes[attr].id = aid
self.attributes[attr].type = table
self.attributes[attr].table = 'att_' + table
self.attributes[attr].modes = {'MODE_E': True}
return aid, tid, 'att_' + table
def __setitem__(self, key, value):
if fn.isNumber(key):
raise Exception('NotAllowed!')
else:
self.mappings[key].set(value)
def get_panic_report(api=None,timeout=3000.,tries=3,
devfilter = '*',
attr = 'ActiveAlarms',
trace=False):
"""
The key of the results:
key.count('/') == 0 : alarm
== 1: server
== 2: device
== 4: summary
"""
if api is None: api = panic.api()
elif isString(api): api = panic.api(api)
if not len(api.servers):
if devfilter == '*':
api.servers.load_by_name('PyAlarm/*')
elif isString(devfilter):
api.servers.load_by_name(devfilter)
else:
[api.servers.load_by_name(d) for d in devfilter]
alldevs = fd.tango.get_all_devices(exported=True)
#print('%d devices in %s'%(len(alldevs),fd.tango.get_tango_host()))
result = fd.dicts.defaultdict(dict)
result['//alarms//'] = api.alarms.keys()
result['//devices//'] = api.devices.keys()
result['//servers//'] = api.servers.keys()
result['//attrs//'] = []
off = []
hung = []
slow = []
aslow = []
errors = {}
for s,ss in api.servers.items():
admin = ss.get_admin_name()
try:
admin = fd.parse_tango_model(admin)['device']
except:
print('unable to parse %s'%admin)
continue
if admin.lower() not in alldevs:
off.append(admin)
result[s]['devices'] = ss.get_classes()['PyAlarm']
for d in result[s]['devices']:
if isSequence(devfilter):
if d not in devfilter:
continue
elif not clsearch(devfilter,d):
continue
if admin in off:
t = END_OF_TIME
off.append(d)
else:
proxy = fd.get_device(d)
t = get_device_timeout(d,timeout,tries,alldevs,
proxy,trace=False,attr=attr)
if t == END_OF_TIME: hung.append(d)
if t*1e3 > timeout: slow.append(d)
result[d]['timeout'] = t
result[d]['attrs'] = 0
result[d]['alarms'] = len(api.devices[d].alarms)
polling = float(api.devices[d].config['PollingPeriod'])
result[d]['polling'] = polling
try:
evals = get_device_eval_times(proxy,timeout)
teval = sum(evals.values())
ratio = teval/float(polling)
result[d]['eval'],result[d]['ratio'] = teval,ratio
except:
evals = {}
result[d]['eval'],result[d]['ratio'] = -1,-1
for a,aa in api.devices[d].alarms.items():
attrs = api.parse_attributes(aa['formula'])
result[d]['attrs'] += len(attrs)
result[a]['device'] = d
result[a]['attrs'] = attrs
result['//attrs//'].extend(attrs)
result[a]['timeout'] = evals.get(a,-1)
if result[a]['timeout'] > polling/result[d]['alarms']:
aslow.append(a)
result['//attrs//'] = sorted(set(map(str.lower,result['//attrs//'])))
result['//off//'] = off
result['//bloat//'] = [k for k in result
if k.count('/')==2 and result[k].get('ratio',-1)>=1.]
result['//hung//'] = hung
result['//slow_devices//'] = slow
result['//slow_alarms//'] = aslow
#print('off devices: %s\n'%(off))
#print('hung devices: %s\n'%(hung))
#print('slow devices: %s\n'%(slow))
#print('bloat devices: %s\n'%([k for k in result
#if '/' in k and result[k].get('ratio',-1)>=1.]))
#print('slow alarms: %s\n'%aslow)
if '-v' in opts:
print(fd.get_tango_host())
for k,v in sorted(result.items()):
if fd.isSequence(v):
print('%s: %d'%(k,len(v)))
elif fd.isNumber(v):
print('%s: %d'%(k,v))
