def addXYModels(self, attrs, t0=None, t1=None): """ Convert model, dates to 'tgarch://alba03.cells.es:10000/sr/id/scw01/pressure?db=*;t0=2019-11-11T11:41:59;t1=2020-01-03T09:03:03;ts', """ c = self.cursor() self.setCursor(Qt.Qt.WaitCursor) attrs = fn.toList(attrs) if not t0 and not t1 and not self.t0 and not self.t1: t0, t1 = self.tc.getTimes() if t0 and t1: t0 = t0 if fn.isNumber(t0) else fn.str2time(t0, relative=True) t1 = t1 if fn.isNumber(t1) else fn.str2time(t1, relative=True) self.t0, self.t1 = fn.time2str(t0, iso=1), fn.time2str(t1, iso=1) self.t0 = self.t0.replace(' ', 'T') self.t1 = self.t1.replace(' ', 'T') ms = [] for attr in attrs: attr = fn.tango.get_full_name(attr, fqdn=True) attr = attr.replace('tango://', '') q = 'tgarch://%s?db=*;t0=%s;t1=%s' % (attr, self.t0, self.t1) m = (q + ';ts', q) ms.append(m) self.plot.onAddXYModel(ms) self.setCursor(c)
def add_periodic_attributes(self, attributes, periods, wait=3.): """ attributes must be a list, periods a number, list or dict """ attributes = sorted( parse_tango_model(a, fqdn=True).fullname.lower() for a in attributes) if fn.isNumber(periods): periods = dict((a, periods) for a in attributes) elif fn.isSequence(periods): periods = dict(zip(attributes, periods)) devs = fn.defaultdict(list) [devs[fn.tango.get_dev_name(a)].append(a) for a in attributes] done = [] for dev, attrs in devs.items(): archiver = self.get_next_periodic_archiver(attrexp=dev + '/*') for attribute in attrs: try: period = periods[attribute] self.info('add_periodic_attribute(%s,%s,%s)' % (attribute, period, archiver)) self.add_periodic_attribute(attribute, period=period, archiver=archiver, wait=wait) done.append((attribute, period, archiver)) except: self.warning(fn.except2str()) return done
def __contains__(self, key): if key in self.mappings: return True elif fn.isNumber(key): #Return a mapped addresses return any(m.has_address(key) for m in self.mappings.values()) else: return False
def decimate_db_table_by_time(db,table,att_id,tstart,tend,period=1, id_column="att_conf_id",time_column='data_time',min_to_delete=3, optimize = False): """ This simplified method will remove all values in a table that are nearer than a given period It doesnt analyze values, it just gets the last value within the interval It is the most suitable for hdb++ and arrays Partition optimization and repair should be called afterwards https://dev.mysql.com/doc/refman/5.6/en/partitioning-maintenance.html ALTER TABLE t1 REBUILD PARTITION p0, p1; ALTER TABLE t1 OPTIMIZE PARTITION p0, p1; ALTER TABLE t1 REPAIR PARTITION p0,p1; """ t0 = fn.now() s0 = db.getTableSize(table) if fn.isNumber(tstart): tstart,tend = fn.time2str(tstart),fn.time2str(tend) q = "select distinct CAST(UNIX_TIMESTAMP(%s) AS DOUBLE) from %s where %s = %s and %s between '%s' and '%s'" % ( time_column, table, id_column, att_id, time_column, tstart, tend) partitions = get_partitions_from_query(db,q) print('Query: '+q) print('table size is %s, partitions affected: %s' % (s0, partitions)) vals = db.Query(q) t1 = fn.now() print('query took %d seconds, %d rows returned' % ((t1-t0), len(vals))) if not vals: return goods,p = [vals[0][0]],vals[0][0] for i,v in enumerate(vals): v = v[0] if v > period+goods[-1] and p!=goods[-1]: goods.append(p) p = v print(fn.now()-t1) print('%d rows to delete, %d to preserve' % (len(vals)-len(goods), len(goods))) for i in range(len(goods)-1): s,e = goods[i],goods[i+1] s,e = fn.time2str(s,us=True),fn.time2str(e,us=True) dq = "delete from %s where %s = %s and %s > '%s' and %s < '%s'" % ( table, id_column, att_id, time_column, s, time_column, e) if not i%1000: print(dq) db.Query(dq) t2 = fn.now() s1 = db.getTableSize(table) print('deleting %d rows took %d seconds' % (s0-s1, t2-t1)) if optimize:# or (goods[-1] - goods[0]) > 86400*5: rq = 'alter table %s optimize partition %s' % (table,partitions) print(rq) db.Query(rq) print('Optimizing took %d seconds' % (fn.now()-t2)) return s1-s0
def __getitem__(self, key): if fn.isNumber(key): key = int(key) for v in self.mappings.values(): #Return a mapped addresses if v.has_address(key): return v.get_address(key) if self.default is not Exception: return self.default raise Exception("AdressNotMapped: '%s'" % key) elif key in self.mappings: return self.mappings[key] #Return a Map else: raise Exception("KeyError: '%s'" % key)
def get_attr_id_type_table(self, attr): if fn.isNumber(attr): where = 'att_conf_id = %s' % attr else: where = "att_name like '%s'" % get_search_model(attr) q = "select att_conf_id,att_conf_data_type_id from att_conf where %s"\ %where ids = self.Query(q) self.debug(str((q, ids))) if not ids: return [] aid, tid = ids[0] table = self.Query("select data_type from att_conf_data_type "\ +"where att_conf_data_type_id = %s"%tid)[0][0] return aid, tid, 'att_' + table
def get_attr_id_type_table(self,attr): if fn.isNumber(attr): where = 'att_conf_id = %s'%attr else: where = "att_name like '%s'"%get_search_model(attr) q = "select att_name,att_conf_id,att_conf_data_type_id from att_conf"\ " where %s"%where ids = self.Query(q) self.debug(str((q,ids))) if not ids: return None,None,'' attr,aid,tid = ids[0] table = self.Query("select data_type from att_conf_data_type "\ +"where att_conf_data_type_id = %s"%tid)[0][0] self.attributes[attr].id = aid self.attributes[attr].type = table self.attributes[attr].table = 'att_'+table self.attributes[attr].modes = {'MODE_E':True} return aid,tid,'att_'+table
def get_attr_id_type_table(self, attr): if fn.isNumber(attr): where = 'att_conf_id = %s' % attr else: where = "att_name like '%s'" % get_search_model(attr) q = "select att_name,att_conf_id,att_conf_data_type_id from att_conf"\ " where %s"%where ids = self.Query(q) self.debug(str((q, ids))) if not ids: return None, None, '' attr, aid, tid = ids[0] table = self.Query("select data_type from att_conf_data_type "\ +"where att_conf_data_type_id = %s"%tid)[0][0] self.attributes[attr].id = aid self.attributes[attr].type = table self.attributes[attr].table = 'att_' + table self.attributes[attr].modes = {'MODE_E': True} return aid, tid, 'att_' + table
def __setitem__(self, key, value): if fn.isNumber(key): raise Exception('NotAllowed!') else: self.mappings[key].set(value)
def get_panic_report(api=None,timeout=3000.,tries=3, devfilter = '*', attr = 'ActiveAlarms', trace=False): """ The key of the results: key.count('/') == 0 : alarm == 1: server == 2: device == 4: summary """ if api is None: api = panic.api() elif isString(api): api = panic.api(api) if not len(api.servers): if devfilter == '*': api.servers.load_by_name('PyAlarm/*') elif isString(devfilter): api.servers.load_by_name(devfilter) else: [api.servers.load_by_name(d) for d in devfilter] alldevs = fd.tango.get_all_devices(exported=True) #print('%d devices in %s'%(len(alldevs),fd.tango.get_tango_host())) result = fd.dicts.defaultdict(dict) result['//alarms//'] = api.alarms.keys() result['//devices//'] = api.devices.keys() result['//servers//'] = api.servers.keys() result['//attrs//'] = [] off = [] hung = [] slow = [] aslow = [] errors = {} for s,ss in api.servers.items(): admin = ss.get_admin_name() try: admin = fd.parse_tango_model(admin)['device'] except: print('unable to parse %s'%admin) continue if admin.lower() not in alldevs: off.append(admin) result[s]['devices'] = ss.get_classes()['PyAlarm'] for d in result[s]['devices']: if isSequence(devfilter): if d not in devfilter: continue elif not clsearch(devfilter,d): continue if admin in off: t = END_OF_TIME off.append(d) else: proxy = fd.get_device(d) t = get_device_timeout(d,timeout,tries,alldevs, proxy,trace=False,attr=attr) if t == END_OF_TIME: hung.append(d) if t*1e3 > timeout: slow.append(d) result[d]['timeout'] = t result[d]['attrs'] = 0 result[d]['alarms'] = len(api.devices[d].alarms) polling = float(api.devices[d].config['PollingPeriod']) result[d]['polling'] = polling try: evals = get_device_eval_times(proxy,timeout) teval = sum(evals.values()) ratio = teval/float(polling) result[d]['eval'],result[d]['ratio'] = teval,ratio except: evals = {} result[d]['eval'],result[d]['ratio'] = -1,-1 for a,aa in api.devices[d].alarms.items(): attrs = api.parse_attributes(aa['formula']) result[d]['attrs'] += len(attrs) result[a]['device'] = d result[a]['attrs'] = attrs result['//attrs//'].extend(attrs) result[a]['timeout'] = evals.get(a,-1) if result[a]['timeout'] > polling/result[d]['alarms']: aslow.append(a) result['//attrs//'] = sorted(set(map(str.lower,result['//attrs//']))) result['//off//'] = off result['//bloat//'] = [k for k in result if k.count('/')==2 and result[k].get('ratio',-1)>=1.] result['//hung//'] = hung result['//slow_devices//'] = slow result['//slow_alarms//'] = aslow #print('off devices: %s\n'%(off)) #print('hung devices: %s\n'%(hung)) #print('slow devices: %s\n'%(slow)) #print('bloat devices: %s\n'%([k for k in result #if '/' in k and result[k].get('ratio',-1)>=1.])) #print('slow alarms: %s\n'%aslow) if '-v' in opts: print(fd.get_tango_host()) for k,v in sorted(result.items()): if fd.isSequence(v): print('%s: %d'%(k,len(v))) elif fd.isNumber(v): print('%s: %d'%(k,v))