def get_last_value(self,attribute,value=-1):
if value == -1:
self.debug_stream('load_last_values(%s)' % attribute)
value = self.api.load_last_values(attribute)
if hasattr(value,'values'):
value = value.values()[0]
if (fn.isSequence(value) and len(value) and
fn.isSequence(value[0]) and len(value[0])>1):
value = value[0]
if value and isinstance(value[0],fn.datetime.datetime):
value = (fn.date2time(value[0]),value[1])
return value
def get_last_value(self, attribute, value=-1):
if value == -1:
self.debug_stream('load_last_values(%s)' % attribute)
value = self.api.load_last_values(attribute)
if hasattr(value, 'values'):
value = value.values()[0]
if (fn.isSequence(value) and len(value) and fn.isSequence(value[0])
and len(value[0]) > 1):
value = value[0]
if value and isinstance(value[0], fn.datetime.datetime):
value = (fn.date2time(value[0]), value[1])
return value
def updateAttributes(self, model=None):
print('AttributesPreview.updateAttributes(%s)' % model)
if not model and self.source:
try:
if hasattr(self.source, 'formula'): model = self.source.formula
elif hasattr(self.source, '__call__'): model = self.source()
else: model = str(self.source or '')
except:
print(traceback.format_exc())
if not fandango.isSequence(model):
ms, model = self.test.parse_variables(model or ''), set()
for var in ms:
dev, attr = var[0], var[1]
if ':' in dev and not dev.startswith('tango://'):
dev = 'tango://' + dev
model.add(dev + '/' + attr)
self.model = sorted(model)
print('In AttributesPreview.updateAttributes(%s)' % model)
self.taurusForm.setModel(model)
[
tvalue.setLabelConfig("<attr_fullname>")
for tvalue in self.taurusForm.getItems()
]
def export_attributes_to_pck(filein='ui_exported_devices.txt',
fileout='ui_attribute_values.pck'):
print('export_attributes:' + str((filein, fileout)))
if fandango.isSequence(filein):
devs = filein
else:
devs = map(str.strip, open(filein).readlines())
proxies = dict((d, PyTango.DeviceProxy(d)) for d in devs)
devs = defaultdict(Struct)
for d, dp in sorted(proxies.items()):
print('%s (%d/%d)' % (d, 1 + len(devs), len(proxies)))
obj = devs[d]
obj.dev_class, obj.attrs, obj.comms = '', defaultdict(Struct), {}
obj.props = dict(
(k, v if not 'vector' in str(type(v)).lower() else list(v))
for k, v in fandango.tango.get_matching_device_properties(
d, '*').items() if 'dynamicattributes' not in k.lower())
if fandango.check_device(d):
devs[d].name = d
devs[d].dev_class = dp.info().dev_class
for c in dp.command_list_query():
if c.cmd_name.lower() not in ('state', 'status', 'init'):
obj.comms[c.cmd_name] = (str(c.in_type), str(c.out_type))
for a in dp.get_attribute_list():
if a.lower() == 'status':
continue
obj.attrs[a] = fandango.tango.export_attribute_to_dict(
d, a, as_struct=True)
pickle.dump(devs, open(fileout, 'w'))
return (fileout)
def setModel(self, model):
print '*' * 80
self.info('VaccaProfilePlot.setModel(%s)' % model)
print '*' * 80
try:
#if self._profile_loaded: return
if fandango.isSequence(
model
) or 'attributename' in fandango.tango.parse_tango_model(model):
self.info('setting an attribute model')
TaurusPlot.setModel(self, model) # model = model[0]# str(
# model).rsplit('/',1)[0]
else:
self.info('setting a composer model')
assert fandango.check_device(model)
dev = taurus.Device(model)
if all(a in map(str.lower, dev.get_attribute_list())
for a in ('ccgaxxis', 'ccgpressures', 'ipaxxis',
'ippressures', 'thermoaxxis', 'thermocouples',
'axxispositions', 'axxislabels')):
TaurusPlot.setModel(self, [])
setup_profile_plot(self, model)
else:
self.warning('%s has not all required attributes' % model)
if len(self._positions) and len(self._labels):
self.info('Setting CustomXLabels ...')
self.setAxisCustomLabels(Qwt5.QwtPlot.xBottom,
zip(self._positions, self._labels),
60)
except Exception, e:
self.warning('VaccaProfilePlot.setModel(%s) failed!: %s' %
(model, e))
def add_periodic_attributes(self, attributes, periods, wait=3.):
"""
attributes must be a list, periods a number, list or dict
"""
attributes = sorted(
parse_tango_model(a, fqdn=True).fullname.lower()
for a in attributes)
if fn.isNumber(periods):
periods = dict((a, periods) for a in attributes)
elif fn.isSequence(periods):
periods = dict(zip(attributes, periods))
devs = fn.defaultdict(list)
[devs[fn.tango.get_dev_name(a)].append(a) for a in attributes]
done = []
for dev, attrs in devs.items():
archiver = self.get_next_periodic_archiver(attrexp=dev + '/*')
for attribute in attrs:
try:
period = periods[attribute]
self.info('add_periodic_attribute(%s,%s,%s)' %
(attribute, period, archiver))
self.add_periodic_attribute(attribute,
period=period,
archiver=archiver,
wait=wait)
done.append((attribute, period, archiver))
except:
self.warning(fn.except2str())
return done
def extract_array_index(values,array_index,decimate=False,asHistoryBuffer=False):
"""
Applying array_index to the obtained results, it has to be applied after
attribute loading to allow reusing cache in array-indexed attributes
decimate is just evaluated as True/False
"""
last,l0 = (0,None),len(values)
# Check if it has been already parsed
for r in values:
if r[1] is not None:
if not fandango.isSequence(r[1]):
return values
break
#print('extract_array_index(%s) to the obtained results'%array_index)
array_index = int(array_index)
new_values = [] # We create a new list on purpose to not modify the cached values
fin = ((lambda v: (v.time,v.value[array_index] if v.value is not None and len(v.value)>array_index else None))
if asHistoryBuffer else
(lambda v: (v[0],v[1][array_index] if v[1] is not None and len(v[1])>array_index else None)))
fcmp = (lambda l: (l[0].tv_sec,l[1])) if asHistoryBuffer else (lambda l: l)
fout = (lambda vv: FakeAttributeHistory(*(vv))) if asHistoryBuffer else (lambda vv: vv)
for i,v in enumerate(values):
try:
if v is None: continue
vv = fin(v)
nxt = ((values[i+1] and fin(values[i+1]) or None) if i+1<l0 else None)
if not decimate or nxt is None or not new_values or data_has_changed(fcmp(vv),fcmp(last),fcmp(nxt)):
new_values.append(fout(vv))
last = vv
except Exception,e:
print('extract_array_index(...,asHistoryBuffer=%s): Unable to parse %d[%s]:(%s); %s'%(asHistoryBuffer,i,array_index,v,traceback.format_exc()))
def do_repair(user,passwd,condition="engine is null",database="information_schema",force=False,days=0,db_host='localhost') :
sql = "select CONCAT(table_schema, '.', table_name) from tables where %s" % condition
db_con = MySQLdb.connect(db_host, port=3306, user=user,passwd=passwd,db=database)
cursor = db_con.cursor()
cursor.execute(sql)
rset = cursor.fetchall()
print '%d tables match condition'%len(rset)
now = time.time()
days = days or 60
tlimit = fandango.time2str(now-days*24*3600);
now = fandango.time2str(now);
for item in rset :
try:
if fandango.isSequence(item):
item = item[0]
if force:
raise Exception,'force=True, all tables to be checked'
elif 'att_' in item:
q = "select count(*) from %s where time between '%s' and '%s' order by time"%(item,tlimit,now)
cursor.execute(q)
count = cursor.fetchone()[0]
q = "select * from %s where time between '%s' and '%s' order by time"%(item,tlimit,now)
print q
cursor.execute(q) # desc limit 1'%item);
l = len(cursor.fetchall())
if abs(count-l)>5:
raise Exception('%d!=%d'%(count,l))
else:
raise Exception,'%s is a config table'%item
except Exception,e:
print e
print 'Repairing %s ...' % item
cursor.execute('repair table %s' % item)
print '[OK]\n'
time.sleep(.001)
def trigger_callbacks(self, regs=None):
"""
regs = list of addresses that changed
"""
if not self.callbacks:
return
for key, cb in self.callbacks.items():
try:
rs, push, org = regs, False, cb
if fn.isSequence(cb):
rs = regs and [r for r in regs if r in cb]
if not regs or rs or (len(cb) == 1
and fn.isCallable(cb[0])):
cb = cb[-1]
else:
continue
msg = ('%s: %s.trigger_callbacks(%s,%s): %s:%s' %
(fn.time2str(), self.name, fn.shortstr(
regs, 40), rs, key, org))
if self.plc_obj is not None:
self.plc_obj.debug(msg)
else:
print(msg)
if fn.isCallable(cb):
cb(key) #,push=push)
else:
cb = getattr(cb, 'push_event',
getattr(cb, 'event_received', None))
cb and cb(key)
fn.wait(1.e-4)
except Exception as e:
print(fn.except2str())
print('%s.callback(%s,%s): %s' % (self, key, cb, e))
def getSchema(k,schema,tango='',prop='',logger=None, write=False):
if schema.startswith('#') and EXPERT_MODE:
schema = schema.strip('#')
print('%s is enabled'%schema)
if schema in k.SCHEMAS:
# Failed schemas should be also returned (to avoid unneeded retries)
return k.SCHEMAS[schema]
dct = SchemaDict({'schema':schema,'dbname':schema,
'match':clmatch,'clmatch':clmatch})
try:
tango = fandango.tango.get_database(tango)
props = prop or tango.get_property('PyTangoArchiving',schema)[schema]
assert len(props)
if fandango.isSequence(props):
props = [map(str.strip,t.split('=',1)) for t in props]
dct.update(props)
dct['logger'] = logger
except Exception,e:
print('getSchema(%s): failed!'%schema)
if logger:
exc = traceback.format_exc()
try: logger.warning(exc)
except: print(exc)
dct = None
def test_AlarmDS(device=''):
if not device:
device = Fn.first((d for d,v in api.devices.items() if v.ping() is not None),None)
Fn.log.info('Testing AlarmDS(%s)'%device)
if device:
device = api.devices.get(device)
assert Fn.isSequence(device.get_active_alarms())
return True
def attrs2dict(self, attrs, keep=False, log=False):
vals = {}
failed = []
devs = fn.dicts.defaultdict(list)
[devs[a.rsplit('/', 1)[0]].append(a) for a in attrs]
for d, attrs in devs.items():
if not ft.check_device(d):
print('%s device is not running!' % d)
for t in attrs:
vals[t] = EMPTY_ATTR
vals[t]['model'] = t
vals[t]['label'] = vals[t]['name'] = t.rsplit('/')[-1]
vals[t]['device'] = d
else:
for t in attrs:
try:
v = ft.export_attribute_to_dict(t)
if log: print(v['model'], v['string'])
if v['value'] is None:
v['color'] = 'Grey'
v['string'] = '...'
elif fn.isSequence(v['string']):
sep = '\n' if v['data_type'] == 'DevString' else ','
v['string'] = sep.join(v['string'])
v['string'] = unicode(v['string'], 'latin-1')
v['tooltip'] = v['model'] + ':' + v['string']
try:
json.dumps(v)
except:
if log: print('json.dumps(%s) failed' % t)
if fn.isSequence(v['value']):
v['value'] = list(v['value'])
if fn.isBool(v['value']):
v['value'] = bool(v['value'])
vals[v['model']] = v
except:
if log:
print('export_attribute_to_dict(%s) failed' % t)
#
vals[t] = None
failed.append(t)
if failed:
print('%d failed attributes!: %s' %
(len(failed), ' '.join(failed)))
return vals
def setStateBackground(self, child, color):
if not isinstance(color, Qt.QColor):
if DEVICE_STATE_PALETTE.has(color):
qc = Qt.QColor(*DEVICE_STATE_PALETTE.rgb(color))
else:
qc = Qt.QColor(
color) if not fandango.isSequence(color) else Qt.QColor(
*color)
child.setBackground(0, Qt.QBrush(qc))
def test_AlarmDS(device=''):
if not device:
device = Fn.first(
(d for d, v in api.devices.items() if v.ping() is not None), None)
Fn.log.info('Testing AlarmDS(%s)' % device)
if device:
device = api.devices.get(device)
assert Fn.isSequence(device.get_active_alarms())
return True
def get_periodic_archivers(self):
#archs = fn.tango.get_class_devices('PyHdbppPeriodicArchiver')
archivers = fn.tango.get_device_property(self.manager,
'PeriodicArchivers')
if not fn.isSequence(archivers):
archivers = fn.toList(archivers)
try:
return sorted(archivers)
except:
return []
def setSeverity(self,tag,severity):
tags = tag if fandango.isSequence(tag) else [tag]
self.setAllowedUsers(self.api.get_admins_for_alarm(len(tags)==1 and tags[0]))
if not self.validate('setSeverity(%s,%s)'%(tags,severity)):
return
for tag in tags:
severity = str(severity).upper().strip()
if severity not in panic.ALARM_SEVERITIES: raise Exception(severity)
self.AlarmRows[tag].get_alarm_object().setup(severity=severity.upper(),write=True)
[f.setAlarmData() for f in WindowManager.WINDOWS if isinstance(f,AlarmForm)]
def get_attributes_values(self,tables='',start_date=None,stop_date=None,
desc=False,N=0,unixtime=True,extra_columns='quality',
decimate=0,human=False):
if not fn.isSequence(tables):
tables = self.get_archived_attributes(tables)
return dict((t,self.get_attribute_values(t,start_date,stop_date,desc,
N,unixtime,extra_columns,decimate,human))
for t in tables)
def get_attributes_values(self,tables='',start_date=None,stop_date=None,
desc=False,N=0,unixtime=True,extra_columns='quality',
decimate=0,human=False):
if not fn.isSequence(tables):
tables = self.get_archived_attributes(tables)
return dict((t,self.get_attribute_values(t,start_date,stop_date,desc,
N,unixtime,extra_columns,decimate,human))
for t in tables)
def getSchema(k, schema, tango='', prop='', logger=None):
if schema.startswith('#') and EXPERT_MODE:
schema = schema.strip('#')
print('%s available only in EXPERT_MODE' % schema)
if schema in k.SCHEMAS:
# Failed schemas should be also returned (to avoid unneeded retries)
return k.SCHEMAS[schema]
dct = {
'schema': schema,
'dbname': schema,
'match': clmatch,
'clmatch': clmatch
}
try:
tango = fandango.tango.get_database(tango)
props = prop or tango.get_property('PyTangoArchiving',
schema)[schema]
if fandango.isSequence(props):
props = [map(str.strip, t.split('=', 1)) for t in props]
dct.update(props)
rd = dct.get('reader')
if rd:
m = rd.split('(')[0].rsplit('.', 1)[0]
c = rd[len(m) + 1:]
if m not in k.MODULES:
fandango.evalX('import %s' % m, modules=k.MODULES)
#print('getSchema(%s): load %s reader'%(schema,dct.get('reader')))
dct['logger'] = logger
dct['reader'] = rd = fandango.evalX(dct.get('reader'),
modules=k.MODULES,
_locals=dct)
if not hasattr(rd, 'is_attribute_archived'):
rd.is_attribute_archived = lambda *a, **k: True
if not hasattr(rd, 'get_attributes'):
rd.get_attributes = lambda *a, **k: []
if not hasattr(rd, 'get_attribute_values'):
if dct['method']:
rd.get_attribute_values = getattr(rd, dct['method'])
if not hasattr(rd, 'schema'): rd.schema = dct['schema']
except Exception, e:
print('Reader.getSchema(%s): failed!' % schema)
if logger:
try:
logger.debug(traceback.format_exc())
except:
pass
dct = None
def initContexts(self, attrlist=[], sid=None):
"""
This method overrides SnapDialog.initContexts()
"""
#self.contextComboBox.blockSignals(True)
#self.contextComboBox.clear()
contexts = None
self._Form.setWindowTitle(
QtGui.QApplication.translate(
"Form",
fandango.get_tango_host().split(':', 1)[0] + ' -> Snapshoting',
None, QtGui.QApplication.UnicodeUTF8))
print('SnapForm.initContexts(%s(%s),%s(%s))' %
(type(attrlist), attrlist, type(sid), sid))
try:
if attrlist:
if all(map(fandango.isNumber, (attrlist, sid))):
print('SnapForm.initContexts(int(%s),%s)' %
(attrlist, sid))
contexts = {attrlist: self.snapapi.get_context(attrlist)}
self._Form.setWindowTitle(
QtGui.QApplication.translate(
"Form",
fandango.get_tango_host().split(':', 1)[0] +
' -> Snapshots for Context "' + str(
self.snapapi.db.get_id_contexts(attrlist)[0]
['name']) + '"', None,
QtGui.QApplication.UnicodeUTF8))
elif fandango.isSequence(attrlist):
contexts = dict(
(ID, self.snapapi.get_context(ID))
for ID in self.snapapi.db.find_context_for_attribute(
attrlist))
self._Form.setWindowTitle(
QtGui.QApplication.translate(
"Form",
fandango.get_tango_host().split(':', 1)[0] +
' -> Snapshots filtered by window content', None,
QtGui.QApplication.UnicodeUTF8))
else:
contexts = self.snapapi.get_contexts()
except:
Qt.QMessageBox.critical(self,"Tango Archiving Problem",
"Could not talk with SnapManager DS.<br>" + \
"Please check if DS is running.")
if contexts is not None:
ctxs = sorted(contexts.values(), key=lambda s: s.name.lower())
for context in ctxs:
self.contextComboBox.addItem(
"%s [%d]" % (context.name, context.ID),
Qt.QVariant(context.ID))
#self.contextComboBox.model().sort(0, Qt.Qt.AscendingOrder)
if sid >= 0: self.listWidget.setCurrentRow(sid)
def mysqldump(schema,user,password,filename,tables='',where=''):
cmd = "mysqldump --single-transaction --force --compact --no-create-db "\
"--skip-lock-tables --complete-insert --quick -u %s -p%s" % (user,password)
cmd += " " + schema
if where:
cmd += ' --where=" %s"' % where
if tables:
cmd += ' --tables '+(' '.join(tables) if fn.isSequence(tables)
else tables)
cmd += ' > '+filename
print(cmd)
fn.linos.shell_command(cmd)
def check_archived_attribute(attribute, value = False, state = CheckState.OK,
default = CheckState.LOST, cache = None, tref = None,
check_events = True):
"""
generic method to check the state of an attribute (readability/events)
value = AttrValue object returned by check_attribute
cache = result from check_db_schema containing archived values
this method will not query the database; database values should be
given using the chache dictionary argument
"""
# readable and/or no reason known for archiving failure
state = default # do not remove this line
# Get current value/timestamp
if cache:
stored = cache.values[attribute]
#evs = cache.evs[attribute]
if stored is None or (fn.isSequence(stored) and not len(stored)):
return CheckState.UNK
else:
t,v = stored[0],stored[1]
if t>=tref and not isinstance(v,(type(None),Exception)):
print('%s should not be in check list! (%s,%s)' % (attribute,t,v))
return CheckState.OK
if value is False:
value = fn.check_attribute(attribute, brief=False)
vv,t = getattr(value,'value',value),getattr(value,'time',0)
t = t and fn.ctime2time(t)
if isinstance(vv,(type(None),Exception)):
# attribute is not readable
state = CheckState.NO_READ
elif cache and stored and 0 < t <= stored[0]:
# attribute timestamp doesnt change
state = CheckState.STALL
elif cache and stored and fbool(vv == stored[1]):
# attribute value doesnt change
state = CheckState.STALL
elif check_events:
# READABLE NOT STORED WILL ARRIVE HERE
evs = fn.tango.check_attribute_events(attribute)
if cache:
cache.evs[attribute] = evs
if not evs:
# attribute doesnt send events
state = CheckState.NO_EVENTS
return state
def get_attribute_indexes(self, table):
index = None
if fn.isSequence(table):
aid, tid, table = table
else:
if '[' in table:
try:
index = int(fn.clsearch('\[([0-9]+)\]', table).groups()[0])
table = table.split('[')[0]
except:
pass
aid, tid, table = self.get_attr_id_type_table(table)
return aid, tid, table, index
def get_panel(i):
pargs = EXTRA_PANELS[i]
if not fn.isSequence(pargs): return pargs
elif len(pargs) == 3:
return PanelDescription(pargs[0],
classname=pargs[1],
model=pargs[2])
else:
return PanelDescription(pargs[0],
classname=pargs[1],
model=pargs[2],
sharedDataRead=pargs[3],
sharedDataWrite=pargs[4])
def get_archivers_filters(archivers=''):
"""
Returns the value of AttributeFilters property for each archiver in the list
:param archivers: sequence or regular expression for matching archivers
:return: dictionary {archiver:regexp}
"""
archs = archs if fn.isSequence(archivers) else fn.find_devices(archivers)
filters = fn.SortedDict(
sorted((k, v['AttributeFilters'])
for k, v in fn.tango.get_matching_device_properties(
archiver, 'AttributeFilters').items()))
return filters
def updateAttributes(self,model=None):
print('AttributesPreview.updateAttributes(%s)'%model)
if not model and self.source:
try:
if hasattr(self.source,'formula'): model = self.source.formula
elif hasattr(self.source,'__call__'): model = self.source()
else: model = str(self.source or '')
except: print(traceback.format_exc())
print 'In AttributesPreview.updateAttributes(%s)'%model
if fandango.isSequence(model): model = sorted(model)
else: model = sorted(set('%s/%s'%(var[0],var[1]) for var in self.test.parse_variables(model or '')))
self.model = model
self.taurusForm.setModel(model)
[tvalue.setLabelConfig("attr_fullname") for tvalue in self.taurusForm.getItems()]
def updateAttributes(self,model=None):
print('AttributesPreview.updateAttributes(%s)'%model)
if not model and self.source:
try:
if hasattr(self.source,'formula'): model = self.source.formula
elif hasattr(self.source,'__call__'): model = self.source()
else: model = str(self.source or '')
except: print(traceback.format_exc())
print 'In AttributesPreview.updateAttributes(%s)'%model
if fandango.isSequence(model): model = sorted(model)
else: model = sorted(set('%s/%s'%(var[0],var[1]) for var in self.test.parse_variables(model or '')))
self.model = model
self.taurusForm.setModel(model)
[tvalue.setLabelConfig("<attr_fullname>") for tvalue in self.taurusForm.getItems()]
def getSchema(k, schema, tango='', prop='', logger=None, write=False):
if schema.startswith('#') and EXPERT_MODE:
schema = schema.strip('#')
print('%s is enabled' % schema)
if schema in k.SCHEMAS:
# Failed schemas should be also returned (to avoid unneeded retries)
return k.SCHEMAS[schema]
dct = {
'schema': schema,
'dbname': schema,
'match': clmatch,
'clmatch': clmatch
}
try:
tango = fandango.tango.get_database(tango)
props = prop or tango.get_property('PyTangoArchiving',
schema)[schema]
assert len(props)
if fandango.isSequence(props):
props = [map(str.strip, t.split('=', 1)) for t in props]
dct.update(props)
rd = dct.get('reader', dct.get('api'))
if rd:
dct['logger'] = logger
dct['reader'] = rd = k._load_object(rd, dct)
if not hasattr(rd, 'is_attribute_archived'):
rd.is_attribute_archived = lambda *a, **k: True
if not hasattr(rd, 'get_attributes'):
rd.get_attributes = lambda *a, **k: []
if not hasattr(rd, 'get_attribute_values'):
if dct['method']:
rd.get_attribute_values = getattr(rd, dct['method'])
if not hasattr(rd, 'schema'): rd.schema = dct['schema']
except Exception, e:
print('getSchema(%s): failed!' % schema)
if logger:
exc = traceback.format_exc()
try:
logger.warning(exc)
except:
print(exc)
dct = None
def check_attribute_ok(self, a, v, t=0):
"""
arguments are attribute name and last value from db, plus ref. time
"""
r = check_attribute_value(a)
rv = getattr(r, 'value', None)
if isinstance(rv, (type(None), Exception)):
# Attribute not readable
self.attr_nok.append(a)
elif self.is_hpp and not check_attribute_events(a):
self.attr_nevs.append(a)
else:
if v is None or fn.isSequence(v) and not len(v):
# Attribute has no values in DB
self.attr_lost.append(a)
else:
# Time is compared against last update, current or read time
t = min((t or fn.now(), fn.ctime2time(r.time)))
v = self.get_last_value(a, v)
try:
diff = v[1] != rv
except:
diff = 1
if v[0] < t - 3600:
if any(diff) if fn.isSequence(diff) else bool(diff):
# Last value much older than current data
self.attr_lost.append(a)
else:
self.attr_stall.append(a)
self.attr_ok.append(a)
elif v[1] is None:
# Value is readable but not from DB
self.attr_err.append(a)
else:
self.attr_ok.append(a)
return
def check_attribute_ok(self,a,v,t=0):
"""
arguments are attribute name and last value from db, plus ref. time
"""
r = check_attribute_value(a)
rv = getattr(r,'value',None)
if isinstance(rv,(type(None),Exception)):
# Attribute not readable
self.attr_nok.append(a)
elif self.is_hpp and not check_attribute_events(a):
self.attr_nevs.append(a)
else:
if v is None or fn.isSequence(v) and not len(v):
# Attribute has no values in DB
self.attr_lost.append(a)
else:
# Time is compared against last update, current or read time
t = min((t or fn.now(),fn.ctime2time(r.time)))
v = self.get_last_value(a,v)
try:
diff = v[1]!=rv
except:
diff = 1
if v[0] < t-3600:
if any(diff) if fn.isSequence(diff) else bool(diff):
# Last value much older than current data
self.attr_lost.append(a)
else:
self.attr_stall.append(a)
self.attr_ok.append(a)
elif v[1] is None:
# Value is readable but not from DB
self.attr_err.append(a)
else:
self.attr_ok.append(a)
return
def do_repair(user,
passwd,
condition="engine is null",
database="information_schema",
force=False,
days=0,
db_host='localhost'):
sql = "select CONCAT(table_schema, '.', table_name) from tables where %s" % condition
db_con = MySQLdb.connect(db_host,
port=3306,
user=user,
passwd=passwd,
db=database)
cursor = db_con.cursor()
cursor.execute(sql)
rset = cursor.fetchall()
print '%d tables match condition' % len(rset)
now = time.time()
days = days or 60
tlimit = fandango.time2str(now - days * 24 * 3600)
now = fandango.time2str(now)
for item in rset:
try:
if fandango.isSequence(item):
item = item[0]
if force:
raise Exception, 'force=True, all tables to be checked'
elif 'att_' in item:
q = "select count(*) from %s where time between '%s' and '%s' order by time" % (
item, tlimit, now)
cursor.execute(q)
count = cursor.fetchone()[0]
q = "select * from %s where time between '%s' and '%s' order by time" % (
item, tlimit, now)
print q
cursor.execute(q) # desc limit 1'%item);
l = len(cursor.fetchall())
if abs(count - l) > 5:
raise Exception('%d!=%d' % (count, l))
else:
raise Exception, '%s is a config table' % item
except Exception, e:
print e
print 'Repairing %s ...' % item
cursor.execute('repair table %s' % item)
print '[OK]\n'
time.sleep(.001)
def getSchema(k, schema, tango='', prop='', logger=None, write=False):
"""
Get schema specs as defined in Tango.FreePropertis.DbSchemas
"""
if schema.startswith('#') and EXPERT_MODE:
schema = schema.strip('#')
#print('%s is enabled'%schema)
if schema in k.SCHEMAS:
# Failed schemas should be also returned (to avoid unneeded retries)
return k.SCHEMAS[schema]
dct = {'match': clmatch, 'clmatch': clmatch}
if ';' in schema:
schema, dct = schema.split(';', 1)
dct = dict(d.split('=', 1) for d in dct.split(';'))
dct['schema'] = schema
dct = SchemaDict(dct)
props = []
try:
tango = fn.tango.get_database(tango)
props = prop or tango.get_property('PyTangoArchiving',
schema)[schema]
assert len(props)
if fn.isSequence(props):
props = dict(map(str.strip, t.split('=', 1)) for t in props)
if 'check' in dct:
props.pop('check')
dct.update(props)
dct['logger'] = logger
except Exception as e:
print('getSchema(%s): failed!' % schema)
print(dct, props)
exc = traceback.format_exc()
try:
logger.warning(exc)
except:
print(exc)
dct = None
if write:
k.SCHEMAS[schema] = dct
return dct
def setHighlightedItems(self, models=[], color=Qt.Qt.red):
"""
:param models: List of model to be Highlighted
:param color: Color, by default Qt.Qt.red
"""
if fandango.isSequence(models):
models = '(%s)' % ')|('.join(models)
self.info('setHighLightedItems(%s)' % models)
try:
default = self.scene().selectionColor()
self.scene().setSelectionColor(color)
self.scene().selectGraphicItem(models)
self.scene().setSelectionColor(default)
except:
err = traceback.format_exc()
self.error(err)
finally:
self.scene().setSelectionColor(default)
def test(self, tests=[]):
"""
Tests would be a list of (name,result,args,kwargs) values
"""
print('test(', tests, ')')
try:
tests = tests or self.tests
if not fn.isSequence(tests): tests = [tests]
passed = 0
if fn.isMapping(tests):
tests = [
[k] +
list(t if not isMapping(t) else (t.get('result', None),
t.get('args', []),
t.get('kwargs', [])))
for k, t in tests.items()
]
for t in tests:
t = fn.toList(t)
print t
t[0] = t[0]
t[1] = (t[1:] or [None])[0]
t[2] = (t[2:] or [[]])[0]
t[3] = (t[3:] or [{}])[0]
v = self.test_object(t[0], t[1], *t[2], **t[3])
if v: passed += 1
self.results[t[0]] = v
print('-' * 80)
for t in tests:
v = self.results[fn.toList(t)[0]]
print('%s testing: %s : %s' %
(self.module, t, ['Failed', 'Ok'][bool(v)]))
print('%s : %d / %d tests passed' %
(self.module, passed, len(tests)))
except:
traceback.print_exc()
print(tests)
return passed
def save(self, device, filename, data, add_timestamp=False, asynch=True):
"""
FolderDS device, filename WITHOUT PATH!, data to be saved
add_timestamp: whether to add or not timestamp at the end of filename
asynch: if True, the save() call will not wait for the command to complete
"""
# Remove device path from filename
filename = (filename or device).split('/')[-1]
if add_timestamp:
t = fn.time2str().replace(' ', '_').replace(':',
'').replace('-', '')
p, s = (filename.rsplit('.', 1)) if '.' in filename else (filename,
'')
filename = '.'.join(filter(bool, (p, t, s)))
d = self.get_device(device) #
assert tango.check_device(d)
if fn.isSequence(data): data = '\n'.join(data)
if self.char_mode:
cmd = 'SaveCharBuffer'
argin = filename + '\n' + data
#print(argin)
argin = map(ord, filename + '\n' + data)
#print(argin)
else:
cmd = 'SaveFile'
argin = [filename, data]
if asynch:
#d.command_inout_asynch('SaveFile',argin,True)
d.command_inout_asynch(cmd, argin, True)
r = len(data)
elif self.char_moode:
r = d.SaveCharBuffer(argin).split()[-1]
else:
r = d.SaveFile(argin)
print('%s.%s() : %s)' % (device, cmd, r))
print('mem: %s' % get_memory())
return r
def getSchema(k,schema,tango='',prop='',logger=None, write=False):
if schema.startswith('#') and EXPERT_MODE:
schema = schema.strip('#')
print('%s is enabled'%schema)
if schema in k.SCHEMAS:
# Failed schemas should be also returned (to avoid unneeded retries)
return k.SCHEMAS[schema]
dct = {'match':clmatch,'clmatch':clmatch}
if ';' in schema:
schema,dct = schema.split(';',1)
dct = dict(d.split('=',1) for d in dct.split(';'))
dct.update({'schema':schema,'dbname':schema})
dct = SchemaDict(dct)
props = []
try:
tango = fn.tango.get_database(tango)
props = prop or tango.get_property('PyTangoArchiving',
schema)[schema]
assert len(props)
if fn.isSequence(props):
props = dict(map(str.strip,t.split('=',1)) for t in props)
if 'check' in dct:
props.pop('check')
dct.update(props)
dct['logger'] = logger
except Exception,e:
print('getSchema(%s): failed!'%schema)
print(dct,props)
exc = traceback.format_exc()
try:
logger.warning(exc)
except:
print(exc)
dct = None
def setModel(self, model):
try:
self.model = str(model)
print('TaurusSingleValue.setModel(%s)' % self.model)
self.setWindowTitle(self.model)
self.rvalue = fn.tango.check_attribute(model)
if isinstance(self.value, Exception):
raise self.rvalue
mp = fn.tango.parse_tango_model(model)
self.device = mp.devicename
self.config = fn.tango.get_attribute_config(model)
self.label.setText(self.device + '/' +
(self.config.label or mp.attrname))
self.label.setToolTip(self.model)
self.units.setText(self.config.unit)
svalue = self.config.format or '%s'
if fn.isSequence(self.rvalue.value) or '%' not in svalue:
svalue = str(self.rvalue.value)[:25]
else:
svalue = (svalue % self.rvalue.value)[:25]
self.value.setText(svalue)
self.value.setToolTip(str(self.rvalue.value))
self.setColor({
AttrQuality.ATTR_VALID: 'lightgreen',
AttrQuality.ATTR_INVALID: 'lightgrey',
AttrQuality.ATTR_ALARM: 'lightred',
AttrQuality.ATTR_WARNING: 'lightorange',
AttrQuality.ATTR_CHANGING: 'lightblue',
}[self.rvalue.quality])
except Exception as e:
print(self.model, self.config.format, self.rvalue)
traceback.print_exc()
self.setColor('grey')
self.label.setText(str(model))
self.value.setText(str(e)[:25])
self.value.setToolTip(str(e))
self.units.setText('')
def test(self,tests=[]):
"""
Tests would be a list of (name,result,args,kwargs) values
"""
print('test(',tests,')')
try:
tests = tests or self.tests
if not fn.isSequence(tests): tests = [tests]
passed = 0
if fn.isMapping(tests):
tests = [
[k]+list(t if not isMapping(t) else
(t.get('result',None),t.get('args',[]),t.get('kwargs',[]))
)
for k,t in tests.items()]
for t in tests:
t = fn.toList(t)
print t
t[0] = t[0]
t[1] = (t[1:] or [None])[0]
t[2] = (t[2:] or [[]])[0]
t[3] = (t[3:] or [{}])[0]
v = self.test_object(t[0],t[1],*t[2],**t[3])
if v: passed += 1
self.results[t[0]] = v
print('-'*80)
for t in tests:
v = self.results[fn.toList(t)[0]]
print('%s testing: %s : %s' % (self.module,t,['Failed','Ok'][bool(v)]))
print('%s : %d / %d tests passed'%(self.module,passed,len(tests)))
except:
traceback.print_exc()
print(tests)
return passed
def get_attribute_values(self,
table,
start_date=None,
stop_date=None,
desc=False,
N=0,
unixtime=True,
extra_columns='quality',
decimate=0,
human=False,
as_double=True,
**kwargs):
"""
This method returns values between dates from a given table.
If stop_date is not given, then anything above start_date is returned.
desc controls the sorting of values
unixtime = True enhances the speed of querying by a 60%!!!!
#(due to MySQLdb implementation of datetime)
If N is specified:
* Query will return last N values if there's no stop_date
* If there is, then it will return the first N values (windowing?)
* IF N is negative, it will return the last N values instead
start_date and stop_date must be in a format valid for SQL
"""
t0 = time.time()
self.warning('HDBpp.get_attribute_values(%s,%s,%s,%s,decimate=%s,%s)' %
(table, start_date, stop_date, N, decimate, kwargs))
if fn.isSequence(table):
aid, tid, table = table
else:
aid, tid, table = self.get_attr_id_type_table(table)
human = kwargs.get('asHistoryBuffer', human)
what = 'UNIX_TIMESTAMP(data_time)' if unixtime else 'data_time'
if as_double:
what = 'CAST(%s as DOUBLE)' % what
if 'array' in table: what += ",idx"
what += ',value_r' if 'value_r' in self.getTableCols(table) \
else ',value'
if extra_columns: what += ',' + extra_columns
interval = 'where att_conf_id = %s'%aid if aid is not None \
else 'where att_conf_id >= 0 '
if start_date or stop_date:
start_date,start_time,stop_date,stop_time = \
Reader.get_time_interval(start_date,stop_date)
if start_date and stop_date:
interval += (" and data_time between '%s' and '%s'" %
(start_date, stop_date))
elif start_date and fandango.str2epoch(start_date):
interval += " and data_time > '%s'" % start_date
query = 'select %s from %s %s order by data_time' \
% (what,table,interval)
if N == 1:
human = 1
if N < 0 or desc:
query += " desc" # or (not stop_date and N>0):
if N:
query += ' limit %s' % abs(N) # if 'array' not in table else 1024)
######################################################################
# QUERY
self.debug(query)
try:
result = self.Query(query)
except MySQLdb.ProgrammingError as e:
if 'DOUBLE' in str(e) and "as DOUBLE" in query:
return self.get_attribute_values((aid, tid, table),
start_date,
stop_date,
desc,
N,
unixtime,
extra_columns,
decimate,
human,
as_double=False,
**kwargs)
self.debug('read [%d] in %f s' % (len(result), time.time() - t0))
t0 = time.time()
if not result or not result[0]: return []
######################################################################
if 'array' in table:
data = fandango.dicts.defaultdict(list)
for t in result:
data[float(t[0])].append(t[1:])
result = []
for k, v in sorted(data.items()):
l = [0] * (1 + max(t[0] for t in v))
for i, t in enumerate(v):
if None in t:
l = None
break
l[t[0]] = t[1] #Ignoring extra columns (e.g. quality)
result.append((k, l))
if N > 0:
#for k,l in result:
#print((k,l and len(l)))
result = result[-N:]
self.warning('array arranged [%d] in %f s' %
(len(result), time.time() - t0))
t0 = time.time()
# Converting the timestamp from Decimal to float
# Weird results may appear in filter_array comparison if not done
# Although it is INCREDIBLY SLOW!!!
#result = []
#nr = []
#if len(result[0]) == 2:
#for i,t in enumerate(result):
#result[i] = (float(t[0]),t[1])
#elif len(result[0]) == 3:
#for i,t in enumerate(result):
#result[i] = (float(t[0]),t[1],t[2])
#elif len(result[0]) == 4:
#for i,t in enumerate(result):
#result[i] = ((float(t[0]),t[1],t[2],t[3]))
#else:
#for i,t in enumerate(result):
#result[i] = ([float(t[0])]+t[1:])
self.warning('timestamp arranged [%d] in %f s' %
(len(result), time.time() - t0))
t0 = time.time()
# Decimation to be done in Reader object
#if decimate:
## When called from trends, decimate may be the decimation method
## or the maximum sample number
#try:
#N = int(decimate)
##decimate = data_has_changed
#decimate =
#result = PyTangoArchiving.reader.decimation(
#result,decimate,window=0,N=N)
#except:
##N = 1080
#result = PyTangoArchiving.reader.decimation(result,decimate)
if human:
result = [list(t) + [fn.time2str(t[0])] for t in result]
if not desc and ((not stop_date and N > 0) or (N < 0)):
#THIS WILL BE APPLIED ONLY WHEN LAST N VALUES ARE ASKED
#self.warning('reversing ...' )
result = list(reversed(result))
else:
# why?
self.getCursor(klass=MySQLdb.cursors.SSCursor)
self.warning('result arranged [%d]' % len(result))
return result
def setLastArgs(self,model,start=0,stop=None,history=-1,date=None):
#self.info('ArchivedTrendLogger.setLastArgs(%s)'%str((model,start,stop,history,date)))
date = date or time.time()
if fn.isSequence(history): history = len(history)
self.last_args[model] = [start,stop,history,date] #It must be a list, not tuple
def start_attributes_for_archivers(target,attr_regexp='',event_conf={},
load=False, by_class=False, min_polling = 100, overwrite = False,
check = True):
"""
Target may be an attribute list or a device regular expression
if by_class = True, config will be loaded from Tango class properties
"""
import PyTangoArchiving.hdbpp as ptah
if fn.isSequence(target):
if attr_regexp:
attrs = [a for a in target if fn.clmatch(attr_regexp,a.rsplit('/')[-1])]
else:
attrs = target
else:
dev_regexp = target
attrs = fn.find_attributes(dev_regexp+'/'+(attr_regexp or '*'))
if by_class:
classes = fn.defaultdict(dict)
devs = fn.defaultdict(list)
[devs[a.rsplit('/',1)[0]].append(a) for a in attrs]
for d,v in devs.items():
classes[fn.tango.get_device_class(d)][d] = v
attrs = {}
for c,devs in classes.items():
cfg = get_class_archiving(devs.keys()[0])
for d in devs:
raw = devs[d]
for a,v in cfg.items():
for aa in raw:
if fn.clmatch(a,aa.split('/')[-1],terminate=True):
if not attr_regexp or fn.clmatch(attr_regexp,aa):
attrs[aa] = v
elif event_conf:
attrs = dict((a,event_conf) for a in attrs)
else:
attrs = dict((a,get_current_conf(a)) for a in attrs)
print('Starting %d attributes' % (len(attrs)))
archs = ptah.multi.match_attributes_and_archivers(attrs.keys())
rd = PyTangoArchiving.Reader()
#print(archs)
alldbs = ptah.multi.get_hdbpp_databases()
dbs = ptah.multi.get_hdbpp_databases(archs,alldbs)
#return dbs,archs,attrs
for db,rcs in dbs.items():
api = PyTangoArchiving.Schemas.getApi(db)
dbs[db] = dict.fromkeys(rcs)
for d in rcs:
dbs[db][d] = ts = dict.fromkeys(archs[d])
#return ts
for a in ts:
try:
m = fn.parse_tango_model(a,fqdn=True)
dbs[db][d][a] = mode = attrs[a]
if not overwrite and db in rd.is_attribute_archived(a):
print('%s already archived in %s' % (a,db))
continue
events = ft.check_attribute_events(a,ft.EventType.ARCHIVE_EVENT)
ep = events.get(ft.EventType.ARCHIVE_EVENT,False)
if ep is True:
if 'polling' in mode:
mode.pop('polling')
elif isinstance(events.get(ep,(int,float))):
mode['polling'] = min((ep,mode.get('polling',10000)))
mode['polling'] = max((mode['polling'],min_polling))
if not events.get(ft.EventType.CHANGE_EVENT,False):
if mode.get('archive_abs_change',0):
mode['abs_event'] = mode['archive_abs_change']
if mode.get('archive_rel_change',0):
mode['rel_event'] = mode['archive_rel_change']
if mode.get('arch_per_event',0):
mode['per_event'] = mode['archive_per_event']
print('%s.start_archiving(%s,%s,%s): %s' % (db,d,m.fullname,mode,load))
if load:
fn.tango.set_attribute_events(a,**mode)
r = api.start_archiving(m.fullname,d,code_event=True)
assert not check or r
except:
print('%s failed!'%a)
traceback.print_exc()
return dbs
def get_attribute_values(self,table,start_date=None,stop_date=None,
desc=False,N=0,unixtime=True,
extra_columns='quality',decimate=0,human=False,
as_double=True,aggregate='MAX',int_time=True,
**kwargs):
"""
This method returns values between dates from a given table.
If stop_date is not given, then anything above start_date is returned.
desc controls the sorting of values
unixtime = True enhances the speed of querying by a 60%!!!!
#(due to MySQLdb implementation of datetime)
If N is specified:
* Query will return last N values if there's no stop_date
* If there is, then it will return the first N values (windowing?)
* IF N is negative, it will return the last N values instead
start_date and stop_date must be in a format valid for SQL
"""
t0 = time.time()
self.debug('HDBpp.get_attribute_values(%s,%s,%s,%s,decimate=%s,%s)'
%(table,start_date,stop_date,N,decimate,kwargs))
if fn.isSequence(table):
aid,tid,table = table
else:
aid,tid,table = self.get_attr_id_type_table(table)
if not all((aid,tid,table)):
self.warning('%s is not archived' % table)
return []
human = kwargs.get('asHistoryBuffer',human)
what = 'UNIX_TIMESTAMP(data_time)' if unixtime else 'data_time'
if as_double:
what = 'CAST(%s as DOUBLE)' % what
if 'array' in table: what+=",idx"
value = 'value_r' if 'value_r' in self.getTableCols(table) \
else 'value'
if decimate and aggregate in ('AVG','MAX','MIN'):
value = '%s(%s)' % (aggregate,value)
what += ', ' + value
if extra_columns:
what+=','+extra_columns
interval = 'where att_conf_id = %s'%aid if aid is not None \
else 'where att_conf_id >= 0 '
int_time = int_time and 'int_time' in self.getTableCols(table)
if int_time:
self.info('Using int_time indexing for %s' % table)
if start_date or stop_date:
start_date,start_time,stop_date,stop_time = \
Reader.get_time_interval(start_date,stop_date)
if int_time:
def str2mysqlsecs(date):
rt = fn.str2time(date)
return int(rt+self.get_mysqlsecsdiff(date))
if start_date and stop_date:
interval += (" and int_time between %d and %d"
%(str2mysqlsecs(start_date),
str2mysqlsecs(stop_date)))
elif start_date and fandango.str2epoch(start_date):
interval += (" and int_time > %d"
% str2mysqlsecs)
else:
if start_date and stop_date:
interval += (" and data_time between '%s' and '%s'"
%(start_date,stop_date))
elif start_date and fandango.str2epoch(start_date):
interval += " and data_time > '%s'"%start_date
query = 'select %s from %s %s' % (what,table,interval)
if decimate:
if isinstance(decimate,(int,float)):
d = int(decimate) or 1
else:
d = int((stop_time-start_time)/10800) or 1
# decimation on server side
query += ' group by FLOOR(%s/%d)' % (
'int_time' if int_time else 'UNIX_TIMESTAMP(data_time)',d)
query += ' order by %s' % ('int_time' if int_time else 'data_time')
if N == 1:
human = 1
if N < 0 or desc:
query+=" desc" # or (not stop_date and N>0):
if N:
query+=' limit %s'%abs(N if 'array' not in table else N*1024)
######################################################################
# QUERY
t0 = time.time()
self.warning(query.replace('where','\nwhere').replace(
'group,','\ngroup'))
try:
result = self.Query(query)
self.warning('read [%d] in %f s'%(len(result),time.time()-t0))
except MySQLdb.ProgrammingError as e:
result = []
if 'DOUBLE' in str(e) and "as DOUBLE" in query:
return self.get_attribute_values((aid,tid,table),start_date,
stop_date,desc,N,unixtime,extra_columns,decimate,human,
as_double=False,**kwargs)
else:
traceback.print_exc()
if not result or not result[0]:
return []
######################################################################
t0 = time.time()
if 'array' in table:
data = fandango.dicts.defaultdict(list)
for t in result:
data[float(t[0])].append(t[1:])
result = []
for k,v in sorted(data.items()):
l = [0]*(1+max(t[0] for t in v))
for i,t in enumerate(v):
if None in t:
l = None
break
l[t[0]] = t[1] #Ignoring extra columns (e.g. quality)
result.append((k,l))
if N > 0:
#for k,l in result:
#print((k,l and len(l)))
result = result[-N:]
if N < 0 or desc:
result = list(reversed(result))
self.debug('array arranged [%d] in %f s'
% (len(result),time.time()-t0))
t0 = time.time()
# Converting the timestamp from Decimal to float
# Weird results may appear in filter_array comparison if not done
# Although it is INCREDIBLY SLOW!!!
#result = []
#nr = []
#if len(result[0]) == 2:
#for i,t in enumerate(result):
#result[i] = (float(t[0]),t[1])
#elif len(result[0]) == 3:
#for i,t in enumerate(result):
#result[i] = (float(t[0]),t[1],t[2])
#elif len(result[0]) == 4:
#for i,t in enumerate(result):
#result[i] = ((float(t[0]),t[1],t[2],t[3]))
#else:
#for i,t in enumerate(result):
#result[i] = ([float(t[0])]+t[1:])
self.debug('timestamp arranged [%d] in %f s'
% (len(result),time.time()-t0))
t0 = time.time()
# Decimation to be done in Reader object
#if decimate:
## When called from trends, decimate may be the decimation method
## or the maximum sample number
#try:
#N = int(decimate)
##decimate = data_has_changed
#decimate =
#result = PyTangoArchiving.reader.decimation(
#result,decimate,window=0,N=N)
#except:
##N = 1080
#result = PyTangoArchiving.reader.decimation(result,decimate)
if human:
result = [list(t)+[fn.time2str(t[0])] for t in result]
if not desc and ((not stop_date and N>0) or (N<0)):
#THIS WILL BE APPLIED ONLY WHEN LAST N VALUES ARE ASKED
self.warning('reversing ...' )
result = list(reversed(result))
#else:
## why
#self.getCursor(klass=MySQLdb.cursors.SSCursor)
self.debug('result arranged [%d]'%len(result))
return result
