def add_alert(self, pvname=None,name=None,
mailto=None, mailmsg=None, timeout=30,
compare='ne', trippoint=None, **kw):
"""add a new alert"""
if pvname is None: return
pvname = normalize_pvname(pvname)
if name is None: name = pvname
if len(self.pvnames)== 0: self.get_pvnames()
if pvname not in self.pvnames:
self.add_pv(pvname)
active = 'yes'
if mailto is None: active,mailto = ('no','')
if mailmsg is None: active,mailmsg= ('no','')
if trippoint is None: active,trippoint = ('no',0.)
if compare not in self.optokens: compare = 'ne'
self.alerts.insert(name=name,pvname=pvname,active=active,
mailto=mailto,mailmsg=mailmsg, timeout=30,
compare=compare, trippoint=trippoint)
for a in self.get_alerts(pvname=pvname,name=name):
val = epics.caget(pvname)
self.check_alert(a['id'],val)
def add(self,pvname=None,name=None,
mailto=None, mailmsg=None,
compare='ne', trippoint=None, **kw):
if pvname is None:
print 'must provide a pvname for an alert'
return
pvname = normalize_pvname(pvname)
if name is None: name = pvname
if len(self.pvnames)== 0: self.get_pvnames()
if pvname not in self.pvnames: self.add_pv(pvname)
active = 'yes'
if mailto is None:
mailto =''
active = 'no'
if mailmsg is None:
mailmsg =''
active = 'no'
if trippoint is None:
trippoint = 0.
active = 'no'
if compare in self.comps: compare = 'ne'
self.alerts.update(name=name,pvname=pvname,active=active,
mailto=mailto,mailmsg=mailmsg,
compare=compare, trippoint=trippoint)
def setup(self,active_tab=None,formkeys=None,debug=False, helpsection='main', **kw):
""" update self.kw with passed keywords, using
'web form' versions as defaults (see below). Also
starts the html and shows starting links
For each key in formkeys:
if the self.kw has an empty value ('' or None)
the 'form_%s' % key is used as a default value
and self[key] is set to the 'form_%s' version
"""
self.kw.update(kw)
if formkeys is not None and len(formkeys)>0:
for key in formkeys:
val = clean_input(self.kw.get(key,''))
if val == '':
val = clean_input(self.kw.get("form_%s" % key,''))
if val != '':
self.kw[key] = val
self.starthtml()
inst = self.kw.get('inst_id',-1)
pv = self.kw.get('pv','')
pv = normalize_pvname(pv)
self.show_links(pv=pv,inst=inst,help=helpsection,active_tab=active_tab)
if debug: self.show_dict(self.kw)
return
def drop_pv(self,pvname):
"""drop a PV from the caching process -- really this 'suspends updates'
will take effect once a 'process_requests' is executed."""
npv = normalize_pvname(pvname)
if len(self.pvnames)== 0: self.get_pvnames()
if not npv in self.pvnames: return
cmd = "insert into requests (pvname,action) values ('%s','suspend')" % npv
self.db.execute(cmd)
def get_instruments_with_pv(self,pv):
""" return a list of (instrument ids, inst name, station name) for
instruments which contain a named pv"""
inames = []
pvn = normalize_pvname(pv)
for r in self.inst_pvs.select(where="pvname='%s'" % pvn):
inst = self.instruments.select_one(where="id=%i" % r['inst'])
sta = self.stations.select_one(where="id=%i" % inst['station'])
inames.append((r['inst'],inst['name'],sta['name']))
return inames
def get_alerts(self,pvname=None,name=None):
""" return a list of alerts for a pvname"""
where = '1=1'
if pvname is not None:
pvname = normalize_pvname(pvname)
where = "pvname='%s'" % pvname
if name is not None:
where = "%s and name='%s'" % (where,clean_input(name))
return self.alerts.select(where=where)
def show_pvfile(self,file):
try:
f = open(file,"r")
lines = f.readlines()
f.close()
except:
lines = ['', '']
for s in lines:
s = s.strip()
if s.startswith('#') or len(s) < 2:
continue
elif s.startswith('['):
i = s.find(']')
if i == -1: i= len(s)
self.table_label(s[1:i])
elif s.startswith('--'):
self.write(self.hrule_row)
elif s.startswith('<>'):
self.write(self.space_row)
else:
cline = s.split('|')
pvnames = [normalize_pvname(word.strip()) for word in cline[0].split(',')]
desc, format, outtype = None, None, None
if len(cline) > 1:
desc = cline[1].strip()
if len(desc)==0:
desc = None
if len(cline) > 2:
format = cline[2].strip()
if format == 'yes/no':
format = None
outtype = 'yes/no'
if len(pvnames) > 0:
vals = []
labs = []
for pvname in pvnames:
# pvname = pvname.strip()
(label,val) = self.get_pv(pvname,format=format,desc=desc,outtype=outtype)
# self.write("<tr> <td>:: %s</td><td>%s</td></tr>" % (pvname,str(val)))
if (label is None): label = pvname
if (label.startswith('"') and label.endswith('"') or
label.startswith("'") and label.endswith("'")):
label = label[1:len(label)-1]
labs.append(label)
vals.append(val)
if len(pvnames) > 1:
if desc is None: desc = labs
self.linked_row(desc, pvnames, vals)
# self.cache.set_allpairs(pvnames)
else:
if desc is None: desc = labs[0]
self.linked_row(desc, pvnames[0], vals[0])
def request_pv_cache(self,pvname):
"""request a PV to be included in caching.
will take effect once a 'process_requests' is executed."""
self.db.use(master_db)
npv = normalize_pvname(pvname)
if len(self.pvnames)== 0: self.get_pvnames()
if npv in self.pvnames: return
cmd = "insert into requests (pvname,action,ts) values ('%s','add',%f)" % (npv,time.time())
self.db.execute(cmd)
print 'REQUEST_PV_CACHE: add ', pvname
def __make_where(self,id=None, name=None,pvname=None):
where = '1=1'
if id is not None:
where = "%s and id=%i" % (where,int(id))
if name is not None:
where = "%s and name='%s'" % (where,clean_input(name))
if pvname is not None:
where = "%s and pvname='%s'" % (where, normalize_pvname(pvname))
return where
def remove_instrument_pvs(self,pvlist,name=None,station=None,notes=''):
insts = self.get_instruments(name=name,station=station)
if len(insts)> 1:
print 'multiple instruments matched name/station=%s/%s' % (name,station)
return None
if not isinstance(pvlist,(tuple,list)):
print 'must provide list of PVs for instrument %s' % (name)
return None
inst_id = insts[0]['id']
q = "delete from instrument_pvs where inst=%i and pvname='%s'"
for pvname in pvlist:
pvn = normalize_pvname(pvname)
self.inst_pvs.db.execute(q % (inst_id, pvn))
def get_full(self,pv,add=False):
" return full information for a cached pv"
npv = normalize_pvname(pv)
if len(self.pvnames)== 0: self.get_pvnames()
if add and (npv not in self.pvnames):
self.add_pv(npv)
sys.stdout.write('adding PV.....\n')
return self.get_full(pv,add=False)
w = self.q_getfull % npv
try:
return self.sql_exec_fetch(w)[0]
except:
return self.null_pv_value
def get_value_at_time(self,pvname,t):
"return archived value of a pv at one time"
if pvname is None: return None
pvname = normalize_pvname(pvname)
info = self.get_info(pvname)
if info is None: return None
db = self.dbs_for_time(t,t)[0]
self.db.use(db)
qpv = "select data_table,id from pv where name ='%s'" % pvname
qdat = 'select time,value from %s where pv_id=%i and time<=%f order by time desc limit 1'
i = self.db.exec_fetchone(qpv)
r = self.db.exec_fetchone(qdat % (i['data_table'],i['id'],t))
return r['time'],r['value']
def add_pv(self,pvname,set_motor_pairs=True):
"""adds a PV to the cache: actually requests the addition, which will
be handled by the next process_requests in mainloop().
Here, we check for 'Motor' PV typs and make sure all motor fields are
requested together, and that the motor fields are 'related' by assigning
a pair_score = 10.
"""
pvname = normalize_pvname(pvname.strip())
fields = (pvname,)
if not valid_pvname(pvname):
sys.stdout.write("## MasterDB add_pv invalid pvname = '%s'" % pvname)
return fields
prefix = pvname
isMotor = False
if pvname.endswith('.VAL'):
prefix = pvname[:-4]
p = epics.PV(pvname)
p.wait_for_connection(timeout=0.1)
print 'Master Add PVname = ', pvname, p.connected
if p.connected:
self.request_pv_cache(pvname)
if ('.' not in prefix and p.type == 'double'):
rtype = epics.PV(prefix+'.RTYP')
rtype.wait_for_connection(0.1)
if rtype is not None:
isMotor = 'motor' == rtype.get()
if isMotor:
fields = tuple(["%s%s" % (prefix,i) for i in motor_fields])
pvs = []
for pvname in fields:
pvs.append(epics.PV(pvname))
epics.poll()
for p in pvs:
p.wait_for_connection(timeout=0.5)
if p.connected:
self.request_pv_cache(p.pvname)
if isMotor and set_motor_pairs:
time.sleep(0.25)
self.set_allpairs(fields)
return fields
def sync_with_cache(self,update_vals=False):
""" initialize pv lists, insert times, etc with cache
use update_vals to force insert of cache values into
the archive (as on startup)
n """
newpvs = []
cache_values = []
self.get_cache_names()
pvtable_data = self.pv_table.select()
print ' This is sync with cache ', update_vals, len(self.cache_names), len(pvtable_data)
self.db.use(self.master_db)
now = time.time()
print 'masterdb %s / data=%s' % ( self.master_db, len(pvtable_data))
if update_vals:
x = self.db.exec_fetch("select pvname,value,ts from cache")
current_cache = {}
for i in x:
current_cache[i['pvname']] = i
for pvdata in pvtable_data:
name = normalize_pvname(pvdata['name'])
if name not in self.cache_names:
newpvs.append((name, epics.PV(name)))
elif update_vals:
r = current_cache.get(name,None)
if r is not None:
ts = r['ts']
if now - ts > SEC_DAY: ts = now
cache_values.append((name,ts,r['value']))
if len(newpvs)>0:
epics.poll()
m = MasterDB()
for pvname, pv in newpvs:
if pv.connected:
m.add_pv(pvname)
m.close()
# now switch to archiving database, and (perhaps) insert values from cache
if self.dbname is not None:
self.db.use(self.dbname)
for name,ts,value in cache_values:
self.update_value(name,ts,value)
print 'Sync with Cache Done'
def get_related_pvs(self,pv,minscore=1):
"""return a list of related pvs to the provided pv
with a minumum pair score"""
out = []
tmp = []
npv = normalize_pvname(pv)
if len(self.pvnames)== 0: self.get_pvnames()
if npv not in self.pvnames: return out
for i in ('pv1','pv2'):
where = "%s='%s' and score>=%i order by score"
for j in self.pairs.select(where = where % (i,npv,minscore)):
tmp.append((j['score'],j['pv1'],j['pv2']))
tmp.sort()
for r in tmp:
if r[1] == npv: out.append(r[2])
elif r[2] == npv: out.append(r[1])
out.reverse()
return out
def set_instrument_pvs(self,pvlist,name=None,station=None,notes=''):
insts = self.get_instruments(name=name,station=station)
if len(insts)> 1:
print 'multiple instruments matched name/station=%s/%s' % (name,station)
return None
if not isinstance(pvlist,(tuple,list)):
print 'must provide list of PVs for instrument %s' % (name)
return None
inst_id = insts[0]['id']
self.get_pvnames()
for pvname in pvlist:
pvn = normalize_pvname(pvname)
where = "inst=%i and pvname='%s'" % (inst_id,pvn)
x = self.inst_pvs.select_one(where=where)
if x == {}:
x = self.inst_pvs.insert(pvname=pvn,inst=inst_id)
if pvn not in self.pvnames: self.add_pv(pvn)
# set pair scores
allpvs = self.get_instrument_pvs(name=name,station=station)
if len(allpvs) > 1:
self.set_allpairs(allpvs)
def __get_pvpairs(self,pv1,pv2):
"fix and sort 2 pvs for use in the pairs tables"
p = [normalize_pvname(pv1),normalize_pvname(pv2)]
p.sort()
return tuple(p)
def get_data(self,pvname,t0,t1,with_current=None):
"get data from database for a time range"
if pvname is None: return []
pvname = normalize_pvname(pvname)
info = self.get_info(pvname)
if info is None:
return ([],'No PV named %s',pvname)
#
stat = [info]
dat = []
pvquery= "select data_table,id from pv where name ='%s'" % pvname
fquery = 'select time,value from %s where pv_id=%i and time<=%f order by time desc limit 1'
gquery = 'select time,value from %s where pv_id=%i and time>=%f order by time limit 1'
squery = 'select time,value from %s where pv_id=%i and time>=%f and time<=%f order by time'
needs_firstpoint = True
tnow = time.time()
# make sure t0 and t1 are ordered
if t0 > t1: t0,t1 = t1,t0
if t1-t0 < 1800.0: t0 = t1 - 1800.0 # look back at least an hour
# determine if we should append the current (cached) value
if with_current is None:
add_current = abs(t1-tnow) < 1.5 * SEC_DAY
else:
add_current = with_current
# print 'get data for ' ,pvname, t0,t1
# print 'dbs: ', self.dbs_for_time(t0,t1)
try:
for db in self.dbs_for_time(t0,t1):
self.db.use(db)
stat.append(pvquery)
r = self.db.exec_fetchone(pvquery)
try:
table = r['data_table']
pvid = r['id']
except KeyError: # this db doesn't know about this PV -- maybe it's a recent addition?
continue
stat.append((db,table, pvid))
if needs_firstpoint:
q = fquery % (table,pvid,t0)
stat.append(q)
r = self.db.exec_fetchone(q)
try:
dat.append((r['time'],r['value']))
needs_firstpoint = False
except:
stat.append('no data before t0!')
q = squery % (table,pvid,t0,t1)
stat.append(q)
for i in self.exec_fetch(q):
dat.append((i['time'],i['value']))
# add value at time just after selected time range
r = self.db.exec_fetchone(gquery % (table,pvid,t1))
try:
dat.append((r['time'],r['value']))
except KeyError:
pass
# optionally, add current value
if add_current:
stat.append('adding cached value')
r= self.get_cache_full(pvname)
stat.append(r)
if r['value'] is not None:
dat.append((time.time(),r['value']))
except:
stat.append('Exception!')
dat.sort()
self.use_currentDB()
return dat,stat
def show_for_pv(self,pvname):
pvname = normalize_pvname(pvname)
for i in self.alerts.select(where="pvname='%s'" % pvname):
print i
def add_pv(self,name,description=None,graph={},deadtime=None,deadband=None):
"""add PV to the database: expected to take a while"""
pvname = normalize_pvname(name)
t0_start = time.time()
if not valid_pvname(pvname):
sys.stdout.write("## Archiver add_pv invalid pvname = '%s'" % pvname)
return None
if pvname in self.pvinfo:
if 'yes' == self.pvinfo[pvname]['active']:
self.write("PV %s is already in database.\n" % pvname)
else:
self.write("PV %s is in database, reactivating!\n" % pvname)
self.pvinfo[pvname]['active'] = 'yes'
return None
# create an Epics PV, check that it's valid
try:
pv = epics.PV(pvname)
pv.connect()
pv.get_ctrlvars()
typ = pv.type
count = pv.count
prec = pv.precision
connected = pv.connected
except:
typ= 'int'
count = 1
prec = None
connected = False
if not connected:
self.write("cannot add PV '%s': not connected" % pvname)
return None
# determine type
dtype = 'string'
if (typ in ('int','long','short')): dtype = 'int'
if (typ in ('enum',)): dtype = 'enum'
if (typ in ('double','float')): dtype = 'double'
# determine data table
table = "pvdat%3.3i" % ((hash(pvname) % 128) + 1)
# determine descrption (don't try too hard!)
if description is None:
if pvname.endswith('.VAL'):
descpv = "%s.DESC" % pvname[:-4]
else:
descpv = "%s.DESC" % pvname
for f in motor_fields:
if pvname.endswith(f):
descpv = None
if descpv is not None:
try:
dp = epics.PV(descpv)
description = dp.get(as_string=True)
## dp.disconnect()
except:
pass
if description is None: description = ''
# set graph default settings
gr = {'high':'','low':'','type':'normal'}
gr.update(graph)
if dtype == 'enum':
x = pv.get(as_string=True)
gr['type'] = 'discrete'
gr['low'] = 0
gr['high'] = len(pv.enum_strs)
elif dtype == 'double':
gr['type'] = 'normal'
dx = description.lower()
for i in ('cathode','pirani','pressure'):
if dx.find(i) >= 0:
gr['type'] = 'log'
if (deadtime == None):
deadtime = config.pv_deadtime_dble
if dtype in ('enum','string'): deadtime = config.pv_deadtime_enum
if (gr['type'] == 'log'): deadtime = 5.0 # (pressures change very frequently)
if (deadband == None):
deadband = 1.e-5
if (gr['type'] == 'log'): deadband = 1.e-4
if prec is not None: deadband = 10**(-(prec+1))
if dtype in ('enum','string'): deadband = 0.5
self.write('Archiver adding PV: %s, table: %s' % (pvname,table))
self.pv_table.insert(name = pvname,
type = dtype,
description= description,
data_table = table,
deadtime = deadtime,
deadband = deadband,
graph_lo = gr['low'],
graph_hi = gr['high'],
graph_type = gr['type'])
dat = self.pv_table.select_where(name=pvname)[0]
dat['force_time'] = get_force_update_time()
dat['last_ts'] = 0
dat['last_value'] = None
self.pvinfo[pvname] = dat
self.update_value(pvname,time.time(),pv.value)
self.write(" time=%f\n" % (time.time() - t0_start))
sys.stdout.flush()
pv = None
def modify_instrument(self,**kw):
' form to add an instrument'
inst_id = 0
mykw = {'station':'','instrument':'','submit':'', 'form_id':0,'inst_id':inst_id}
mykw.update(kw)
if mykw['inst_id'] != 0:
inst_id = int( mykw['inst_id'] )
else:
tmp = int(mykw['form_id'].strip())
if tmp != 0: inst_id = tmp
if inst_id == 0 and mykw['instrument'] != '' and mykw['station'] != '':
inst_id = self.arch.get_instrument_id(name=mykw['instrument'],
station=mykw['station'])
instrument,station = self.arch.get_instrument_names_from_id(inst_id)
self.starthtml()
self.show_links(help='instruments',active_tab='Instruments')
wr = self.write
if mykw['submit'] != '':
sname = clean_input(mykw['name']).strip()
if sname != instrument and sname != '':
self.arch.instruments.update(name=sname,where="id=%i" % inst_id)
sdesc = clean_input(mykw['desc']).strip()
if sdesc != '':
self.arch.instruments.update(notes=sdesc,where="id=%i" % inst_id)
# add and remove pvs as directed
pvs_add = []
pvs_del = []
for k,v in mykw.items():
if 'remove' == v:
pvs_del.append(k)
elif k.startswith('_addpv_') and len(v) >0:
pvs_add.append(clean_input(normalize_pvname(v)))
if len(pvs_del)>0:
self.arch.remove_instrument_pvs(pvs_del, name=instrument, station=station)
if len(pvs_add)>0:
self.arch.set_instrument_pvs(pvs_add, name=instrument, station=station)
info = self.arch.get_instruments(name=instrument,station=station)[0]
pvlist = self.arch.get_instrument_pvs(name=instrument,station=station)
pvlist.sort()
wr('<form action ="%s/modify_instrument" enctype="multipart/form-data" method ="POST"><p>' % (mainpage))
wr('<input type="hidden" name="form_id" value="%s">' % inst_id)
wr("<h3>Definition for Instrument: %s in Station %s</h3> " % (instrument,station))
wr("""<table><tr><td> Instrument Name:</td><td><input type='text' name='name' value='%s' size=35></td></tr>
<tr><td> Description:</td><td><input type='text' name='desc' value='%s' size=35></td></tr>
<tr><td> </td><td></td></tr>
<tr><td colspan=2> Current PVs in instrument: </td><td></td></tr>
""" % (instrument,info['notes']))
for pvn in pvlist:
self.write("""<tr><td>%s</td><td>
<input type='radio' name='%s' value='remove'>remove
<input type='radio' checked='true' name='%s' value='keep'>keep</td></tr>""" % (pvn,pvn,pvn))
wr("<tr><td colspan=2> <hr> </td></tr> <tr><td colspan=2> Add PVs </td></tr>")
for i in range(4):
self.write("""<tr><td> PV </td><td>
<input type="text" name="_addpv_%i" value="" size=35></td></tr>""" % i)
wr("""<tr><td><input type='submit' name='submit' value='Update Definition'></td>
<td><a href='%s?station=%s&instrument=%s'>View Positions for %s</a></td>
</tr></table>""" % (instpage,station,instrument,instrument))
self.endhtml()
return self.get_buffer()