def reassemble_measurement(self, piece):
# look for the right Measurement to add this Piece to
# if we found it, add it to that Measurement
piece_key = str(piece) #str method is overridden so this is efficient and unique
if piece_key in self.measurements_dict.keys():
m = self.measurements_dict[piece_key]
m.place_piece(piece)
else:
# packet was not in any measurement, so make a new one
new_measurement = Measurement(piece)
self.measurements_dict[str(new_measurement)] = new_measurement
self.measurements_list.append(str(new_measurement))
# check to find Measurements that are complete
# only one Measurement could possibly be completed (the one that the
# current Piece fits into)
measurement = self.measurements_dict[piece_key]
if measurement.is_complete():
# sends Measurement to personality to parse the raw BRAM data,
# then sends the parsed data to the data server for writing
try:
self.record_measurement(measurement)
# self.personality._reconstructMeasurement(measurement))
except Exception, e:
corelog.exception("%s Could not record measurement" % self.name)
# remove it from list of Measurements
self.measurements_list.remove(piece_key)
del self.measurements_dict[piece_key]
def get(self):
"""
Get the TCT time string and parse it.
Returns a tuple containing:
* UnixTime stamp as an integer
* TCT status byte. 0 is good
* Offset between system time and TCT time in seconds as a float
"""
self.ser.flushInput()
txtime = 18*10/9600. #Amount of time to transmit 18 characters at 9600bps
r = self.ser.readline(eol='\r')
now = time.time()-txtime
if len(r) != 18:
corelog.warning("TCT did not transmit")
return None
# 012345678901234567
#Format is YYYYjjjHHMMSSrCCC\r
try:
Y = int(r[:4]) #year
doy = int(r[4:7]) #day of year
H = int(r[7:9]) #hour
M = int(r[9:11]) #min
S = int(r[11:13]) #sec
status = int(r[13],base=16)
checksum = int(r[14:17],16)
except Exception, e:
corelog.exception("Couldn't parse TCT output")
return None
def __init__(self):
Pyro.core.ObjBase.__init__(self)
try:
Synthesizer.__init__(self,config.serialPortMap['Valon'])
except:
corelog.exception("Couldn't open Valon serial port")
return
self.conn.setTimeout(1)
self.setup()
def __init__(self):
try:
self.pdb = gavrtdb.GavrtDB(rw=True)
except:
corelog.exception("Could not connect to GavrtDB")
try:
self.fs = FORXServer()
except:
corelog.exception("Could not connect to Fiber Monitor LabJack. Perhaps FORX mon already running?")
def loop(self,interval=3600):
while True:
corelog.info("Getting UPS data and inserting in database...")
try:
self.getData()
self.insertData()
except:
corelog.exception("Failed to get UPS data and insert to database")
corelog.debug("UPSLogger sleeping for %d seconds" % interval)
time.sleep(interval)
def prepare_for_writing(self, filename):
"""
prepares the real h5 (history) file to be written
"""
self.filename = filename
if os.path.isfile(filename):
corelog.warning("%s h5 filename %s already exists, overwriting..." % (self.name,filename))
with self.h5_lock:
# try to open the file for writing
try:
self.h5 = tables.openFile(self.filename,'w')
except Exception, e:
self.h5 = None
self.filename = None
corelog.exception("%s could not open new h5 file for writing: %s" % (self.name,self.filename))
return
self.comment_table = self.h5.createTable(self.h5.root, "comment_table",
self.comment_table_description())
# set up tables for each iBOB based on each iBOB's personality
personality = self.personality
iBOB_group = self.h5.root
self.h5.createTable(iBOB_group, "file_info", dict(personality=tables.StringCol(128,dflt=' ')))
iBOB_group.file_info.row['personality'] = self.personality.__class__.__name__
iBOB_group.file_info.row.append()
self.h5.flush()
self.iBOB_group = iBOB_group
self.measurements = {}
iBOB_meas = self.measurements
measurement_types = personality._measTypesDict
for measurement_type in measurement_types.keys():
iBOB_meas[measurement_type] = {}
meas_grp = self.h5.createGroup(iBOB_group,measurement_type)
iBOB_meas[measurement_type]['group'] = meas_grp
thistable = \
self.h5.createTable(meas_grp, 'table',
measurement_types[measurement_type]['table'],
expectedrows=2000)
iBOB_meas[measurement_type]['table'] = thistable
iBOB_meas[measurement_type]['arrays'] = {}
for name,shape in measurement_types[measurement_type]['arrays'].items():
fullshape = tuple([0]+list(shape))
thisarr = self.h5.createEArray(meas_grp, name,
tables.Float32Atom(),fullshape)
iBOB_meas[measurement_type]['arrays'][name] = thisarr
self.spec_info_table = self.h5.createTable(iBOB_group, "InfoTable",
personality._infoTable, expectedrows = 2000)
def _init_rtbuf(self):
self.realtime_filename = "/tmp/rt%d.h5" % self.id
corelog.info("Starting realtime data capture. Creating %s" %self.realtime_filename)
# try to open the realtime h5 file for writing
with self.realtime_lock:
try:
self.realtime_h5 = tables.openFile(self.realtime_filename, "w")
except Exception, e:
corelog.exception("could not open new realtime h5 file for writing: %s" % self.realtime_filename)
self.realtime_h5 = None
self.realtime_filename = None
return
# if no iBOBs are registered, error and quit
personality = self.personality
iBOB_group = self.realtime_h5.root
self.realtime_h5.createTable(iBOB_group, "file_info", dict(personality=tables.StringCol(128,dflt=' ')))
iBOB_group.file_info.row['personality'] = self.personality.__class__.__name__
iBOB_group.file_info.row.append()
iBOB_group.file_info.flush()
corelog.debug("set personality: %s" % str(iBOB_group.file_info[:]))
self.realtime_iBOB_group = iBOB_group
self.realtime_measurements = {}
iBOB_meas = self.realtime_measurements
measurement_types = personality._measTypesDict
for measurement_type in measurement_types.keys():
corelog.debug("adding measurement type: %s" % measurement_type)
iBOB_meas[measurement_type] = {}
meas_grp = self.realtime_h5.createGroup(iBOB_group,measurement_type)
iBOB_meas[measurement_type]['group'] = meas_grp
thistable = \
self.realtime_h5.createTable(meas_grp, 'table',
measurement_types[measurement_type]['table'],
expectedrows=2000)
iBOB_meas[measurement_type]['table'] = thistable
iBOB_meas[measurement_type]['arrays'] = {}
iBOB_meas[measurement_type]['index'] = self.realtime_h5.createArray(meas_grp, 'index', np.zeros((1,),dtype='uint32'))
for name,shape in measurement_types[measurement_type]['arrays'].items():
if (name.lower().find('adc') >= 0):
fullshape = tuple([16]+list(shape)) # kludge to reduce wasted space on lots of adc snapshots
elif shape[0] > 1024:
fullshape = tuple([2**20/shape[0]]+list(shape)) #keep size = 1Mpoint
else:
fullshape = tuple([MAX_REALTIME_ROWS]+list(shape))
thisarr = self.realtime_h5.createArray(meas_grp, name, np.zeros(fullshape,dtype='float32'))
iBOB_meas[measurement_type]['arrays'][name] = thisarr
self.realtime_infotable = \
self.realtime_h5.createTable(iBOB_group, "InfoTable",
personality._infoTable, expectedrows = 2000)
def lockStatus(self):
try:
a = self.get_phase_lock(SYNTH_A)
b = self.get_phase_lock(SYNTH_B)
except:
corelog.exception("Couldn't get lock status")
if not (a and b):
corelog.warning("Valon not in lock - A: %s B: %s" %(str(a),str(b)))
else:
corelog.info("Valon synthesizer is in lock")
return (a,b)
def setFreqB(self,freq,chan_spacing=1):
try:
r = self.set_frequency(SYNTH_B,freq,chan_spacing=chan_spacing)
except:
corelog.exception("Couldn't set frequency %f" % freq)
return False
if r:
corelog.info("Set frequency to %f",freq)
else:
corelog.warning("Set frequency reply was NOACK")
return r
def setLevelB(self,level=-1):
try:
r = self.set_rf_level(SYNTH_B,level)
except:
corelog.exception("Couldn't set RF level %f" % level)
return False
if r:
corelog.info("Set level to %d",level)
else:
corelog.warning("Set level reply was NOACK")
return r
def _startRecord(self,ibobids,datapath):
corelog.info("Got request to record from ibobs: %s In path: %s" % (str(ibobids),datapath))
try:
os.mkdir(datapath)
os.chmod(datapath,stat.S_IRWXO | stat.S_IRWXG | stat.S_IRWXU)
except:
corelog.exception("Couldn't create data path %s" % datapath)
for id,ibob in self.iBOBProxies.items():
if id in ibobids:
ibob.start_writing(os.path.join(datapath,'ibob%d.h5' % id))
def set_personality(self, personality,adcClock=1024.0):
"""
sets the personality of a specific iBOB
"""
if self.writing or self.personality:
corelog.warning("%s cannot set personality: it's already set" % self.name)
return
corelog.debug("%s Setting personality %s %s" % (self.name,str(personality), personalities.__file__))
self.personality = personality(adcClock=adcClock) # removed self as parent
try:
self._init_rtbuf()
except Exception,e:
corelog.exception("Could not init realtime buf for ibob %d %s"%(self.id,str(self.personality)))
def getData(self,passwd=SPSS_UPS.passwd):
"""
Retrieve data via FTP from UPS
"""
if passwd is None:
print "Enter password for root on UPS: ",
passwd = raw_input()
corelog.info("Retrieving data from UPS")
try:
self.ftp = FTP(self.addr)
self.ftp.login(user=SPSS_UPS.user,passwd=passwd)
except Exception, e:
corelog.exception("Couldn't connect to UPS by FTP at address %s with user root and password %s" % (self.addr,passwd))
def record_measurement(self, measurement):
"""
save data to realtime h5 file for recent access, save to history file if
necessary
spec_measurement is a tuple from the spectrometer personality,
[0] = name
[1] = arraydict
[2] = tabledict
"""
# print "start of record measurement",iBOB_id
personality = self.personality
if personality is None:
return
spec_measurement = personality._reconstructMeasurement(measurement)
measurement_type = spec_measurement[0]
arrays = spec_measurement[1]
table_data = spec_measurement[2]
# print measurement_type, len(table_data),table_data
if measurement_type == 'S':
acc = table_data['AccNumber']
if acc - self.acc != 1:
print "missed:",self.id,acc,self.acc
self.acc = acc
#self.publish('msr', spec_measurement)
self.number_of_measurements += 1
# always write to realtime h5 file
# table_string = "%s_%s" % (measurement_type, "table_description")
if not self.realtime_h5:
corelog.warning("%s no realtime_h5 file opened yet, record_measurement failed" % self.name)
return
with self.realtime_lock:
table = self.realtime_measurements[measurement_type]['table']
if len(table) >= MAX_REALTIME_ROWS:
# TODO: change this to not delete just one row
table.removeRows(0)
# TODO: hopefully pytables reassigns the indices of table.row, otherwise
# we have to adjust...
for key in table_data.keys():
try:
table.row[key] = table_data[key]
except Exception, e:
corelog.exception("%s could not insert data for key %s" %(self.name,str(key)))
#print "added data for ",key
try:
table.row.append()
except Exception, e:
corelog.exception("%s could not append row to realtime h5 file" % self.name)
def __init__(self,port=None):
if port is None:
try:
port =config.serialPortMap['TCT']
except:
corelog.exception("TCT entry not found in serialPortMap. Manually searching for port")
port = self.findport()
corelog.info("Using port %s",port)
self.ser = serial.Serial(port)
self.ser.setBaudrate(9600)
self.ser.setParity('O')
self.ser.setByteSize(7)
self.ser.setTimeout(2)
self.ser.flushInput()
self.running = True
self.gdb = gavrtdb.GavrtDB(rw=True)
def update(self):
"""
Get the TCT time and status and insert it in the database.
"""
res = self.get()
if res is None:
return
ut,status,offset =res
try:
c = self.gdb.db.cursor()
c.execute("INSERT INTO tct_status (UnixTime,Status,Offset) VALUES (%s,%s,%s);",
(ut,status,offset))
self.gdb.db.commit()
except Exception, e:
corelog.exception("Couldn't write to database")
try:
self.gdb = gavrtdb.GavrtDB(rw=True)
except Exception, e:
corelog.exception("Couldn't reconnect to database")
def parseData(self,fname):
from matplotlib import mlab
corelog.debug("Parsing UPS data file: %s",fname)
def myfloat(x):
try:
return float(x)
except:
return 0.0
names = ['date','time','vmin','vmax','vout','wout','freq','cap','vbat','tupsc']
convd = {}
for k in names:
convd[k] = myfloat
dp = dateutil.parser.parse
convd['date'] = dp
convd['time'] = dp
try:
res = mlab.csv2rec(fname,skiprows=6,names=names,converterd = convd,delimiter='\t')
except Exception,e:
corelog.exception("Could not parse UPS data file")
def send_cmd(self,cmd,timeout=5):
"""
Send a command to XANT and get response.
"""
if not self.checkConnected():
self.connect()
while True:
try:
self.sock.recv(10000)
except:
break
tic = time.time()
try:
self._send(cmd)
except Exception,e:
corelog.exception("Couldn't send command to XANT")
return
def __init__(self,sql = True):
"""
*sql* argument is largely a relic, no longer used without database
"""
Pyro.core.ObjBase.__init__(self)
Pyro.config.PYRO_MULTITHREADED = 0 # do not use multiple threads, because we want to ensure programming finishes
if sql:
self.sql = True
corelog.debug("Connecting to GavrtDB")
self.gdb = gavrtdb.GavrtDB(rw = True)
else:
self.sql = False
self.gdb = None
self.iBOBProcesses = {}
self.iBOBServers = {}
self.serialPorts = [None,None]
self.lj0 = None
self.lj1 = None
self.ljs = [None,None]
self.syns = [None,None]
self.syn0 = None
self.syn1 = None
self.jtagcables = ['usb21','usb22']
corelog.debug("Identifying JTAG programmers from list: %s" % str(self.jtagcables))
try:
self.identifyCables()
except:
corelog.exception("Could not identify JTAG programmers!")
corelog.info("JTAG cables identified")
corelog.debug("Opening LabJacks")
try:
self.openLabJacks()
except:
corelog.exception("Could not open LabJacks!")
# Check if synthesizers are already initialized before clobbering them.
if not self.synsInitialized():
corelog.info("found synthesizers uninitialized, so initializing to 1024 MHz")
self.initialize()
self.setSynths(1024.0)
corelog.info("Started")
def insertData(self):
if self.data is None:
print "no data!"
return
corelog.info("Inserting UPS data: %d values" % (self.data.shape[0]))
ut = [calendar.timegm(x.timetuple()) for x in self.data['time']]
data = self.data
try:
c = self.gdb.cursor()
for k in range(len(ut)):
data = self.data[k]
c.execute("INSERT IGNORE INTO spss_ups (UnixTime,Vmin,Vmax,Vout,Wout,Freq,Capacity,Vbat,Temperature) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)",
(ut[k],data['vmin'],data['vmax'],data['vout'],data['wout'],data['freq'],data['cap'],data['vbat'],data['tupsc']))
self.gdb.commit()
except Exception, e:
corelog.exception("Couldn't write UPS data to database")
try:
self.gdb = gavrtdb.GavrtDB(rw=True)
except Exception, e:
corelog.exception("Couldn't reconnect to database")
def programIBOBs(self,ibobs,ipfid=None,ipffile=None,force=False):
"""
Program a list of iBOBs with a given configuration file.
The configuration file can be explicitly provided or an index to the database of designs can be given
Arguments:
*ibobs* : list of ints
Which ibobs to program
*ipfid* : int
Record ID of row in dss28_spec.ibob_designs that indicates the desired configuration
*ipffile* : str (advanced use only)
Instead of a design in the database, use this IPF explicitly
*force* : bool
Program the ibob even if we think it may already be programmed with same design.
"""
if self.sql:
if ipfid:
try:
resd = self.gdb.getIPF(ipfid)
ipf = resd['IPF']
ipfdir = resd['Directory']
ipffile = os.path.join(ipfdir,ipf)
except Exception, e:
corelog.exception("Could not resolve programming file from database")
raise e
corelog.debug("Found ipfID: %d correspoing to %s" % (ipfid, ipffile))
else:
ipfid = 1 #we are connected to database, but programming an arbitrary file, so
# tell the database that the design is default
if force == False:
droplist = []
stat = self.gdb.getSPSSStatus()
for ii in ibobs:
if stat["iBOBDesignID%d" % ii] == ipfid:
droplist.append(ii)
corelog.info("iBOB%d is already programmed with this ipf, skipping" % ii)
for k in droplist:
ibobs.remove(k)
def loop(self):
while True:
try:
data = self.fs.getStatus()
except:
corelog.exception("Could not get Fiber Monitor data from LabJack")
time.sleep(60)
continue
keystr = ''
valstr = ''
values = []
for k,v in data.items():
keystr = keystr + k + ', '
valstr = valstr + '%s, '
values.append(v)
keystr = keystr + 'UnixTime'
valstr = valstr + '%s'
values.append(time.time())
try:
c = self.pdb.cursor()
c.executemany(
"INSERT INTO forx_mon (" + keystr + ") VALUES (" + valstr + ");",
[tuple(values)])
print "INSERT INTO forx_mon (" + keystr + ") VALUES (" + valstr + ");",tuple(values)
self.pdb.commit()
except Exception, e:
corelog.exception("Could not log to GavrtDB, will try reconnecting")
try:
self.pdb = gavrtdb.GavrtDB(rw=True)
except Exception, e:
corelog.exception("Could not reconnect to GavrtDB")
def setSynth(self,freq,bank):
if bank == 0:
settings = self.syn0.setFreq(freq)
else:
settings = self.syn1.setFreq(freq)
if settings is None:
raise Exception("frequency is unreachable")
fset = settings['output']
if self.sql:
vald = {}
for ibob in range(4):
ibn = ibob + 4*bank
vald['iBOBADCClock%d' % ibn] = fset
vald['UnixTime'] = time.time()
try:
self.gdb.updateValues(vald, 'spss_config')
except:
corelog.exception("Could not record synthesizer setting in Database!")
return settings
def send_cmd(self,cmd,timeout=5):
"""
Main function for sending a commaand string to the RCT and receiving a response.
"""
try:
self.sock.getpeername()
except:
corelog.warn("Trying to send command but found not connected, so trying to connect")
self.connect()
while True:
try:
self.sock.recv(10000)
except:
break
tic = time.time()
try:
self.send(cmd)
corelog.debug("Sent command %s" % cmd)
except Exception,e:
corelog.exception("Could not send command %s" % cmd)
return
"""
:mod:`dss28core.config`
-----------------------
configuration infomation
Provides:
* :attr:`serialPortMap` : Dictionary mapping devices to serial ports (/dev/ttyUSB*)
"""
import grasp.utils as utils
from loggers import corelog
import os
try:
DSS28_DIR = os.environ["DSS28_DIR"] #'/home/dss28'
except:
raise Exception("DSS28_DIR environment variable must be defined!")
bitfiles_dir = os.path.join(DSS28_DIR, "bitfiles")
try:
serialPortMap = utils.unpickle(os.path.join(DSS28_DIR, "serialPorts.pkl"))
except Exception, e:
corelog.exception("Failed to load serialPorts.pkl configuration")
serialPortMap = dict()
corelog.exception("Couldn't send command to XANT")
return
tstart = time.time()
resp = ''
while (time.time()-tstart < timeout):
try:
resp = resp + self.sock.recv(1000)
if resp.find('completed') >= 0:
corelog.debug("Command %s accepted" % cmd)
break
elif resp.find('rejected') >= 0:
corelog.warning("Command %s rejected with response: %s" % (cmd,resp))
break
except:
corelog.exception("Problem waiting for response from XANT")
pass
corelog.debug("Xant response time: %.2f ms" %((time.time()-tic)*1000))
time.sleep(0.1) #sleep a moment
return resp
def runXantInterface():
import Pyro.naming
Pyro.core.initServer()
ns = Pyro.naming.NameServerLocator().getNS()
try:
ns.unregister('XantInterface')
except:
pass
daemon = Pyro.core.Daemon()
corelog.info("Retrieving data from UPS")
try:
self.ftp = FTP(self.addr)
self.ftp.login(user=SPSS_UPS.user,passwd=passwd)
except Exception, e:
corelog.exception("Couldn't connect to UPS by FTP at address %s with user root and password %s" % (self.addr,passwd))
fname = time.strftime(os.path.join(DSS28,'logs/ups/%Y.%j_%H%M%S_ups.txt'))
try:
dfile = open(fname,'w')
except Exception, e:
corelog.exception("Could not open file %s for writing UPS data" % fname)
try:
self.ftp.retrbinary("RETR data.txt", dfile.write)
except Exception, e:
corelog.exception("Could not retrieve UPS data file data.txt")
dfile.close()
self.ftp.close()
self.parseData(dfile.name)
def parseData(self,fname):
from matplotlib import mlab
corelog.debug("Parsing UPS data file: %s",fname)
def myfloat(x):
try:
return float(x)
except:
return 0.0
names = ['date','time','vmin','vmax','vout','wout','freq','cap','vbat','tupsc']
except Exception, e:
corelog.exception("%s could not append row to realtime h5 file" % self.name)
#print "appended row"
table.flush()
#print "added to table"
rtarrays = self.realtime_measurements[measurement_type]['arrays']
index = self.realtime_measurements[measurement_type]['index'][0]
for array_name in arrays.keys():
rtarray = rtarrays[array_name]
array_data = arrays[array_name]
# if too long, do wraparound, index is stored in
# table_dict[array_string + "_index"]
try:
rtarray[index] = array_data[np.newaxis,:]
except Exception, e:
corelog.exception("%s arrays have incompatible shapes %s shape is %s. Tried to append %s" % (self.name, array_name, rtarray[index].shape,array_data[np.newaxis,:].shape))
raise e
self.realtime_measurements[measurement_type]['index'][0] = \
(index + 1) % rtarrays[arrays.keys()[0]].shape[0]
self.realtime_h5.flush()
# array.removeRows(0, 1)
# print "added to array"
# only write to real history h5 file if we should
if self.writing:
# print table_data.keys(),arrays.keys()
if not self.h5:
corelog.warning("%s we are writing but no h5 file opened yet??, record_measurement failed" % self.name)
return
with self.h5_lock:
table = self.measurements[measurement_type]['table']
for key in table_data.keys():
