def get_projects(t, nosql=False):
''' Read all projects from SQL for the current date and return a summary
as a dictionary with keys Timestamp, Project, and EOS (another timestamp)
'''
if nosql == True:
return get_projects_nosql(t)
import dbutil
# timerange is 12 UT to 12 UT on next day, relative to the day in Time() object t
trange = Time([int(t.mjd) + 12. / 24, int(t.mjd) + 36. / 24], format='mjd')
tstart, tend = trange.lv.astype('str')
cursor = dbutil.get_cursor()
mjd = t.mjd
# Get the project IDs for scans during the period
verstrh = dbutil.find_table_version(cursor, trange[0].lv, True)
if verstrh is None:
print 'No scan_header table found for given time.'
return {}
query = 'select Timestamp,Project from hV' + verstrh + '_vD1 where Timestamp between ' + tstart + ' and ' + tend + ' order by Timestamp'
projdict, msg = dbutil.do_query(cursor, query)
if msg != 'Success':
print msg
return {}
elif len(projdict) == 0:
# No Project ID found, so return data and empty projdict dictionary
print 'SQL Query was valid, but no Project data were found.'
return {}
projdict['Timestamp'] = projdict['Timestamp'].astype(
'float') # Convert timestamps from string to float
for i in range(len(projdict['Project'])):
projdict['Project'][i] = projdict['Project'][i].replace('\x00', '')
projdict.update({'EOS': projdict['Timestamp'][1:]})
projdict.update({'Timestamp': projdict['Timestamp'][:-1]})
projdict.update({'Project': projdict['Project'][:-1]})
cursor.close()
return projdict
def find_gaincal(t=None, scan_length=6, findwhat='FEATTNTEST'):
# This will find the project on the day of "t" or, going backwards, the nearest
# day to "t"
# This routine returns a timerange that may be used as the timerange in any
# of the above programs. It will return the appropriate timerange for a
# FEATTNTEST scan on the date for which t is given to this function, or a
# list of appropriate time range. If the length of the scan is not 6 minute,
# please set scan_length to the appropriate length.
''' Makes an SQL query to find the FEATTNTEST or GAINCALTESTscans for the date given in
the Time() object t. A list of timestamps is returned, along with
the timestamp of the object provided.
'''
loop_ = 1
if t is None:
# Get today's date
t = util.Time.now()
timestamp = int(t.lv)
while loop_ == 1:
stimestamp = timestamp - (timestamp % 86400) # Start of day
etimestamp = stimestamp + 86399 # End of day
# Open handle to SQL database
cursor = dbutil.get_cursor()
# Try to find a scan header with project SOLPNTCAL (only works after 2014 Nov. 30)
verstr = dbutil.find_table_version(cursor,timestamp,True)
if verstr is None:
print 'No scan_header table found for given time.'
return [], timestamp
# First retrieve the Project from all scan headers for the day
cursor.execute('select timestamp,Project from hV'+verstr+'_vD1 where timestamp between '+str(stimestamp)+' and '+str(etimestamp)+' order by timestamp')
data = np.transpose(np.array(cursor.fetchall()))
names = stateframedef.numpy.array(cursor.description)[:,0]
cursor.close()
if len(data) == 0:
# No FEATTNTEST found, so return empty list (and timestamp)
return [], timestamp
else:
projdict = dict(zip(names,data))
projdict['timestamp'] = projdict['timestamp'].astype('float') # Convert timestamps from string to float
good = np.where(projdict['Project'] == findwhat)[0]
if len(good) != 0:
if len(good) == 1:
loop_ = 0
tgc = [projdict['timestamp'][good]], timestamp
start_= Time(tgc[0][0], format = 'lv').iso
end_ = Time(tgc[0][0]+60*scan_length, format = 'lv').iso
trange = Time([start_[0] , end_[0]])
return trange
else:
loop_ = 0
tgc = [projdict['timestamp'][good]], timestamp
start_trange = Time(tgc[0][0], format = 'lv').iso
end_trange = Time(tgc[0][0]+60*scan_length, format = 'lv').iso
tranges = []
for i in range(start_trange.shape[0]):
trange = Time([start_trange[i], end_trange[i]])
tranges.append(trange)
return tranges
else:
timestamp = timestamp - 60*60*24
loop_ = 1
def flare_monitor(t):
''' Get all front-end power-detector voltages for the given day
from the stateframe SQL database, and obtain the median of them,
to use as a flare monitor.
Returns ut times in plot_date format and median voltages.
'''
import dbutil
# timerange is 12 UT to 12 UT on next day, relative to the day in Time() object t
trange = Time([int(t.mjd) + 12. / 24, int(t.mjd) + 36. / 24], format='mjd')
tstart, tend = trange.lv.astype('str')
cursor = dbutil.get_cursor()
mjd = t.mjd
verstr = dbutil.find_table_version(cursor, tstart)
if verstr is None:
print 'No stateframe table found for given time.'
return tstart, [], {}
query = 'select Timestamp,Ante_Fron_FEM_HPol_Voltage,Ante_Fron_FEM_VPol_Voltage from fV' + verstr + '_vD15 where timestamp between ' + tstart + ' and ' + tend + ' order by timestamp'
data, msg = dbutil.do_query(cursor, query)
if msg != 'Success':
print msg
return tstart, [], {}
for k, v in data.items():
data[k].shape = (len(data[k]) / 15, 15)
hv = []
try:
ut = Time(data['Timestamp'][:, 0].astype('float'),
format='lv').plot_date
except:
print 'Error for time', t.iso
print 'Query:', query, ' returned msg:', msg
print 'Keys:', data.keys()
print data['Timestamp'][0, 0]
hfac = np.median(data['Ante_Fron_FEM_HPol_Voltage'].astype('float'), 0)
vfac = np.median(data['Ante_Fron_FEM_VPol_Voltage'].astype('float'), 0)
for i in range(4):
if hfac[i] > 0:
hv.append(data['Ante_Fron_FEM_HPol_Voltage'][:, i] / hfac[i])
if vfac[i] > 0:
hv.append(data['Ante_Fron_FEM_VPol_Voltage'][:, i] / vfac[i])
#import pdb; pdb.set_trace()
flm = np.median(np.array(hv), 0)
good = np.where(abs(flm[1:] - flm[:-1]) < 0.1)[0]
projdict = get_projects(t)
return ut[good], flm[good], projdict
def get_fseqfile(t=None):
if t is None:
# 10 s ago...
tlv = Time.now().lv - 10
else:
tlv = int(t.lv)
cursor = db.get_cursor()
ver = db.find_table_version(cursor,tlv)
# Get front end attenuator states
query = 'select Timestamp,LODM_LO1A_FSeqFile from fV'+ver+'_vD1 where Timestamp between '+str(tlv)+' and '+str(tlv+1)+' order by Timestamp'
data, msg = db.do_query(cursor, query)
if msg == 'Success':
fseqfile = data['LODM_LO1A_FSeqFile'][0].replace('\x00','')
if fseqfile == 'none':
fseqfile = None
else:
print 'Error: ',msg
fseqfile = None
cursor.close()
return fseqfile
def find_solpnt(t=None):
''' Makes an SQL query to find the SOLPNTCAL scans for the date given in
the Time() object t. A list of timestamps is returned, along with
the timestamp of the object provided.
'''
import dbutil
if t is None:
# Get today's date
t = util.Time.now()
timestamp = int(t.lv)
stimestamp = timestamp - (timestamp % 86400) # Start of day
etimestamp = stimestamp + 86399 # End of day
# Open handle to SQL database
cursor = dbutil.get_cursor()
# Try to find a scan header with project SOLPNTCAL (only works after 2014 Nov. 30)
verstr = dbutil.find_table_version(cursor, timestamp, True)
if verstr is None:
print 'No scan_header table found for given time.'
return [], timestamp
# First retrieve the Project from all scan headers for the day
cursor.execute('select timestamp,Project from hV' + verstr +
'_vD1 where timestamp between ' + str(stimestamp) +
' and ' + str(etimestamp) + ' order by timestamp')
data = np.transpose(np.array(cursor.fetchall()))
names = stateframedef.numpy.array(cursor.description)[:, 0]
cursor.close()
if len(data) == 0:
# No SOLPNTCAL found, so return empty list (and timestamp)
return [], timestamp
else:
projdict = dict(zip(names, data))
projdict['timestamp'] = projdict['timestamp'].astype(
'float') # Convert timestamps from string to float
good = np.where(projdict['Project'] == 'SOLPNTCAL')[0]
if len(good) != 0:
if len(good) == 1:
return [projdict['timestamp'][good]], timestamp
else:
return projdict['timestamp'][good], timestamp
else:
return [], timestamp
def findscans(trange):
'''Identify phasecal scans from UFDB files
'''
import dbutil
import dump_tsys
tstart, tend = trange.lv.astype(int).astype(str)
cursor = dbutil.get_cursor()
verstr = dbutil.find_table_version(cursor, tstart, True)
query = 'select Timestamp,Project,SourceID from hV'+verstr+'_vD1 where left(Project,8) = "PHASECAL" and Timestamp between '+tstart+' and '+tend+' order by Timestamp'
projdict, msg = dbutil.do_query(cursor, query)
if msg != 'Success':
return {'msg':msg}
if projdict == {}:
return {'msg':'No PHASECAL scans for this day'}
tsint = projdict['Timestamp'].astype(int)
# Check UFDB file to get duration
ufdb = dump_tsys.rd_ufdb(Time(int(tstart),format='lv'))
mjd0 = int(Time(int(tstart),format='lv').mjd)
mjdnow = int(Time.now().mjd)
if mjd0 < mjdnow:
# The date is a previous day, so read a second ufdb file
# to ensure we have the whole local day
try:
ufdb2 = dump_tsys.rd_ufdb(Time(int(tstart)+86400.,format='lv'))
for key in ufdb.keys():
ufdb.update({key: np.append(ufdb[key], ufdb2[key])})
except:
# No previous day, so just skip it.
pass
ufdb_times = ufdb['ST_TS'].astype(float).astype(int)
idx = nearest_val_idx(tsint,ufdb_times)
fpath = '/data1/eovsa/fits/UDB/' + trange[0].iso[:4] + '/'
dur = []
file = []
for i in idx:
dur.append(((ufdb['EN_TS'].astype(float) - ufdb['ST_TS'].astype(float))[i])/60.)
file.append(fpath+ufdb['FILE'][i])
# Fix source ID to remove nulls
srclist = np.array([str(i.replace('\x00','')) for i in projdict['SourceID']])
return {'Timestamp': tsint, 'SourceID': srclist, 'duration': np.array(dur), 'filelist':np.array(file), 'msg': msg}
def find_solpnt(t=None):
''' Makes an SQL query to find the SOLPNTCAL scans for the date given in
the Time() object t. A list of timestamps is returned, along with
the timestamp of the object provided.
'''
import dbutil
if t is None:
# Get today's date
t = util.Time.now()
timestamp = int(t.lv)
stimestamp = timestamp - (timestamp % 86400) # Start of day
etimestamp = stimestamp + 86399 # End of day
# Open handle to SQL database
cursor = dbutil.get_cursor()
# Try to find a scan header with project SOLPNTCAL (only works after 2014 Nov. 30)
verstr = dbutil.find_table_version(cursor,timestamp,True)
if verstr is None:
print 'No scan_header table found for given time.'
return [], timestamp
# First retrieve the Project from all scan headers for the day
cursor.execute('select timestamp,Project from hV'+verstr+'_vD1 where timestamp between '+str(stimestamp)+' and '+str(etimestamp)+' order by timestamp')
data = np.transpose(np.array(cursor.fetchall()))
names = stateframedef.numpy.array(cursor.description)[:,0]
cursor.close()
if len(data) == 0:
# No SOLPNTCAL found, so return empty list (and timestamp)
return [], timestamp
else:
projdict = dict(zip(names,data))
projdict['timestamp'] = projdict['timestamp'].astype('float') # Convert timestamps from string to float
good = np.where(projdict['Project'] == 'SOLPNTCAL')[0]
if len(good) != 0:
if len(good) == 1:
return [projdict['timestamp'][good]], timestamp
else:
return projdict['timestamp'][good], timestamp
else:
return [], timestamp
def get_fem_level(trange, dt=None):
''' Get FEM attenuation levels for a given timerange. Returns a dictionary
with keys as follows:
times: A Time object containing the array of times, size (nt)
hlev: The FEM attenuation level for HPol, size (nt, 15)
vlev: The FEM attenuation level for VPol, size (nt, 15)
dcmattn: The base DCM attenuations for 34 bands x 15 antennas x 2 Poln, size (34,30)
The order is Ant1 H, Ant1 V, Ant2 H, Ant2 V, etc.
dcmoff: If DPPoffset-on is 0, this is None (meaning there are no changes to the
above base attenuations).
If DPPoffset-on is 1, then dcmoff is a table of offsets to the
base attenuation, size (nt, 50). The offset applies to all
antennas/polarizations.
Optional keywords:
dt Seconds between entries to read from SQL stateframe database.
If omitted, 1 s is assumed.
'''
if dt is None:
tstart, tend = [str(i) for i in trange.lv]
else:
# Expand time by 1/2 of dt before and after
tstart = str(np.round(trange[0].lv - dt / 2))
tend = str(np.round(trange[1].lv + dt / 2))
cursor = db.get_cursor()
ver = db.find_table_version(cursor, trange[0].lv)
# Get front end attenuator states
query = 'select Timestamp,Ante_Fron_FEM_Clockms,' \
+'Ante_Fron_FEM_HPol_Regi_Level,Ante_Fron_FEM_VPol_Regi_Level from fV' \
+ver+'_vD15 where Timestamp >= '+tstart+' and Timestamp <= '+tend+' order by Timestamp'
data, msg = db.do_query(cursor, query)
if msg == 'Success':
if dt:
# If we want other than full cadence, get new array shapes and times
n = len(data['Timestamp']) # Original number of times
new_n = (
n / 15 / dt
) * 15 * dt # Truncated number of times equally divisible by dt
new_shape = (n / 15 / dt, dt, 15) # New shape of truncated arrays
times = Time(data['Timestamp'][:new_n].astype('int')[::15 * dt],
format='lv')
else:
times = Time(data['Timestamp'].astype('int')[::15], format='lv')
hlev = data['Ante_Fron_FEM_HPol_Regi_Level']
vlev = data['Ante_Fron_FEM_VPol_Regi_Level']
ms = data['Ante_Fron_FEM_Clockms']
nt = len(hlev) / 15
hlev.shape = (nt, 15)
vlev.shape = (nt, 15)
ms.shape = (nt, 15)
# Find any entries for which Clockms is zero, which indicates where no
# gain-state measurement is available.
for i in range(15):
bad, = np.where(ms[:, i] == 0)
if bad.size != 0 and bad.size != nt:
# Find nearest adjacent good value
good, = np.where(ms[:, i] != 0)
idx = nearest_val_idx(bad, good)
hlev[bad, i] = hlev[good[idx], i]
vlev[bad, i] = vlev[good[idx], i]
if dt:
# If we want other than full cadence, find mean over dt measurements
hlev = np.mean(hlev[:new_n / 15].reshape(new_shape), 1)
vlev = np.mean(vlev[:new_n / 15].reshape(new_shape), 1)
# Put results in canonical order [nant, nt]
hlev = hlev.T
vlev = vlev.T
else:
print 'Error reading FEM levels:', msg
return {}
# Get back end attenuator states
xml, buf = ch.read_cal(2, t=trange[0])
dcmattn = stf.extract(buf, xml['Attenuation'])
dcmattn.shape = (34, 15, 2)
# Put into canonical order [nant, npol, nband]
dcmattn = np.moveaxis(dcmattn, 0, 2)
# See if DPP offset is enabled
query = 'select Timestamp,DPPoffsetattn_on from fV' \
+ver+'_vD1 where Timestamp >= '+tstart+' and Timestamp <= '+tend+'order by Timestamp'
data, msg = db.do_query(cursor, query)
if msg == 'Success':
dppon = data['DPPoffsetattn_on']
if np.where(dppon > 0)[0].size == 0:
dcm_off = None
else:
query = 'select Timestamp,DCMoffset_attn from fV' \
+ver+'_vD50 where Timestamp >= '+tstart+' and Timestamp <= '+tend+' order by Timestamp'
data, msg = db.do_query(cursor, query)
if msg == 'Success':
otimes = Time(data['Timestamp'].astype('int')[::15],
format='lv')
dcmoff = data['DCMoffset_attn']
dcmoff.shape = (nt, 50)
# We now have a time-history of offsets, at least some of which are non-zero.
# Offsets by slot number do us no good, so we need to translate to band number.
# Get fseqfile name at mean of timerange, from stateframe SQL database
fseqfile = get_fseqfile(
Time(int(np.mean(trange.lv)), format='lv'))
if fseqfile is None:
print 'Error: No active fseq file.'
dcm_off = None
else:
# Get fseqfile from ACC and return bandlist
bandlist = fseqfile2bandlist(fseqfile)
# Use bandlist to covert nt x 50 array to nt x 34 band array of DCM attn offsets
# Note that this assumes DCM offset is the same for any multiply-sampled bands
# in the sequence.
dcm_off = np.zeros((nt, 34), float)
dcm_off[:, bandlist - 1] = dcmoff
# Put into canonical order [nband, nt]
dcm_off = dcm_off.T
if dt:
# If we want other than full cadence, find mean over dt measurements
new_nt = len(times)
dcm_off = dcm_off[:, :new_nt * dt]
dcm_off.shape = (34, dt, new_nt)
dcm_off = np.mean(dcm_off, 1)
else:
print 'Error reading DCM attenuations:', msg
dcm_off = None
else:
print 'Error reading DPPon state:', msg
dcm_off = None
cursor.close()
return {
'times': times,
'hlev': hlev.astype(int),
'vlev': vlev.astype(int),
'dcmattn': dcmattn,
'dcmoff': dcm_off
}
def get_solpnt(t=None):
''' Get the SOLPNT data from the SQL database, occurring after
time given in the Time() object t. If omitted, the first
SOLPNT scan for the current day is used (if it exists). '''
import dbutil
tstamps, timestamp = find_solpnt(t)
# Find first SOLPNTCAL occurring after timestamp (time given by Time() object)
if tstamps != []:
print 'SOLPNTCAL scans were found at ',
for tstamp in tstamps:
if type(tstamp) is np.ndarray:
# Annoyingly necessary when only one time in tstamps
tstamp = tstamp[0]
t1 = util.Time(tstamp,format='lv')
print t1.iso,';',
print ' '
good = np.where(tstamps >= timestamp)[0]
# This is the timestamp of the first SOLPNTCAL scan after given time
if good.shape[0] == 0:
stimestamp = tstamps[0]
else:
stimestamp = tstamps[good][0]
else:
print 'Warning: No SOLPNTCAL scan found, so interpreting given time as SOLPNTCAL time.'
stimestamp = timestamp
# Grab 300 records after the start time
cursor = dbutil.get_cursor()
# Now version independent!
verstr = dbutil.find_table_version(cursor,timestamp)
if verstr is None:
print 'No stateframe table found for the given time.'
return {}
solpntdict = dbutil.get_dbrecs(cursor,version=int(verstr),dimension=15,timestamp=stimestamp,nrecs=300)
# Need dimension-1 data to get antennas in subarray -- Note: sometimes the antenna list
# is zero (an unlikely value!) around the time of the start of a scan, so keep searching
# first 100 records until non-zero:
for i in range(100):
blah = dbutil.get_dbrecs(cursor,version=int(verstr),dimension=1,timestamp=stimestamp+i,nrecs=1)
if blah['LODM_Subarray1'][0] != 0:
break
cursor.close()
sub1 = blah['LODM_Subarray1'][0]
subarray1 = []
antlist = []
for i in range(16):
subarray1.append(sub1 & (1<<i) > 0)
if subarray1[-1]:
antlist.append(i)
# print 'Antlist:',antlist
# rao = np.zeros([15,300],dtype='int')
# deco = np.zeros([15,300],dtype='int')
# trk = np.zeros([15,300],dtype='bool')
# hpol = np.zeros([15,300],dtype='float')
# vpol = np.zeros([15,300],dtype='float')
# ra = np.zeros(15,dtype='float')
# dec = np.zeros(15,dtype='float')
ra = (solpntdict['Ante_Cont_RAVirtualAxis'][:,antlist]*np.pi/10000./180.).astype('float')
dec = (solpntdict['Ante_Cont_DecVirtualAxis'][:,antlist]*np.pi/10000./180.).astype('float')
hpol = (solpntdict['Ante_Fron_FEM_HPol_Voltage'][:,antlist]).astype('float')
vpol = (solpntdict['Ante_Fron_FEM_VPol_Voltage'][:,antlist]).astype('float')
rao = (solpntdict['Ante_Cont_RAOffset'][:,antlist]).astype('float')
deco = (solpntdict['Ante_Cont_DecOffset'][:,antlist]).astype('float')
times = solpntdict['Timestamp'][:,0].astype('int64').astype('float')
# Convert pointing info to track information
outdict = stateframe.azel_from_sqldict(solpntdict)
trk = np.logical_and(outdict['dAzimuth'][:,antlist]<0.0020,outdict['dElevation'][:,antlist]<0.0020)
return {'Timestamp':stimestamp,'tstamps':times,'antlist':antlist,'trjfile':'SOLPNT.TRJ','ra':ra,'dec':dec,
'rao':rao,'deco':deco,'trk':trk,'hpol':hpol,'vpol':vpol}
def flare_monitor(t):
''' Get all front-end power-detector voltages for the given day
from the stateframe SQL database, and obtain the median of them,
to use as a flare monitor.
Returns ut times in plot_date format and median voltages.
'''
import dbutil
# timerange is 13 UT to 02 UT on next day, relative to the day in Time() object t
trange = Time([int(t.mjd) + 13./24,int(t.mjd) + 26./24],format='mjd')
tstart, tend = trange.lv.astype('str')
cursor = dbutil.get_cursor()
mjd = t.mjd
try:
verstr = dbutil.find_table_version(cursor,tstart)
if verstr is None:
print 'No stateframe table found for given time.'
return tstart, [], {}
cursor.execute('select * from fV'+verstr+'_vD15 where I15 < 4 and timestamp between '+tstart+' and '+tend+' order by timestamp')
except:
print 'Error with query of SQL database.'
return tstart, [], {}
data = np.transpose(np.array(cursor.fetchall(),'object'))
if len(data) == 0:
# No data found, so return timestamp and empty lists
print 'SQL Query was valid, but no data for',t.iso[:10],'were found (yet).'
return tstart, [], {}
ncol,ntot = data.shape
data.shape = (ncol,ntot/4,4)
names = np.array(cursor.description)[:,0]
mydict = dict(zip(names,data))
hv = []
ut = Time(mydict['Timestamp'][:,0].astype('float'),format='lv').plot_date
hfac = np.median(mydict['Ante_Fron_FEM_HPol_Voltage'].astype('float'),0)
vfac = np.median(mydict['Ante_Fron_FEM_VPol_Voltage'].astype('float'),0)
for i in range(4):
if hfac[i] > 0:
hv.append(mydict['Ante_Fron_FEM_HPol_Voltage'][:,i]/hfac[i])
if vfac[i] > 0:
hv.append(mydict['Ante_Fron_FEM_VPol_Voltage'][:,i]/vfac[i])
flm = np.median(np.array(hv),0)
good = np.where(abs(flm[1:]-flm[:-1])<0.01)[0]
# Get the project IDs for scans during the period
verstrh = dbutil.find_table_version(cursor,trange[0].lv,True)
if verstrh is None:
print 'No scan_header table found for given time.'
return ut[good], flm[good], {}
cursor.execute('select Timestamp,Project from hV'+verstrh+'_vD1 where Timestamp between '+tstart+' and '+tend+' order by Timestamp')
data = np.transpose(np.array(cursor.fetchall()))
names = np.array(cursor.description)[:,0]
if len(data) == 0:
# No Project ID found, so return data and empty projdict dictionary
print 'SQL Query was valid, but no Project data were found.'
return ut[good], flm[good], {}
projdict = dict(zip(names,data))
projdict['Timestamp'] = projdict['Timestamp'].astype('float') # Convert timestamps from string to float
# Get the times when scanstate is -1
cursor.execute('select Timestamp,Sche_Data_ScanState from fV'+verstr+'_vD1 where Timestamp between '+tstart+' and '+tend+' and Sche_Data_ScanState = -1 order by Timestamp')
scan_off_times = np.transpose(np.array(cursor.fetchall()))[0] #Just list of timestamps
if len(scan_off_times) > 2:
gaps = scan_off_times[1:] - scan_off_times[:-1] - 1
eos = np.where(gaps > 10)[0]
if len(eos) > 1:
if scan_off_times[eos[1]] < projdict['Timestamp'][0]:
# Gaps are not lined up, so drop the first:
eos = eos[1:]
EOS = scan_off_times[eos]
if scan_off_times[eos[0]] <= projdict['Timestamp'][0]:
# First EOS is earlier than first Project ID, so make first Project ID None.
projdict['Timestamp'] = np.append([scan_off_times[0]],projdict['Timestamp'])
projdict['Project'] = np.append(['None'],projdict['Project'])
if scan_off_times[eos[-1]+1] >= projdict['Timestamp'][-1]:
# Last EOS is later than last Project ID, so make last Project ID None.
projdict['Timestamp'] = np.append(projdict['Timestamp'],[scan_off_times[eos[-1]+1]])
projdict['Project'] = np.append(projdict['Project'],['None'])
EOS = np.append(EOS,[scan_off_times[eos[-1]+1],scan_off_times[-1]])
projdict.update({'EOS': EOS})
else:
# Not enough scan changes to determine EOS (end-of-scan) times
projdict.update({'EOS': []})
cursor.close()
return ut[good],flm[good],projdict
def flare_monitor(t):
''' Get all front-end power-detector voltages for the given day
from the stateframe SQL database, and obtain the median of them,
to use as a flare monitor.
Returns ut times in plot_date format and median voltages.
'''
import dbutil
# timerange is 13 UT to 02 UT on next day, relative to the day in Time() object t
trange = Time([int(t.mjd) + 13. / 24, int(t.mjd) + 26. / 24], format='mjd')
tstart, tend = trange.lv.astype('str')
cursor = dbutil.get_cursor()
mjd = t.mjd
try:
verstr = dbutil.find_table_version(cursor, tstart)
if verstr is None:
print 'No stateframe table found for given time.'
return tstart, [], {}
cursor.execute('select * from fV' + verstr +
'_vD15 where I15 < 4 and timestamp between ' + tstart +
' and ' + tend + ' order by timestamp')
except:
print 'Error with query of SQL database.'
return tstart, [], {}
data = np.transpose(np.array(cursor.fetchall(), 'object'))
if len(data) == 0:
# No data found, so return timestamp and empty lists
print 'SQL Query was valid, but no data for', t.iso[:
10], 'were found (yet).'
return tstart, [], {}
ncol, ntot = data.shape
data.shape = (ncol, ntot / 4, 4)
names = np.array(cursor.description)[:, 0]
mydict = dict(zip(names, data))
hv = []
ut = Time(mydict['Timestamp'][:, 0].astype('float'), format='lv').plot_date
hfac = np.median(mydict['Ante_Fron_FEM_HPol_Voltage'].astype('float'), 0)
vfac = np.median(mydict['Ante_Fron_FEM_VPol_Voltage'].astype('float'), 0)
for i in range(4):
if hfac[i] > 0:
hv.append(mydict['Ante_Fron_FEM_HPol_Voltage'][:, i] / hfac[i])
if vfac[i] > 0:
hv.append(mydict['Ante_Fron_FEM_VPol_Voltage'][:, i] / vfac[i])
flm = np.median(np.array(hv), 0)
good = np.where(abs(flm[1:] - flm[:-1]) < 0.01)[0]
# Get the project IDs for scans during the period
verstrh = dbutil.find_table_version(cursor, trange[0].lv, True)
if verstrh is None:
print 'No scan_header table found for given time.'
return ut[good], flm[good], {}
cursor.execute('select Timestamp,Project from hV' + verstrh +
'_vD1 where Timestamp between ' + tstart + ' and ' + tend +
' order by Timestamp')
data = np.transpose(np.array(cursor.fetchall()))
names = np.array(cursor.description)[:, 0]
if len(data) == 0:
# No Project ID found, so return data and empty projdict dictionary
print 'SQL Query was valid, but no Project data were found.'
return ut[good], flm[good], {}
projdict = dict(zip(names, data))
projdict['Timestamp'] = projdict['Timestamp'].astype(
'float') # Convert timestamps from string to float
# Get the times when scanstate is -1
cursor.execute('select Timestamp,Sche_Data_ScanState from fV' + verstr +
'_vD1 where Timestamp between ' + tstart + ' and ' + tend +
' and Sche_Data_ScanState = -1 order by Timestamp')
scan_off_times = np.transpose(np.array(
cursor.fetchall()))[0] #Just list of timestamps
if len(scan_off_times) > 2:
gaps = scan_off_times[1:] - scan_off_times[:-1] - 1
eos = np.where(gaps > 10)[0]
if len(eos) > 1:
if scan_off_times[eos[1]] < projdict['Timestamp'][0]:
# Gaps are not lined up, so drop the first:
eos = eos[1:]
EOS = scan_off_times[eos]
if scan_off_times[eos[0]] <= projdict['Timestamp'][0]:
# First EOS is earlier than first Project ID, so make first Project ID None.
projdict['Timestamp'] = np.append([scan_off_times[0]],
projdict['Timestamp'])
projdict['Project'] = np.append(['None'], projdict['Project'])
if scan_off_times[eos[-1] + 1] >= projdict['Timestamp'][-1]:
# Last EOS is later than last Project ID, so make last Project ID None.
projdict['Timestamp'] = np.append(projdict['Timestamp'],
[scan_off_times[eos[-1] + 1]])
projdict['Project'] = np.append(projdict['Project'], ['None'])
EOS = np.append(EOS,
[scan_off_times[eos[-1] + 1], scan_off_times[-1]])
projdict.update({'EOS': EOS})
else:
# Not enough scan changes to determine EOS (end-of-scan) times
projdict.update({'EOS': []})
cursor.close()
return ut[good], flm[good], projdict
def gain_state(trange=None):
''' Read and assemble the gain state for the given timerange from
the SQL database, or for the last 10 minutes if trange is None.
Returns the complex attenuation of the FEM for the timerange
as an array of size (nant, npol, ntimes) [not band dependent],
and the complex attenuation of the DCM for the same timerange
as an array of size (nant, npol, nbands, ntimes). Also returns
the time as a Time() object array.
'''
from util import Time
import dbutil as db
from fem_attn_calib import fem_attn_update
import cal_header as ch
if trange is None:
t = Time.now()
t2 = Time(t.jd - 600. / 86400., format='jd')
trange = Time([t2.iso, t.iso])
ts = trange[0].lv # Start timestamp
te = trange[1].lv # End timestamp
cursor = db.get_cursor()
# First get FEM attenuation for timerange
D15dict = db.get_dbrecs(cursor, dimension=15, timestamp=trange)
DCMoffdict = db.get_dbrecs(cursor, dimension=50, timestamp=trange)
DCMoff_v_slot = DCMoffdict['DCMoffset_attn']
# DCMoff_0 = D15dict['DCM_Offset_Attn'][:,0] # All ants are the same
fem_attn = {}
fem_attn['timestamp'] = D15dict['Timestamp'][:, 0]
nt = len(fem_attn['timestamp'])
junk = np.zeros([nt, 1], dtype='int') #add the non-existing antenna 16
fem_attn['h1'] = np.append(D15dict['Ante_Fron_FEM_HPol_Atte_First'],
junk,
axis=1) #FEM hpol first attn value
fem_attn['h2'] = np.append(D15dict['Ante_Fron_FEM_HPol_Atte_Second'],
junk,
axis=1) #FEM hpol second attn value
fem_attn['v1'] = np.append(D15dict['Ante_Fron_FEM_VPol_Atte_First'],
junk,
axis=1) #FEM vpol first attn value
fem_attn['v2'] = np.append(D15dict['Ante_Fron_FEM_VPol_Atte_Second'],
junk,
axis=1) #FEM vpol second attn value
fem_attn['ants'] = np.append(D15dict['I15'][0, :], [15])
# Add corrections from SQL database for start time of timerange
fem_attn_corr = fem_attn_update(fem_attn, trange[0])
# Next get DCM attenuation for timerange
# Getting next earlier scan header
ver = db.find_table_version(cursor, ts, True)
query = 'select top 50 Timestamp,FSeqList from hV' + ver + '_vD50 where Timestamp <= ' + str(
ts) + ' order by Timestamp desc'
fseq, msg = db.do_query(cursor, query)
if msg == 'Success':
fseqlist = fseq['FSeqList'][::-1] # Reverse the order
bandlist = ((np.array(fseqlist) - 0.44) * 2).astype(int)
cursor.close()
# Read current DCM_table from database
xml, buf = ch.read_cal(3, trange[0])
orig_table = stf.extract(buf, xml['Attenuation']).astype('int')
orig_table.shape = (50, 15, 2)
xml, buf = ch.read_cal(6, trange[0])
dcm_attn_bitv = np.nan_to_num(stf.extract(
buf, xml['DCM_Attn_Real'])) + np.nan_to_num(
stf.extract(buf, xml['DCM_Attn_Imag'])) * 1j
# # Add one more bit (all zeros) to take care of unit bit
# dcm_attn_bitv = np.concatenate((np.zeros((16,2,1),'int'),dcm_attn_bitv),axis=2)
# We now have:
# orig_table the original DCM at start of scan, size (nslot, nant=15, npol)
# DCMoff_0 the offset applied to all antennas and slots (ntimes)
# DCMoff_v_slot the offest applied to all antennas but varies by slot (ntimes, nslot)
# dcm_attn_bitv the measured (non-nominal) attenuations for each bit value (nant=16, npol, nbit) -- complex
# Now I need to convert slot to band, add appropriately, and organize as (nant=16, npol, nband, ntimes)
# Add one more antenna (all zeros) to orig_table
orig_table = np.concatenate((orig_table, np.zeros((50, 1, 2), 'int')),
axis=1)
ntimes, nslot = DCMoff_v_slot.shape
dcm_attn = np.zeros((16, 2, 34, ntimes), np.int)
for i in range(ntimes):
for j in range(50):
idx = bandlist[j] - 1
# This adds attenuation for repeated bands--hopefully the same value for each repeat
dcm_attn[:, :, idx, i] += orig_table[j, :, :] + DCMoff_v_slot[i, j]
# Normalize repeated bands by finding number of repeats and dividing.
for i in range(1, 35):
n = len(np.where(bandlist == i)[0])
if n > 1:
dcm_attn[:, :, i - 1, :] /= n
# Make sure attenuation is in range
dcm_attn = np.clip(dcm_attn, 0, 30)
# Finally, correct for non-nominal (measured) bit values
# Start with 0 attenuation as reference
dcm_attn_corr = dcm_attn * (0 + 0j)
att = np.zeros((16, 2, 34, ntimes, 5), np.complex)
# Calculate resulting attenuation based on bit attn values (2,4,8,16)
for i in range(4):
# Need dcm_attn_bitv[...,i] to be same shape as dcm_attn
bigger_bitv = np.broadcast_to(dcm_attn_bitv[..., i],
(ntimes, 34, 16, 2))
bigger_bitv = np.swapaxes(
np.swapaxes(np.swapaxes(bigger_bitv, 0, 3), 1, 2), 0, 1)
att[..., i] = (np.bitwise_and(dcm_attn, 2**(i + 1)) >>
(i + 1)) * bigger_bitv
dcm_attn_corr = dcm_attn_corr + att[..., i]
# Move ntimes column to next to last position, and then sum over last column (the two attenuators)
fem_attn_corr = np.sum(np.rollaxis(fem_attn_corr, 0, 3), 3)
# Output is FEM shape (nant, npol, ntimes) = (16, 2, ntimes)
# DCM shape (nant, npol, nband, ntimes) = (16, 2, 34, ntimes)
# Arrays are complex, in dB units
tjd = Time(fem_attn['timestamp'].astype('int'), format='lv').jd
return fem_attn_corr, dcm_attn_corr, tjd
def get_gain_state(trange, dt=None, relax=False):
''' Get all gain-state information for a given timerange. Returns a dictionary
with keys as follows:
times: A Time object containing the array of times, size (nt)
h1: The first HPol attenuator value for 15 antennas, size (nt, 15)
v1: The first VPol attenuator value for 15 antennas, size (nt, 15)
h2: The second HPol attenuator value for 15 antennas, size (nt, 15)
v2: The second VPol attenuator value for 15 antennas, size (nt, 15)
dcmattn: The base DCM attenuations for nbands x 15 antennas x 2 Poln, size (34 or 52,30)
The order is Ant1 H, Ant1 V, Ant2 H, Ant2 V, etc.
dcmoff: If DPPoffset-on is 0, this is None (meaning there are no changes to the
above base attenuations).
If DPPoffset-on is 1, then dcmoff is a table of offsets to the
base attenuation, size (nt, 50). The offset applies to all
antennas/polarizations.
Optional keywords:
dt Seconds between entries to read from SQL stateframe database.
If omitted, 1 s is assumed.
relax Used for gain of reference time, in case there are no SQL data for the
requested time. In that case it finds the data for the nearest later time.
'''
if dt is None:
tstart, tend = [str(i) for i in trange.lv]
else:
# Expand time by 1/2 of dt before and after
tstart = str(np.round(trange[0].lv - dt / 2))
tend = str(np.round(trange[1].lv + dt / 2))
cursor = db.get_cursor()
ver = db.find_table_version(cursor, trange[0].lv)
# Get front end attenuator states
# Attempt to solve the problem if there are no data
if relax:
# Special case of reference gain, where we want the first nt records after tstart, in case there
# are no data at time tstart
nt = int(float(tend) - float(tstart) - 1) * 15
query = 'select top '+str(nt)+' Timestamp,Ante_Fron_FEM_HPol_Atte_First,Ante_Fron_FEM_HPol_Atte_Second,' \
+'Ante_Fron_FEM_VPol_Atte_First,Ante_Fron_FEM_VPol_Atte_Second,Ante_Fron_FEM_Clockms from fV' \
+ver+'_vD15 where Timestamp >= '+tstart+' order by Timestamp'
else:
query = 'select Timestamp,Ante_Fron_FEM_HPol_Atte_First,Ante_Fron_FEM_HPol_Atte_Second,' \
+'Ante_Fron_FEM_VPol_Atte_First,Ante_Fron_FEM_VPol_Atte_Second,Ante_Fron_FEM_Clockms from fV' \
+ver+'_vD15 where Timestamp >= '+tstart+' and Timestamp < '+tend+' order by Timestamp'
#if dt:
# # If dt (seconds between measurements) is set, add appropriate SQL statement to query
# query += ' and (cast(Timestamp as bigint) % '+str(dt)+') = 0 '
data, msg = db.do_query(cursor, query)
if msg == 'Success':
if dt:
# If we want other than full cadence, get new array shapes and times
n = len(data['Timestamp']) # Original number of times
new_n = (
n / 15 / dt
) * 15 * dt # Truncated number of times equally divisible by dt
new_shape = (n / 15 / dt, dt, 15) # New shape of truncated arrays
times = Time(data['Timestamp'][:new_n].astype('int')[::15 * dt],
format='lv')
else:
times = Time(data['Timestamp'].astype('int')[::15], format='lv')
# Change tstart and tend to correspond to actual times from SQL
tstart, tend = [str(i) for i in times[[0, -1]].lv]
h1 = data['Ante_Fron_FEM_HPol_Atte_First']
h2 = data['Ante_Fron_FEM_HPol_Atte_Second']
v1 = data['Ante_Fron_FEM_VPol_Atte_First']
v2 = data['Ante_Fron_FEM_VPol_Atte_Second']
ms = data['Ante_Fron_FEM_Clockms']
nt = len(h1) / 15
h1.shape = (nt, 15)
h2.shape = (nt, 15)
v1.shape = (nt, 15)
v2.shape = (nt, 15)
ms.shape = (nt, 15)
# Find any entries for which Clockms is zero, which indicates where no
# gain-state measurement is available.
for i in range(15):
bad, = np.where(ms[:, i] == 0)
if bad.size != 0 and bad.size != nt:
# Find nearest adjacent good value
good, = np.where(ms[:, i] != 0)
idx = nearest_val_idx(bad, good)
h1[bad, i] = h1[good[idx], i]
h2[bad, i] = h2[good[idx], i]
v1[bad, i] = v1[good[idx], i]
v2[bad, i] = v2[good[idx], i]
if dt:
# If we want other than full cadence, find mean over dt measurements
h1 = np.mean(h1[:new_n / 15].reshape(new_shape), 1)
h2 = np.mean(h2[:new_n / 15].reshape(new_shape), 1)
v1 = np.mean(v1[:new_n / 15].reshape(new_shape), 1)
v2 = np.mean(v2[:new_n / 15].reshape(new_shape), 1)
# Put results in canonical order [nant, nt]
h1 = h1.T
h2 = h2.T
v1 = v1.T
v2 = v2.T
else:
print 'Error reading FEM attenuations:', msg
return {}
# Get back end attenuator states
xml, buf = ch.read_cal(2, t=trange[0])
dcmattn = stf.extract(buf, xml['Attenuation'])
nbands = dcmattn.shape[0]
dcmattn.shape = (nbands, 15, 2)
# Put into canonical order [nant, npol, nband]
dcmattn = np.moveaxis(dcmattn, 0, 2)
# See if DPP offset is enabled
query = 'select Timestamp,DPPoffsetattn_on from fV' \
+ver+'_vD1 where Timestamp >= '+tstart+' and Timestamp <= '+tend+'order by Timestamp'
data, msg = db.do_query(cursor, query)
if msg == 'Success':
dppon = data['DPPoffsetattn_on']
if np.where(dppon > 0)[0].size == 0:
dcm_off = None
else:
query = 'select Timestamp,DCMoffset_attn from fV' \
+ver+'_vD50 where Timestamp >= '+tstart+' and Timestamp <= '+tend
#if dt:
# # If dt (seconds between measurements) is set, add appropriate SQL statement to query
# query += ' and (cast(Timestamp as bigint) % '+str(dt)+') = 0 '
query += ' order by Timestamp'
data, msg = db.do_query(cursor, query)
if msg == 'Success':
otimes = Time(data['Timestamp'].astype('int')[::15],
format='lv')
dcmoff = data['DCMoffset_attn']
dcmoff.shape = (nt, 50)
# We now have a time-history of offsets, at least some of which are non-zero.
# Offsets by slot number do us no good, so we need to translate to band number.
# Get fseqfile name at mean of timerange, from stateframe SQL database
fseqfile = get_fseqfile(
Time(int(np.mean(trange.lv)), format='lv'))
if fseqfile is None:
print 'Error: No active fseq file.'
dcm_off = None
else:
# Get fseqfile from ACC and return bandlist
bandlist = fseqfile2bandlist(fseqfile)
nbands = len(bandlist)
# Use bandlist to covert nt x 50 array to nt x nbands array of DCM attn offsets
# Note that this assumes DCM offset is the same for any multiply-sampled bands
# in the sequence.
dcm_off = np.zeros((nt, nbands), float)
dcm_off[:, bandlist - 1] = dcmoff
# Put into canonical order [nband, nt]
dcm_off = dcm_off.T
if dt:
# If we want other than full cadence, find mean over dt measurements
new_nt = len(times)
dcm_off = dcm_off[:, :new_nt * dt]
dcm_off.shape = (nbands, dt, new_nt)
dcm_off = np.mean(dcm_off, 1)
else:
print 'Error reading DCM attenuations:', msg
dcm_off = None
else:
print 'Error reading DPPon state:', msg
dcm_off = None
cursor.close()
return {
'times': times,
'h1': h1,
'v1': v1,
'h2': h2,
'v2': v2,
'dcmattn': dcmattn,
'dcmoff': dcm_off
}
def flare_monitor(t):
''' Get all front-end power-detector voltages for the given day
from the stateframe SQL database, and obtain the median of them,
to use as a flare monitor.
Returns ut times in plot_date format and median voltages.
'''
import dbutil
# timerange is 12 UT to 12 UT on next day, relative to the day in Time() object t
trange = Time([int(t.mjd) + 12. / 24, int(t.mjd) + 36. / 24], format='mjd')
tstart, tend = trange.lv.astype('str')
cursor = dbutil.get_cursor()
mjd = t.mjd
verstr = dbutil.find_table_version(cursor, tstart)
if verstr is None:
print 'No stateframe table found for given time.'
return tstart, [], {}
query = 'select Timestamp,Ante_Fron_FEM_HPol_Voltage,Ante_Fron_FEM_VPol_Voltage from fV' + verstr + '_vD15 where timestamp between ' + tstart + ' and ' + tend + ' order by timestamp'
data, msg = dbutil.do_query(cursor, query)
if msg != 'Success':
print msg
return tstart, [], {}
for k, v in data.items():
data[k].shape = (len(data[k]) / 15, 15)
hv = []
try:
ut = Time(data['Timestamp'][:, 0].astype('float'),
format='lv').plot_date
except:
print 'Error for time', t.iso
print 'Query:', query, ' returned msg:', msg
print 'Keys:', data.keys()
print data['Timestamp'][0, 0]
hfac = np.median(data['Ante_Fron_FEM_HPol_Voltage'].astype('float'), 0)
vfac = np.median(data['Ante_Fron_FEM_VPol_Voltage'].astype('float'), 0)
for i in range(4):
if hfac[i] > 0:
hv.append(data['Ante_Fron_FEM_HPol_Voltage'][:, i] / hfac[i])
if vfac[i] > 0:
hv.append(data['Ante_Fron_FEM_VPol_Voltage'][:, i] / vfac[i])
flm = np.median(np.array(hv), 0)
good = np.where(abs(flm[1:] - flm[:-1]) < 0.01)[0]
# Get the project IDs for scans during the period
verstrh = dbutil.find_table_version(cursor, trange[0].lv, True)
if verstrh is None:
print 'No scan_header table found for given time.'
return ut[good], flm[good], {}
query = 'select Timestamp,Project from hV' + verstrh + '_vD1 where Timestamp between ' + tstart + ' and ' + tend + ' order by Timestamp'
projdict, msg = dbutil.do_query(cursor, query)
if msg != 'Success':
print msg
return ut[good], flm[good], {}
elif len(projdict) == 0:
# No Project ID found, so return data and empty projdict dictionary
print 'SQL Query was valid, but no Project data were found.'
return ut[good], flm[good], {}
projdict['Timestamp'] = projdict['Timestamp'].astype(
'float') # Convert timestamps from string to float
for i in range(len(projdict['Project'])):
projdict['Project'][i] = projdict['Project'][i].replace('\x00', '')
# # Get the times when scanstate is -1
# cursor.execute('select Timestamp,Sche_Data_ScanState from fV'+verstr+'_vD1 where Timestamp between '+tstart+' and '+tend+' and Sche_Data_ScanState = -1 order by Timestamp')
# scan_off_times = np.transpose(np.array(cursor.fetchall()))[0] #Just list of timestamps
# if len(scan_off_times) > 2:
# gaps = scan_off_times[1:] - scan_off_times[:-1] - 1
# eos = np.where(gaps > 10)[0]
# if len(eos) > 1:
# if scan_off_times[eos[1]] < projdict['Timestamp'][0]:
# # Gaps are not lined up, so drop the first:
# eos = eos[1:]
# EOS = scan_off_times[eos]
# if scan_off_times[eos[0]] <= projdict['Timestamp'][0]:
# # First EOS is earlier than first Project ID, so make first Project ID None.
# projdict['Timestamp'] = np.append([scan_off_times[0]],projdict['Timestamp'])
# projdict['Project'] = np.append(['None'],projdict['Project'])
# if scan_off_times[eos[-1]+1] >= projdict['Timestamp'][-1]:
# # Last EOS is later than last Project ID, so make last Project ID None.
# projdict['Timestamp'] = np.append(projdict['Timestamp'],[scan_off_times[eos[-1]+1]])
# projdict['Project'] = np.append(projdict['Project'],['None'])
# EOS = np.append(EOS,[scan_off_times[eos[-1]+1],scan_off_times[-1]])
# projdict.update({'EOS': EOS})
# else:
# # Not enough scan changes to determine EOS (end-of-scan) times
# projdict.update({'EOS': []})
# This turns out to be a more rational, and "good-enough"" approach to the end of scan problem.
# The last scan, though, will be ignored...
projdict.update({'EOS': projdict['Timestamp'][1:]})
projdict.update({'Timestamp': projdict['Timestamp'][:-1]})
projdict.update({'Project': projdict['Project'][:-1]})
cursor.close()
return ut[good], flm[good], projdict
def tp_bgnd(tpdata):
''' Create time-variable background from ROACH inlet temperature
This version is far superior to the earlier, crude version, but
beware that it works best for a long timerange of data, especially
when there is a flare in the data.
Inputs:
tpdata dictionary returned by read_idb() NB: tpdata is not changed.
Returns:
bgnd The background fluctuation array of size (nf,nt) to be
subtracted from any antenna's total power (or mean of
antenna total powers)
'''
import dbutil as db
from util import Time, nearest_val_idx
outfghz = tpdata['fghz']
try:
outtime = tpdata['time']
trange = Time(outtime[[0, -1]], format='jd')
except:
outtime = tpdata['ut_mjd']
trange = Time(outtime[[0, -1]], format='mjd')
tstr = trange.lv.astype(int).astype(str)
nt = len(outtime)
if nt < 1200:
print 'TP_BGND: Error, timebase too small. Must have at least 1200 time samples.'
return None
nf = len(outfghz)
outpd = Time(outtime, format='jd').plot_date
cursor = db.get_cursor()
version = db.find_table_version(cursor, int(tstr[0]))
query = 'select * from fV' + version + '_vD8 where (Timestamp between ' + tstr[
0] + ' and ' + tstr[1] + ')'
data, msg = db.do_query(cursor, query)
pd = Time(data['Timestamp'][::8].astype(int), format='lv').plot_date
inlet = data['Sche_Data_Roac_TempInlet'].reshape(
len(pd), 8) # Inlet temperature variation
sinlet = np.sum(inlet.astype(float), 1)
# Eliminate 0 values in sinlet by replacing with nearest good value
bad, = np.where(sinlet == 0)
good, = np.where(sinlet != 0)
idx = nearest_val_idx(
bad, good) # Find locations of nearest good values to bad ones
sinlet[bad] = sinlet[good[idx]] # Overwrite bad values with good ones
sinlet -= np.mean(
sinlet) # Remove offset, to provide zero-mean fluctuation
sinlet = np.roll(
sinlet,
-110) # Shift phase of variation by 110 s earlier (seems to be needed)
# Interpolate sinlet values to the times in the data
sint = np.interp(outpd, pd, sinlet)
sdev = np.std(sint)
sint_ok = np.abs(sint) < 2 * sdev
bgnd = np.zeros((nf, nt), float)
for i in range(nf):
wlen = min(nt, 2000)
if wlen % 2 != 0:
wlen -= 1
# Subtract smooth trend from data
sig = tpdata['p'][i] - smooth(tpdata['p'][i], wlen,
'blackman')[wlen / 2:-(wlen / 2 - 1)]
# Eliminate the worst outliers and repeat
stdev = np.nanstd(sig)
good, = np.where(np.abs(sig) < stdev)
if len(good) > nt * 0.1:
wlen = min(len(good), 2000)
if wlen % 2 != 0:
wlen -= 1
# Subtract smooth trend from data
sig = tpdata['p'][i, good] - smooth(
tpdata['p'][i,
good], wlen, 'blackman')[wlen / 2:-(wlen / 2 - 1)]
sint_i = sint[good]
stdev = np.std(sig)
# Final check for data quality
good, = np.where(np.logical_and(sig < 2 * stdev, sint_ok[good]))
if len(good) > nt * 0.1:
p = np.polyfit(sint_i[good], sig[good], 1)
else:
p = [1., 0.]
# Apply correction for this frequency
bgnd[i] = sint * p[0] + p[1]
return bgnd
