def get_sql_info(trange):
''' Get all antenna information from the SQL database for a given
timerange, including TrackFlag and Parallactic Angle
Also determines if the RFSwitch state (i.e. which 27-m receiver
is being used).
'''
cursor = db.get_cursor()
sqldict = db.get_dbrecs(cursor, dimension=15, timestamp=trange)
azeldict = stateframe.azel_from_sqldict(sqldict)
time = Time(sqldict['Timestamp'][:, 0].astype(int), format='lv')
azeldict.update({'Time': time})
sqldict = db.get_dbrecs(cursor, dimension=1, timestamp=trange)
azeldict.update({'RFSwitch': sqldict['FEMA_Powe_RFSwitchStatus']})
azeldict.update({'LF_Rcvr': sqldict['FEMA_Rece_LoFreqEnabled']})
if np.median(azeldict['RFSwitch']) == 0.0 and np.median(
azeldict['LF_Rcvr']) == 1.0:
azeldict.update({'Receiver': 'Low'})
elif np.median(azeldict['RFSwitch']) == 1.0 and np.median(
azeldict['LF_Rcvr']) == 0.0:
azeldict.update({'Receiver': 'High'})
else:
azeldict.update({'Receiver': 'Unknown'})
cursor.close()
return azeldict
def get_sql_info(trange):
''' Get all antenna information from the SQL database for a given
timerange, including TrackFlag and Parallactic Angle
'''
cursor = db.get_cursor()
sqldict = db.get_dbrecs(cursor,dimension=15,timestamp=trange)
azeldict = stateframe.azel_from_sqldict(sqldict)
time = Time(sqldict['Timestamp'][:,0].astype(int),format='lv')
azeldict.update({'Time':time})
cursor.close()
return azeldict
def get_sql_info(trange):
''' Get all antenna information from the SQL database for a given
timerange, including TrackFlag and Parallactic Angle
'''
cursor = db.get_cursor()
sqldict = db.get_dbrecs(cursor, dimension=15, timestamp=trange)
azeldict = stateframe.azel_from_sqldict(sqldict)
time = Time(sqldict['Timestamp'][:, 0].astype(int), format='lv')
azeldict.update({'Time': time})
cursor.close()
return azeldict
def get_solpnt(t=None):
''' Get the SOLPNT data from the SQL database, occurring after
time given in the Time() object t. If omitted, the first
SOLPNT scan for the current day is used (if it exists). '''
import dbutil
tstamps, timestamp = find_solpnt(t)
# Find first SOLPNTCAL occurring after timestamp (time given by Time() object)
if tstamps != []:
print 'SOLPNTCAL scans were found at ',
for tstamp in tstamps:
if type(tstamp) is np.ndarray:
# Annoyingly necessary when only one time in tstamps
tstamp = tstamp[0]
t1 = util.Time(tstamp,format='lv')
print t1.iso,';',
print ' '
good = np.where(tstamps >= timestamp)[0]
# This is the timestamp of the first SOLPNTCAL scan after given time
if good.shape[0] == 0:
stimestamp = tstamps[0]
else:
stimestamp = tstamps[good][0]
else:
print 'Warning: No SOLPNTCAL scan found, so interpreting given time as SOLPNTCAL time.'
stimestamp = timestamp
# Grab 300 records after the start time
cursor = dbutil.get_cursor()
# Now version independent!
verstr = dbutil.find_table_version(cursor,timestamp)
if verstr is None:
print 'No stateframe table found for the given time.'
return {}
solpntdict = dbutil.get_dbrecs(cursor,version=int(verstr),dimension=15,timestamp=stimestamp,nrecs=300)
# Need dimension-1 data to get antennas in subarray -- Note: sometimes the antenna list
# is zero (an unlikely value!) around the time of the start of a scan, so keep searching
# first 100 records until non-zero:
for i in range(100):
blah = dbutil.get_dbrecs(cursor,version=int(verstr),dimension=1,timestamp=stimestamp+i,nrecs=1)
if blah['LODM_Subarray1'][0] != 0:
break
cursor.close()
sub1 = blah['LODM_Subarray1'][0]
subarray1 = []
antlist = []
for i in range(16):
subarray1.append(sub1 & (1<<i) > 0)
if subarray1[-1]:
antlist.append(i)
# print 'Antlist:',antlist
# rao = np.zeros([15,300],dtype='int')
# deco = np.zeros([15,300],dtype='int')
# trk = np.zeros([15,300],dtype='bool')
# hpol = np.zeros([15,300],dtype='float')
# vpol = np.zeros([15,300],dtype='float')
# ra = np.zeros(15,dtype='float')
# dec = np.zeros(15,dtype='float')
ra = (solpntdict['Ante_Cont_RAVirtualAxis'][:,antlist]*np.pi/10000./180.).astype('float')
dec = (solpntdict['Ante_Cont_DecVirtualAxis'][:,antlist]*np.pi/10000./180.).astype('float')
hpol = (solpntdict['Ante_Fron_FEM_HPol_Voltage'][:,antlist]).astype('float')
vpol = (solpntdict['Ante_Fron_FEM_VPol_Voltage'][:,antlist]).astype('float')
rao = (solpntdict['Ante_Cont_RAOffset'][:,antlist]).astype('float')
deco = (solpntdict['Ante_Cont_DecOffset'][:,antlist]).astype('float')
times = solpntdict['Timestamp'][:,0].astype('int64').astype('float')
# Convert pointing info to track information
outdict = stateframe.azel_from_sqldict(solpntdict)
trk = np.logical_and(outdict['dAzimuth'][:,antlist]<0.0020,outdict['dElevation'][:,antlist]<0.0020)
return {'Timestamp':stimestamp,'tstamps':times,'antlist':antlist,'trjfile':'SOLPNT.TRJ','ra':ra,'dec':dec,
'rao':rao,'deco':deco,'trk':trk,'hpol':hpol,'vpol':vpol}
def rd_calpnt(filename):
''' Read and return contents of output of CALPNT or CALPNT2M observation
Note that the "x" offsets are dRA and dAZ. To apply these in subsequent
routines, dRA is converted to dHA by inverting the sign, while dAZ is
converted to dXEL (angle on the sky) by multiplying by cos(EL).
'''
import dbutil
f = open(filename, 'r')
lines = f.readlines()
f.close()
ants = lines[0].split('Ant ')[1:]
antlist = []
try:
for ant in ants:
antlist.append(int(ant[:2]))
except:
print 'Unrecognized format (header line) for', filename
return None
nants = len(ants)
lines = lines[1:]
nlines = len(lines)
dra = np.zeros((nants, nlines), np.float)
ddec = np.zeros((nants, nlines), np.float)
ha = []
dec = []
source = []
timstr = []
for i, line in enumerate(lines):
if len(line) < 9:
dra = dra[:, :i]
ddec = ddec[:, :i]
break
vals = line[9:].split()
if len(vals) != nants * 2 + 4:
print 'Error reading line', i + 2, 'of', filename
dra = dra[:, :i]
ddec = ddec[:, :i]
break
else:
try:
source.append(line[:8])
timstr.append(vals[0] + ' ' + vals[1])
ha.append(vals[2])
dec.append(vals[3])
dra[:, i] = np.array(vals[4::2]).astype(float)
ddec[:, i] = np.array(vals[5::2]).astype(float)
except:
print 'Error parsing line', i + 2, 'of', filename
# Convert HA, Dec from degrees to radians, and then convert HA to RA
ha = np.array(ha).astype(float) * np.pi / 180.
dec = np.array(dec).astype(float) * np.pi / 180.
times = Time(timstr)
ra = np.zeros_like(ha)
for i in range(len(ha)):
ra[i] = eovsa_lst(times[i]) - ha[i]
# Read pointing parameters from SQL database at time of first observation
params_old = np.zeros((9, 15), int)
cursor = dbutil.get_cursor()
timestamp = times[0].lv
# Read stateframe data at time of first observation
D15data = dbutil.get_dbrecs(cursor,
dimension=15,
timestamp=timestamp,
nrecs=1)
for p in range(9):
params_old[p], = D15data['Ante_Cont_PointingCoefficient' + str(p + 1)]
params_old = params_old[:, np.array(antlist) -
1] # Pare down to only antennas in antlist
return {
'filename': filename,
'source': source,
'time': times,
'params_old': params_old,
'ra': ra,
'dec': dec,
'ha': ha,
'antlist': antlist,
'dra': dra,
'ddec': ddec
}
def gain_state(trange=None):
''' Read and assemble the gain state for the given timerange from
the SQL database, or for the last 10 minutes if trange is None.
Returns the complex attenuation of the FEM for the timerange
as an array of size (nant, npol, ntimes) [not band dependent],
and the complex attenuation of the DCM for the same timerange
as an array of size (nant, npol, nbands, ntimes). Also returns
the time as a Time() object array.
'''
from util import Time
import dbutil as db
from fem_attn_calib import fem_attn_update
import cal_header as ch
if trange is None:
t = Time.now()
t2 = Time(t.jd - 600. / 86400., format='jd')
trange = Time([t2.iso, t.iso])
ts = trange[0].lv # Start timestamp
te = trange[1].lv # End timestamp
cursor = db.get_cursor()
# First get FEM attenuation for timerange
D15dict = db.get_dbrecs(cursor, dimension=15, timestamp=trange)
DCMoffdict = db.get_dbrecs(cursor, dimension=50, timestamp=trange)
DCMoff_v_slot = DCMoffdict['DCMoffset_attn']
# DCMoff_0 = D15dict['DCM_Offset_Attn'][:,0] # All ants are the same
fem_attn = {}
fem_attn['timestamp'] = D15dict['Timestamp'][:, 0]
nt = len(fem_attn['timestamp'])
junk = np.zeros([nt, 1], dtype='int') #add the non-existing antenna 16
fem_attn['h1'] = np.append(D15dict['Ante_Fron_FEM_HPol_Atte_First'],
junk,
axis=1) #FEM hpol first attn value
fem_attn['h2'] = np.append(D15dict['Ante_Fron_FEM_HPol_Atte_Second'],
junk,
axis=1) #FEM hpol second attn value
fem_attn['v1'] = np.append(D15dict['Ante_Fron_FEM_VPol_Atte_First'],
junk,
axis=1) #FEM vpol first attn value
fem_attn['v2'] = np.append(D15dict['Ante_Fron_FEM_VPol_Atte_Second'],
junk,
axis=1) #FEM vpol second attn value
fem_attn['ants'] = np.append(D15dict['I15'][0, :], [15])
# Add corrections from SQL database for start time of timerange
fem_attn_corr = fem_attn_update(fem_attn, trange[0])
# Next get DCM attenuation for timerange
# Getting next earlier scan header
ver = db.find_table_version(cursor, ts, True)
query = 'select top 50 Timestamp,FSeqList from hV' + ver + '_vD50 where Timestamp <= ' + str(
ts) + ' order by Timestamp desc'
fseq, msg = db.do_query(cursor, query)
if msg == 'Success':
fseqlist = fseq['FSeqList'][::-1] # Reverse the order
bandlist = ((np.array(fseqlist) - 0.44) * 2).astype(int)
cursor.close()
# Read current DCM_table from database
xml, buf = ch.read_cal(3, trange[0])
orig_table = stf.extract(buf, xml['Attenuation']).astype('int')
orig_table.shape = (50, 15, 2)
xml, buf = ch.read_cal(6, trange[0])
dcm_attn_bitv = np.nan_to_num(stf.extract(
buf, xml['DCM_Attn_Real'])) + np.nan_to_num(
stf.extract(buf, xml['DCM_Attn_Imag'])) * 1j
# # Add one more bit (all zeros) to take care of unit bit
# dcm_attn_bitv = np.concatenate((np.zeros((16,2,1),'int'),dcm_attn_bitv),axis=2)
# We now have:
# orig_table the original DCM at start of scan, size (nslot, nant=15, npol)
# DCMoff_0 the offset applied to all antennas and slots (ntimes)
# DCMoff_v_slot the offest applied to all antennas but varies by slot (ntimes, nslot)
# dcm_attn_bitv the measured (non-nominal) attenuations for each bit value (nant=16, npol, nbit) -- complex
# Now I need to convert slot to band, add appropriately, and organize as (nant=16, npol, nband, ntimes)
# Add one more antenna (all zeros) to orig_table
orig_table = np.concatenate((orig_table, np.zeros((50, 1, 2), 'int')),
axis=1)
ntimes, nslot = DCMoff_v_slot.shape
dcm_attn = np.zeros((16, 2, 34, ntimes), np.int)
for i in range(ntimes):
for j in range(50):
idx = bandlist[j] - 1
# This adds attenuation for repeated bands--hopefully the same value for each repeat
dcm_attn[:, :, idx, i] += orig_table[j, :, :] + DCMoff_v_slot[i, j]
# Normalize repeated bands by finding number of repeats and dividing.
for i in range(1, 35):
n = len(np.where(bandlist == i)[0])
if n > 1:
dcm_attn[:, :, i - 1, :] /= n
# Make sure attenuation is in range
dcm_attn = np.clip(dcm_attn, 0, 30)
# Finally, correct for non-nominal (measured) bit values
# Start with 0 attenuation as reference
dcm_attn_corr = dcm_attn * (0 + 0j)
att = np.zeros((16, 2, 34, ntimes, 5), np.complex)
# Calculate resulting attenuation based on bit attn values (2,4,8,16)
for i in range(4):
# Need dcm_attn_bitv[...,i] to be same shape as dcm_attn
bigger_bitv = np.broadcast_to(dcm_attn_bitv[..., i],
(ntimes, 34, 16, 2))
bigger_bitv = np.swapaxes(
np.swapaxes(np.swapaxes(bigger_bitv, 0, 3), 1, 2), 0, 1)
att[..., i] = (np.bitwise_and(dcm_attn, 2**(i + 1)) >>
(i + 1)) * bigger_bitv
dcm_attn_corr = dcm_attn_corr + att[..., i]
# Move ntimes column to next to last position, and then sum over last column (the two attenuators)
fem_attn_corr = np.sum(np.rollaxis(fem_attn_corr, 0, 3), 3)
# Output is FEM shape (nant, npol, ntimes) = (16, 2, ntimes)
# DCM shape (nant, npol, nband, ntimes) = (16, 2, 34, ntimes)
# Arrays are complex, in dB units
tjd = Time(fem_attn['timestamp'].astype('int'), format='lv').jd
return fem_attn_corr, dcm_attn_corr, tjd
def get_attncal(trange, do_plot=False, dataonly=False):
''' Finds GAINCALTEST scans from FDB files corresponding to the days
present in trange Time() object (can be multiple days), calculates
the attenuation differences for the various FEMATTN states 1-8
relative to FEMATTN state 0, and optionally plots the results for
states 1 and 2 (the most commonly used). To analyze only a single
day, trange Time() object can have the same time repeated, or can
be a single time.
Returns a list of dictionaries, each pertaining to one of the days
in trange, with keys defined as follows:
'time': The start time of the GAINCALTEST scan, as a Time() object
'fghz': The list of frequencies [GHz] at which attenuations are measured
'attn': The array of attenuations [dB] of size (nattn, nant, npol, nf),
where nattn = 8, nant = 13, npol = 2, and nf is variable
'rcvr': The array of receiver noise level (raw units) of size
(nant, npol, nf), where nant = 13, npol = 2, and nf is variable
'rcvr_auto': Same as rcvr, but for auto-correlation (hence it is complex)
N.B.: Ignores days with other than one GAINCALTEST measurement, e.g. 0 or 2,
the first is obvious, while the second is because there is no way to
tell which of the 2 are good.
The dataonly parameter tells the routine to skip calculating the attenuation
and only return the IDB data from the (first) gaincal.
'''
from util import get_idbdir, fname2mjd, nearest_val_idx
import socket
import dbutil
if type(trange.mjd) == np.float:
# Interpret single time as both start and end time
mjd1 = int(trange.mjd)
mjd2 = mjd1
else:
mjd1, mjd2 = trange.mjd.astype(int)
if do_plot:
import matplotlib.pylab as plt
f, ax = plt.subplots(4, 13)
f.set_size_inches((14, 5))
ax[0, 0].set_ylabel('Atn1X [dB]')
ax[1, 0].set_ylabel('Atn1Y [dB]')
ax[2, 0].set_ylabel('Atn2X [dB]')
ax[3, 0].set_ylabel('Atn2Y [dB]')
for i in range(13):
ax[0, i].set_title('Ant ' + str(i + 1))
ax[3, i].set_xlabel('Freq [GHz]')
for j in range(2):
ax[j, i].set_ylim(1, 3)
ax[j + 2, i].set_ylim(3, 5)
outdict = []
for mjd in range(mjd1, mjd2 + 1):
fdb = dt.rd_fdb(Time(mjd, format='mjd'))
gcidx, = np.where(fdb['PROJECTID'] == 'GAINCALTEST')
if len(gcidx) == 1:
print fdb['FILE'][gcidx]
gcidx = gcidx[0]
else:
for i, fname in enumerate(fdb['FILE'][gcidx]):
print str(i) + ': GAINCALTEST File', fname
idex = input('There is more than one GAINCALTEST. Select: ' +
str(np.arange(len(gcidx))) + ':')
gcidx = gcidx[idex]
datadir = get_idbdir(Time(mjd, format='mjd'))
# Add date path if on pipeline
# if datadir.find('eovsa') != -1: datadir += fdb['FILE'][gcidx][3:11]+'/'
host = socket.gethostname()
if host == 'pipeline': datadir += fdb['FILE'][gcidx][3:11] + '/'
file = datadir + fdb['FILE'][gcidx]
out = ri.read_idb([file])
if dataonly:
return out
# Get time from filename and read 120 records of attn state from SQL database
filemjd = fname2mjd(fdb['FILE'][gcidx])
cursor = dbutil.get_cursor()
d15 = dbutil.get_dbrecs(cursor,
dimension=15,
timestamp=Time(filemjd, format='mjd'),
nrecs=120)
cursor.close()
# Find time indexes of the 62 dB attn state
# Uses only ant 1 assuming all are the same
dtot = (d15['Ante_Fron_FEM_HPol_Atte_Second'] +
d15['Ante_Fron_FEM_HPol_Atte_First'])[:, 0]
# Use system clock day number to identify bad SQL entries and eliminate them
good, = np.where(d15['Ante_Cont_SystemClockMJDay'][:, 0] != 0)
#import pdb; pdb.set_trace()
# Indexes into SQL records where a transition occurred.
transitions, = np.where(dtot[good] - np.roll(dtot[good], 1) != 0)
# Eliminate any zero-index transition (if it exists)
if transitions[0] == 0:
transitions = transitions[1:]
# These now have to be translated into indexes into the data, using the times
idx = nearest_val_idx(d15['Timestamp'][good, 0][transitions],
Time(out['time'], format='jd').lv)
#import pdb; pdb.set_trace()
vx = np.nanmedian(
out['p'][:13, :, :, np.arange(idx[0] + 1, idx[1] - 1)], 3)
va = np.mean(out['a'][:13, :2, :,
np.arange(idx[0] + 1, idx[1] - 1)], 3)
vals = []
attn = []
for i in range(1, 10):
vals.append(
np.nanmedian(
out['p'][:13, :, :,
np.arange(idx[i] + 1, idx[i + 1] - 1)], 3) - vx)
attn.append(np.log10(vals[0] / vals[-1]) * 10.)
#vals = []
#attna = []
#for i in range(1,10):
# vals.append(np.median(out['a'][:13,:2,:,np.arange(idx[i],idx[i+1])],3) - va)
# attna.append(np.log10(vals[0]/vals[-1])*10.)
if do_plot:
for i in range(13):
for j in range(2):
ax[j, i].plot(out['fghz'],
attn[1][i, j],
'.',
markersize=3)
#ax[j,i].plot(out['fghz'],attna[1][i,j],'.',markersize=1)
ax[j + 2, i].plot(out['fghz'],
attn[2][i, j],
'.',
markersize=3)
#ax[j+2,i].plot(out['fghz'],attna[2][i,j],'.',markersize=1)
outdict.append({
'time': Time(out['time'][0], format='jd'),
'fghz': out['fghz'],
'rcvr_auto': va, # 'attna': np.array(attna[1:]),
'rcvr': vx,
'attn': np.array(attn[1:])
})
return outdict
def offsets2ants(t,xoff,yoff,ant_str=None):
''' Given a start time (Time object) and a list of offsets output by sp_offsets()
for 13 antennas, convert to pointing coefficients (multiply by 10000),
add to coefficients listed in stateframe, and send to the relevant
antennas. The antennas to update are specified with ant_str
(defaults to no antennas, for safety).
'''
def send_cmds(cmds,acc):
''' Sends a series of commands to ACC. The sequence of commands
is not checked for validity!
cmds a list of strings, each of which must be a valid command
'''
import socket
for cmd in cmds:
#print 'Command:',cmd
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
s.connect((acc['host'],acc['scdport']))
s.send(cmd)
time.sleep(0.01)
s.close()
except:
print 'Error: Could not send command',cmd,' to ACC.'
return
oldant = [8,9,10,12]
if ant_str is None:
print 'No antenna list specified, so there is nothing to do!'
return
try:
timestamp = int(Time(t,format='mjd').lv)
except:
print 'Error interpreting time as Time() object'
return
from util import ant_str2list
import dbutil as db
import stateframe as stf
accini = stf.rd_ACCfile()
acc = {'host': accini['host'], 'scdport':accini['scdport']}
antlist = ant_str2list(ant_str)
if antlist is None:
return
cursor = db.get_cursor()
# Read current stateframe data (as of 10 s ago)
D15data = db.get_dbrecs(cursor,dimension=15,timestamp=timestamp,nrecs=1)
p1_cur, = D15data['Ante_Cont_PointingCoefficient1']
p7_cur, = D15data['Ante_Cont_PointingCoefficient7']
for i in antlist:
if i in oldant:
# Change sign of RA offset to be HA, for old antennas (9, 10, 11 or 13)
p1_inc = int(-xoff[i]*10000)
else:
p1_inc = int(xoff[i]*10000)
p7_inc = int(yoff[i]*10000)
p1_new = p1_cur[i] + p1_inc
p7_new = p7_cur[i] + p7_inc
print 'Updating P1 for Ant',i+1,'P1_old =',p1_cur[i],'P1_inc =',p1_inc,'P1_new =',p1_new
cmd1 = 'pointingcoefficient1 '+str(p1_new)+' ant'+str(i+1)
print 'Updating P7 for Ant',i+1,'P7_old =',p7_cur[i],'P7_inc =',p7_inc,'P7_new =',p7_new
cmd7 = 'pointingcoefficient7 '+str(p7_new)+' ant'+str(i+1)
print 'Commands to be sent:'
print cmd1
print cmd7
send_cmds([cmd1],acc)
send_cmds([cmd7],acc)
def tp_bgnd_all(tpdata):
''' Create time-variable background from ROACH inlet temperature
This version is far superior to the earlier, crude version, but
beware that it works best for a long timerange of data, especially
when there is a flare in the data.
Inputs:
tpdata dictionary returned by read_idb() NB: tpdata is not changed.
Returns:
bgnd The background fluctuation array of size (nf,nt) to be
subtracted from any antenna's total power (or mean of
antenna total powers)
'''
import dbutil as db
from util import Time, nearest_val_idx
outfghz = tpdata['fghz']
try:
outtime = tpdata['time']
trange = Time(outtime[[0, -1]], format='jd')
except:
outtime = tpdata['ut_mjd']
trange = Time(outtime[[0, -1]], format='mjd')
nt = len(outtime)
if nt < 1200:
print 'TP_BGND: Error, timebase too small. Must have at least 1200 time samples.'
return None
nf = len(outfghz)
outpd = Time(outtime, format='jd').plot_date
cursor = db.get_cursor()
data = db.get_dbrecs(cursor, dimension=8, timestamp=trange)
pd = Time(data['Timestamp'][:, 0].astype(int), format='lv').plot_date
inlet = data['Sche_Data_Roac_TempInlet'] # Inlet temperature variation
sinlet = np.sum(inlet.astype(float), 1)
# Eliminate 0 values in sinlet by replacing with nearest good value
bad, = np.where(sinlet == 0)
good, = np.where(sinlet != 0)
idx = nearest_val_idx(
bad, good) # Find locations of nearest good values to bad ones
sinlet[bad] = sinlet[good[idx]] # Overwrite bad values with good ones
sinlet -= np.mean(
sinlet) # Remove offset, to provide zero-mean fluctuation
sinlet = np.roll(
sinlet,
-110) # Shift phase of variation by 110 s earlier (seems to be needed)
# Interpolate sinlet values to the times in the data
sint = np.interp(outpd, pd, sinlet)
# sint = np.roll(sint,-90) # Shift phase of variation by 90 s earlier
# sint -= np.mean(sint) # Remove offset, to provide zero-mean fluctuation
sdev = np.std(sint)
sint_ok = np.abs(sint) < 2 * sdev
bgnd = np.zeros((13, 2, nf, nt), float)
for ant in range(13):
for pol in range(2):
for i in range(nf):
# Subtract smooth trend from data
nt = len(tpdata['p'][ant, pol, i])
wlen = min(nt, 2000)
if wlen % 2 != 0:
wlen -= 1
sig = tpdata['p'][ant, pol, i] - smooth(
tpdata['p'][ant, pol, i], wlen,
'blackman')[wlen / 2:-(wlen / 2 - 1)]
# Eliminate the worst outliers and repeat
stdev = np.nanstd(sig)
good, = np.where(np.abs(sig) < 2 * stdev)
if len(good) > nt * 0.1:
wlen = min(len(good), 2000)
if wlen % 2 != 0:
wlen -= 1
sig = tpdata['p'][ant, pol, i, good] - smooth(
tpdata['p'][ant, pol, i, good], wlen,
'blackman')[wlen / 2:-(wlen / 2 - 1)]
sint_i = sint[good]
stdev = np.std(sig)
# Final check for data quality
good, = np.where(
np.logical_and(sig < 2 * stdev, sint_ok[good]))
if len(good) > nt * 0.1:
p = np.polyfit(sint_i[good], sig[good], 1)
else:
p = [1., 0.]
# Apply correction for this frequency
bgnd[ant, pol, i] = sint * p[0] + p[1]
return bgnd
