def unrot_refcal(refcal_in):
''' Apply feed-rotation correction to data read with rd_refcal(), returning updated data in
the same format for further processing.
'''
import dbutil as db
import copy
import chan_util_bc as cu
import cal_header as ch
from stateframe import extract
refcal = copy.deepcopy(refcal_in)
xml, buf = ch.read_cal(11, Time(refcal['times'][0][0], format='jd'))
dph = extract(buf, xml['XYphase'])
xi_rot = extract(buf, xml['Xi_Rot'])
freq = extract(buf, xml['FGHz'])
freq = freq[np.where(freq != 0)]
band = []
for f in freq:
band.append(cu.freq2bdname(f))
bds, sidx = np.unique(band, return_index=True)
nbd = len(bds)
eidx = np.append(sidx[1:], len(band))
dxy = np.zeros((14, 34), dtype=np.float)
xi = np.zeros(34, dtype=np.float)
fghz = np.zeros(34)
# average dph and xi_rot frequencies within each band, to convert to 34-band representation
for b, bd in enumerate(bds):
fghz[bd - 1] = np.nanmean(freq[sidx[b]:eidx[b]])
xi[bd - 1] = np.nanmean(xi_rot[sidx[b]:eidx[b]])
for a in range(14):
dxy[a, bd - 1] = np.angle(np.sum(np.exp(1j * dph[a, sidx[b]:eidx[b]])))
nscans = len(refcal['scanlist'])
for i in range(nscans):
# Read parallactic angles for this scan
trange = Time([refcal['tstlist'][i].iso, refcal['tedlist'][i].iso])
times, chi = db.get_chi(trange)
tchi = times.jd
t = refcal['times'][i]
if len(t) > 0:
vis = copy.deepcopy(refcal['vis'][i])
idx = nearest_val_idx(t, tchi)
pa = chi[idx] # Parallactic angle for the times of this refcal.
pa[:, [8, 9, 10, 12]] = 0.0
nt = len(idx) # Number of times in this refcal
# Apply X-Y delay phase correction
for a in range(13):
a1 = lobe(dxy[a] - dxy[13])
a2 = -dxy[13] - xi
a3 = dxy[a] - xi + np.pi
for j in range(nt):
vis[a, 1, :, j] *= np.exp(1j * a1)
vis[a, 2, :, j] *= np.exp(1j * a2)
vis[a, 3, :, j] *= np.exp(1j * a3)
for j in range(nt):
for a in range(13):
refcal['vis'][i][a, 0, :, j] = vis[a, 0, :, j] * np.cos(pa[j, a]) + vis[a, 3, :, j] * np.sin(pa[j, a])
refcal['vis'][i][a, 2, :, j] = vis[a, 2, :, j] * np.cos(pa[j, a]) + vis[a, 1, :, j] * np.sin(pa[j, a])
refcal['vis'][i][a, 3, :, j] = vis[a, 3, :, j] * np.cos(pa[j, a]) - vis[a, 0, :, j] * np.sin(pa[j, a])
refcal['vis'][i][a, 1, :, j] = vis[a, 1, :, j] * np.cos(pa[j, a]) - vis[a, 2, :, j] * np.sin(pa[j, a])
return refcal
def unrot_refcal(refcal_in):
''' Apply feed-rotation correction to data read with rd_refcal(), returning updated data in
the same format for further processing.
'''
import dbutil as db
import copy
import chan_util_bc as cu
refcal = copy.deepcopy(refcal_in)
blah = np.load('/common/tmp/Feed_rotation/20170702121949_delay_phase.npz')
dph = blah['dph']
band=[]
for f in blah['fghz']:
band.append(cu.freq2bdname(f))
bds, sidx = np.unique(band, return_index=True)
nbd = len(bds)
eidx = np.append(sidx[1:], len(band))
dxy = np.zeros((14, 34), dtype=np.float)
fghz = np.zeros(34)
# average dph frequencies within each band, to convert to 34-band representation
for b, bd in enumerate(bds):
fghz[bd - 1] = np.nanmean(blah['fghz'][sidx[b]:eidx[b]])
for a in range(14):
dxy[a,bd-1] = np.angle(np.sum(np.exp(1j*dph[a, sidx[b]:eidx[b]])))
# Read parallactic angles for entire time range
trange = Time([refcal['tstlist'][0].iso,refcal['tedlist'][-1].iso])
times, chi = db.get_chi(trange)
tchi = times.jd
nscans = len(refcal['scanlist'])
for i in range(nscans):
t = refcal['times'][i]
vis = copy.deepcopy(refcal['vis'][i])
idx = nearest_val_idx(t,tchi)
pa = chi[idx] # Parallactic angle for the times of this refcal.
pa[:,[8,9,10,12]] = 0.0
nt = len(idx) # Number of times in this refcal
# Apply X-Y delay phase correction
for a in range(13):
a1 = lobe(dxy[a] - dxy[13])
a2 = -dxy[13] + np.pi/2
a3 = dxy[a] - np.pi/2
for j in range(nt):
vis[a,1,:,j] *= np.exp(1j*a1)
vis[a,2,:,j] *= np.exp(1j*a2)
vis[a,3,:,j] *= np.exp(1j*a3)
for j in range(nt):
for a in range(13):
refcal['vis'][i][a,0,:,j] = vis[a,0,:,j]*np.cos(pa[j,a]) + vis[a,3,:,j]*np.sin(pa[j,a])
refcal['vis'][i][a,2,:,j] = vis[a,2,:,j]*np.cos(pa[j,a]) + vis[a,1,:,j]*np.sin(pa[j,a])
refcal['vis'][i][a,3,:,j] = vis[a,3,:,j]*np.cos(pa[j,a]) - vis[a,0,:,j]*np.sin(pa[j,a])
refcal['vis'][i][a,1,:,j] = vis[a,1,:,j]*np.cos(pa[j,a]) - vis[a,2,:,j]*np.sin(pa[j,a])
return refcal
def allday_process(path=None):
''' Process an all day list of corrected data files to create total power
and baseline amplitude FITS spectrograms (planned for submission to
NASA SDAC for support of the Parker Solar Probe).
Fixed a problem when nans appear in the data--use nanmean() and nanmedian()
'''
import glob
import read_idb as ri
from xspfits2 import tp_writefits
if path is None:
path = './'
files = glob.glob(path + 'IDB*')
files.sort()
for file in files:
out = ri.read_idb([file])
nant, npol, nf, nt = out['p'].shape
nant = 13
# Use only data from tracking antennas
azeldict = get_sql_info(Time(out['time'], format='jd')[[0, -1]])
idx = nearest_val_idx(out['time'], azeldict['Time'].jd)
tracking = azeldict['TrackFlag'].T
# Flag any data where the antennas are not tracking
for i in range(nant):
out['p'][i, :, :, ~tracking[i, idx]] = np.nan
# Determine best 8 antennas
med = np.nanmean(np.nanmedian(out['p'][:nant], 3), 1) # size nant,nf
medspec = np.nanmedian(med, 0) # size nf
p = np.polyfit(out['fghz'], medspec, 2)
spec = np.polyval(p, out['fghz']).repeat(nant).reshape(
nf, nant) # size nf, nant
stdev = np.std(med - np.transpose(spec), 1) # size nant
idx = stdev.argsort()[:8] # List of 8 best-fitting antennas
# Use list of antennas to get final median total power dynamic spectrum
med = np.nanmean(np.nanmedian(out['p'][idx], 0), 0)
# Write the total power spectrum to a FITS file
tp_writefits(out, med.astype(np.float32), filestem='TP_', outpath='./')
# Form sum of intermediate baselines
baseidx = np.array([
29, 30, 31, 32, 33, 34, 42, 43, 44, 45, 46, 54, 55, 56, 57, 65, 66,
67, 75, 76, 84
])
# Get uv distance for mid-time
#uvdist = np.sqrt(out['uvw'][:,nt/2,0]**2 + out['uvw'][:,nt/2,1]**2 + out['uvw'][:,nt/2,2]**2)
# Sort from low to high uv distance
#bah = uvdist.argsort()
# Use "intermediate" lengths, i.e. 20th to 39th in list, and sum amplitudes
med = np.abs(np.nansum(np.nansum(out['x'][baseidx], 0), 0))
# Write the baseline amplitude spectrum to a FITS file
tp_writefits(out, med.astype(np.float32), filestem='XP_', outpath='./')
def findscans(trange):
'''Identify phasecal scans from UFDB files
'''
import dbutil
import dump_tsys
tstart, tend = trange.lv.astype(int).astype(str)
cursor = dbutil.get_cursor()
verstr = dbutil.find_table_version(cursor, tstart, True)
query = 'select Timestamp,Project,SourceID from hV'+verstr+'_vD1 where left(Project,8) = "PHASECAL" and Timestamp between '+tstart+' and '+tend+' order by Timestamp'
projdict, msg = dbutil.do_query(cursor, query)
if msg != 'Success':
return {'msg':msg}
if projdict == {}:
return {'msg':'No PHASECAL scans for this day'}
tsint = projdict['Timestamp'].astype(int)
# Check UFDB file to get duration
ufdb = dump_tsys.rd_ufdb(Time(int(tstart),format='lv'))
mjd0 = int(Time(int(tstart),format='lv').mjd)
mjdnow = int(Time.now().mjd)
if mjd0 < mjdnow:
# The date is a previous day, so read a second ufdb file
# to ensure we have the whole local day
try:
ufdb2 = dump_tsys.rd_ufdb(Time(int(tstart)+86400.,format='lv'))
for key in ufdb.keys():
ufdb.update({key: np.append(ufdb[key], ufdb2[key])})
except:
# No previous day, so just skip it.
pass
ufdb_times = ufdb['ST_TS'].astype(float).astype(int)
idx = nearest_val_idx(tsint,ufdb_times)
fpath = '/data1/eovsa/fits/UDB/' + trange[0].iso[:4] + '/'
dur = []
file = []
for i in idx:
dur.append(((ufdb['EN_TS'].astype(float) - ufdb['ST_TS'].astype(float))[i])/60.)
file.append(fpath+ufdb['FILE'][i])
# Fix source ID to remove nulls
srclist = np.array([str(i.replace('\x00','')) for i in projdict['SourceID']])
return {'Timestamp': tsint, 'SourceID': srclist, 'duration': np.array(dur), 'filelist':np.array(file), 'msg': msg}
def apply_attn_corr(data, tref=None):
''' Applys the attenuator state corrections to the given data dictionary,
corrected to the gain-state at time given by Time() object tref.
Inputs:
data A dictionary returned by udb_util.py's readXdata().
tref A Time() object with the reference time, or if None,
the gain state of the nearest earlier REFCAL is
used.
Output:
cdata A dictionary with the gain-corrected data. The keys
px, py, and x, are updated.
NB: This is the same routine as in gaincal2.py, but modified
to handle the different ordering/format of data from udb_util.py's
readXdata() routine.
'''
from gaincal2 import get_gain_state
from util import common_val_idx, nearest_val_idx
import copy
if tref is None:
# No reference time specified, so get nearest earlier REFCAL
trange = Time(data['time'][[0, -1]], format='jd')
xml, buf = ch.read_cal(8, t=trange[0])
tref = Time(stateframe.extract(buf, xml['Timestamp']), format='lv')
# Get the gain state at the reference time (actually median over 1 minute)
trefrange = Time([tref.iso, Time(tref.lv + 60, format='lv').iso])
ref_gs = get_gain_state(trefrange) # refcal gain state for 60 s
# Get median of refcal gain state (which should be constant anyway)
ref_gs['h1'] = np.median(ref_gs['h1'], 1)
ref_gs['h2'] = np.median(ref_gs['h2'], 1)
ref_gs['v1'] = np.median(ref_gs['v1'], 1)
ref_gs['v2'] = np.median(ref_gs['v2'], 1)
# Get timerange from data
trange = Time([data['time'][0], data['time'][-1]], format='jd')
# Get time cadence
dt = np.int(
np.round(np.median(data['time'][1:] - data['time'][:-1]) * 86400))
if dt == 1: dt = None
# Get the gain state of the requested timerange
src_gs = get_gain_state(trange,
dt) # solar gain state for timerange of file
nt = len(src_gs['times'])
antgain = np.zeros((15, 2, 34, nt),
np.float32) # Antenna-based gains vs. band
for i in range(15):
for j in range(34):
antgain[i, 0, j] = src_gs['h1'][i] + src_gs['h2'][i] - ref_gs[
'h1'][i] - ref_gs['h2'][i] + src_gs['dcmattn'][
i, 0, j] - ref_gs['dcmattn'][i, 0, j]
antgain[i, 1, j] = src_gs['v1'][i] + src_gs['v2'][i] - ref_gs[
'v1'][i] - ref_gs['v2'][i] + src_gs['dcmattn'][
i, 1, j] - ref_gs['dcmattn'][i, 1, j]
cdata = copy.deepcopy(data)
# Create giant array of baseline-based gains, translated to baselines and frequencies
fghz = data['fghz']
nf = len(fghz)
blist = (fghz * 2 - 1).astype(
int) - 1 # Band list corresponding to frequencies in data
blgain = np.zeros((nf, 136, 4, nt),
float) # Baseline-based gains vs. frequency
for k, bl in enumerate(get_bl_order()):
i, j = bl
if i < 15 and j < 15:
blgain[:, k,
0] = 10**((antgain[i, 0, blist] + antgain[j, 0, blist]) /
20.)
blgain[:, k,
1] = 10**((antgain[i, 1, blist] + antgain[j, 1, blist]) /
20.)
blgain[:, k,
2] = 10**((antgain[i, 0, blist] + antgain[j, 1, blist]) /
20.)
blgain[:, k,
3] = 10**((antgain[i, 1, blist] + antgain[j, 0, blist]) /
20.)
# Reorder antgain axes to put frequencies in first slot, to match data
antgain = np.swapaxes(np.swapaxes(antgain, 1, 2), 0, 1)
antgainf = 10**(antgain[blist] / 10.)
idx = nearest_val_idx(data['time'], src_gs['times'].jd)
# Correct the auto- and cross-correlation data
cdata['x'] *= blgain[:, :, :, idx]
# Reshape px and py arrays
cdata['px'].shape = (134, 16, 3, nt)
cdata['py'].shape = (134, 16, 3, nt)
# Correct the power
cdata['px'][:, :15, 0] *= antgainf[:, :, 0, idx]
cdata['py'][:, :15, 0] *= antgainf[:, :, 1, idx]
# Correct the power-squared
cdata['px'][:, :15, 1] *= antgainf[:, :, 0, idx]**2
cdata['py'][:, :15, 1] *= antgainf[:, :, 1, idx]**2
# Reshape px and py arrays back to original
cdata['px'].shape = (134 * 16 * 3, nt)
cdata['py'].shape = (134 * 16 * 3, nt)
return cdata
def unrot(data, azeldict=None):
''' Apply the correction to differential feed rotation to data, and return
the corrected data.
Inputs:
data A dictionary returned by udb_util.py's readXdata().
azeldict The dictionary returned from get_sql_info(), or if None, the appropriate
get_sql_info() call is done internally.
Output:
cdata A dictionary with the phase-corrected data. Only the key
x is updated.
'''
import copy
from util import lobe
trange = Time(data['time'][[0, -1]], format='jd')
if azeldict is None:
azeldict = get_sql_info(trange)
chi = azeldict['ParallacticAngle'] # (nt, nant)
# Correct parallactic angle for equatorial mounts, relative to Ant14
for i in [8, 9, 10, 12, 13]:
chi[:, i] -= chi[:, 13]
# Ensure that nearest valid parallactic angle is used for times in the data
good, = np.where(azeldict['ActualAzimuth'][0] != 0)
tidx = nearest_val_idx(data['time'], azeldict['Time'][good].jd)
# Read X-Y Delay phase from SQL database and get common frequencies
xml, buf = ch.read_cal(11, t=trange[0])
fghz = stateframe.extract(buf, xml['FGHz'])
good, = np.where(fghz != 0.)
fghz = fghz[good]
dph = stateframe.extract(buf, xml['XYphase'])
dph = dph[:, good]
fidx1, fidx2 = common_val_idx(data['fghz'], fghz, precision=4)
missing = np.setdiff1d(np.arange(len(data['fghz'])), fidx1)
nf, nbl, npol, nt = data['x'].shape
nf = len(fidx1)
# Correct data for X-Y delay phase
for k, bl in enumerate(get_bl_order()):
i, j = bl
if i < 14 and j < 14 and i != j:
a1 = lobe(dph[i, fidx2] - dph[j, fidx2])
a2 = -dph[j, fidx2] + np.pi / 2
a3 = dph[i, fidx2] - np.pi / 2
data['x'][fidx1, k, 1] *= np.repeat(np.exp(1j * a1),
nt).reshape(nf, nt)
data['x'][fidx1, k, 2] *= np.repeat(np.exp(1j * a2),
nt).reshape(nf, nt)
data['x'][fidx1, k, 3] *= np.repeat(np.exp(1j * a3),
nt).reshape(nf, nt)
# Correct data for differential feed rotation
cdata = copy.deepcopy(data)
for n in range(nt):
for k, bl in enumerate(get_bl_order()):
i, j = bl
if i < 14 and j < 14 and i != j:
dchi = chi[n, i] - chi[n, j]
cchi = np.cos(dchi)
schi = np.sin(dchi)
cdata['x'][:, k, 0,
n] = data['x'][:, k, 0,
n] * cchi + data['x'][:, k, 3,
n] * schi
cdata['x'][:, k, 2,
n] = data['x'][:, k, 2,
n] * cchi + data['x'][:, k, 1,
n] * schi
cdata['x'][:, k, 3,
n] = data['x'][:, k, 3,
n] * cchi - data['x'][:, k, 0,
n] * schi
cdata['x'][:, k, 1,
n] = data['x'][:, k, 1,
n] * cchi - data['x'][:, k, 2,
n] * schi
# Set flags for any missing frequencies (hopefully this also works when missing is np.array([]))
cdata[missing] = np.ma.masked
return cdata
def apply_fem_level(data, gctime=None):
''' Applys the FEM level corrections to the given data dictionary.
Inputs:
data A dictionary such as that returned by read_idb().
gctime A Time() object whose date specifies which GAINCALTEST
measurements to use. If omitted, the date of the data
is used.
Output:
cdata A dictionary with the level-corrected data. The keys
p, x, p2, and a are all updated.
'''
from util import common_val_idx, nearest_val_idx, bl2ord
import attncal as ac
from gaincal2 import get_fem_level
import copy
# Get timerange from data
trange = Time([data['time'][0], data['time'][-1]], format='jd')
if gctime is None:
gctime = trange[0]
# Get time cadence
dt = np.int(
np.round(np.median(data['time'][1:] - data['time'][:-1]) * 86400))
if dt == 1: dt = None
# Get the FEM levels of the requested timerange
src_lev = get_fem_level(trange,
dt) # solar gain state for timerange of file
nf = len(data['fghz'])
nt = len(src_lev['times'])
attn = ac.read_attncal(
gctime
)[0] # Reads attn from SQL database (returns a list, but use first, generally only, one)
# attn = ac.get_attncal(gctime)[0] # Analyzes GAINCALTEST (returns a list, but use first, generally only, one)
antgain = np.zeros((15, 2, nf, nt),
np.float32) # Antenna-based gains [dB] vs. frequency
# Find common frequencies of attn with data
idx1, idx2 = common_val_idx(data['fghz'], attn['fghz'], precision=4)
# Currently, GAINCALTEST measures 8 levels of attenuation (16 dB). I assumed this would be enough,
# but the flare of 2017-09-10 actually went to 10 levels (20 dB), so we have no choice but to extend
# to higher levels using only the nominal, 2 dB steps above the 8th level. This part of the code
# extends to the maximum 14 levels.
a = np.zeros((14, 13, 2, nf), float) # Extend attenuation to 14 levels
a[:8, :, :, idx1] = attn[
'attn'][:, :13, :,
idx2] # Use GAINCALTEST results in the first 8 levels
for i in range(7, 13):
# Extend to levels 9-14 by adding 2 dB to each previous level
a[i + 1] = a[i] + 2.
for i in range(13):
for k, j in enumerate(idx1):
antgain[i, 0, j] = a[src_lev['hlev'][i], i, 0, j]
antgain[i, 1, j] = a[src_lev['vlev'][i], i, 0, j]
cdata = copy.deepcopy(data)
nblant = 136
blgain = np.zeros((nf, nblant, 4, nt),
float) # Baseline-based gains vs. frequency
for i in range(14):
for j in range(i, 14):
k = bl2ord[i, j]
blgain[:, k, 0] = 10**((antgain[i, 0] + antgain[j, 0]) / 20.)
blgain[:, k, 1] = 10**((antgain[i, 1] + antgain[j, 1]) / 20.)
blgain[:, k, 2] = 10**((antgain[i, 0] + antgain[j, 1]) / 20.)
blgain[:, k, 3] = 10**((antgain[i, 1] + antgain[j, 0]) / 20.)
# Reorder antgain axes to put frequencies in first slot, to match data
antgain = np.swapaxes(np.swapaxes(antgain, 1, 2), 0, 1)
antgainf = 10**(antgain / 10.)
idx = nearest_val_idx(data['time'], src_lev['times'].jd)
nt = len(idx) # New number of times
# Correct the auto- and cross-correlation data
cdata['x'] *= blgain[:, :, :, idx]
# Reshape px and py arrays
cdata['px'].shape = (nf, 16, 3, nt)
cdata['py'].shape = (nf, 16, 3, nt)
# Correct the power
cdata['px'][:, :15, 0] *= antgainf[:, :, 0, idx]
cdata['py'][:, :15, 0] *= antgainf[:, :, 1, idx]
# Correct the power-squared
cdata['px'][:, :15, 1] *= antgainf[:, :, 0, idx]**2
cdata['py'][:, :15, 1] *= antgainf[:, :, 1, idx]**2
# Reshape px and py arrays back to original
cdata['px'].shape = (nf * 16 * 3, nt)
cdata['py'].shape = (nf * 16 * 3, nt)
return cdata
def unrot(data, azeldict=None):
''' Apply the correction to differential feed rotation to data, and return
the corrected data. This also applies flags to data whose antennas are
not tracking.
Inputs:
data A dictionary returned by udb_util.py's readXdata().
azeldict The dictionary returned from get_sql_info(), or if None, the appropriate
get_sql_info() call is done internally.
Output:
cdata A dictionary with the phase-corrected data. Only the key
x is updated.
'''
import copy
from util import lobe, bl2ord
trange = Time(data['time'][[0, -1]], format='jd')
if azeldict is None:
azeldict = get_sql_info(trange)
chi = azeldict['ParallacticAngle'] * np.pi / 180. # (nt, nant)
# Correct parallactic angle for equatorial mounts, relative to Ant14
chi[:,
[8, 9, 10, 12, 13]] = 0 # Currently 0, but can be measured and updated
# Which antennas are tracking
track = azeldict['TrackFlag'] # True if tracking
# Ensure that nearest valid parallactic angle is used for times in the data
good = np.where(azeldict['ActualAzimuth'] != 0)
tidx = [] # List of arrays of indexes for each antenna
for i in range(14):
gd = good[0][np.where(good[1] == i)]
tidx.append(nearest_val_idx(data['time'], azeldict['Time'][gd].jd))
# Read X-Y Delay phase from SQL database and get common frequencies
xml, buf = ch.read_cal(11, t=trange[0])
fghz = stateframe.extract(buf, xml['FGHz'])
good, = np.where(fghz != 0.)
fghz = fghz[good]
dph = stateframe.extract(buf, xml['XYphase'])
dph = dph[:, good]
xi_rot = stateframe.extract(buf, xml['Xi_Rot'])
xi_rot = xi_rot[good]
fidx1, fidx2 = common_val_idx(data['fghz'], fghz, precision=4)
missing = np.setdiff1d(np.arange(len(data['fghz'])), fidx1)
nf, nbl, npol, nt = data['x'].shape
nf = len(fidx1)
# Correct data for X-Y delay phase
for i in range(13):
for j in range(i + 1, 14):
k = bl2ord[i, j]
a1 = lobe(dph[i, fidx2] - dph[j, fidx2])
a2 = -dph[j, fidx2] - xi_rot[fidx2]
a3 = dph[i, fidx2] - xi_rot[fidx2] + np.pi
data['x'][fidx1, k, 1] *= np.repeat(np.exp(1j * a1),
nt).reshape(nf, nt)
data['x'][fidx1, k, 2] *= np.repeat(np.exp(1j * a2),
nt).reshape(nf, nt)
data['x'][fidx1, k, 3] *= np.repeat(np.exp(1j * a3),
nt).reshape(nf, nt)
# Correct data for differential feed rotation
cdata = copy.deepcopy(data)
for n in range(nt):
for i in range(13):
for j in range(i + 1, 14):
k = bl2ord[i, j]
ti = tidx[i][n]
tj = tidx[j][n]
if track[ti, i] and track[tj, j]:
dchi = chi[ti, i] - chi[tj, j]
cchi = np.cos(dchi)
schi = np.sin(dchi)
cdata['x'][:, k, 0,
n] = data['x'][:, k, 0,
n] * cchi + data['x'][:, k, 3,
n] * schi
cdata['x'][:, k, 2,
n] = data['x'][:, k, 2,
n] * cchi + data['x'][:, k, 1,
n] * schi
cdata['x'][:, k, 3,
n] = data['x'][:, k, 3,
n] * cchi - data['x'][:, k, 0,
n] * schi
cdata['x'][:, k, 1,
n] = data['x'][:, k, 1,
n] * cchi - data['x'][:, k, 2,
n] * schi
else:
cdata['x'][:, k, :, n] = np.ma.masked
# Set flags for any missing frequencies (hopefully this also works when "missing" is np.array([]))
cdata['x'][missing] = np.ma.masked
return cdata
def apply_gain_corr(data, tref=None):
''' Applys the gain_state() corrections to the given data dictionary,
corrected to the gain-state at time given by Time() object tref.
Inputs:
data A dictionary such as that returned by read_idb().
tref A Time() object with the reference time, or if None,
the gain state of the nearest earlier REFCAL is
used.
Output:
cdata A dictionary with the gain-corrected data. The keys
p, x, p2, and a are all updated.
'''
from util import fname2mjd, common_val_idx, nearest_val_idx
import copy
if tref is None:
# No reference time specified, so get nearest PHASECAL scan (should guarantee femauto-off state)
mjd = int(Time(data['time'][0], format='jd').mjd)
trange = Time([mjd + 10. / 24., mjd + 1.], format='mjd')
import pcal_anal as pa
try:
# Get filename of first PHASECAL on this day, and use it to get reference time
scanfile = pa.findfile(trange)['scanlist'][0][0]
tref = Time(
fname2mjd(scanfile) + 60. / 86400,
format='mjd') # Add one minute to ensure scan is active
# Get the gain state at the reference time (actually median over 1 minute)
trefrange = Time([tref.iso, Time(tref.lv + 61, format='lv').iso])
ref_gs = get_gain_state(trefrange) # refcal gain state for 60 s
except:
# No phasecal in timerange, so just use an early time as reference
tref = Time(trange[0].iso[:10] + ' 13:30')
# Get the gain state at the reference time (actually median over 1 minute)
trefrange = Time([tref.iso, Time(tref.lv + 61, format='lv').iso])
ref_gs = get_gain_state(
trefrange,
relax=True) # refcal gain state for 60 s after ref time
if trange[0].mjd - tref.mjd > 2:
# Reference calibration is too old, so just use an early time as reference
tref = Time(trange[0].iso[:10] + ' 13:30')
# Get the gain state at the reference time (actually median over 1 minute)
trefrange = Time([tref.iso, Time(tref.lv + 61, format='lv').iso])
ref_gs = get_gain_state(
trefrange,
relax=True) # refcal gain state for 60 s after ref time
# Get median of refcal gain state (which should be constant anyway)
ref_gs['h1'] = np.median(ref_gs['h1'], 1)
ref_gs['h2'] = np.median(ref_gs['h2'], 1)
ref_gs['v1'] = np.median(ref_gs['v1'], 1)
ref_gs['v2'] = np.median(ref_gs['v2'], 1)
# Get timerange from data
trange = Time([data['time'][0], data['time'][-1]], format='jd')
# Get time cadence
dt = np.int(
np.round(np.median(data['time'][1:] - data['time'][:-1]) * 86400))
if dt == 1: dt = None
# Get the gain state of the requested timerange
src_gs = get_gain_state(trange,
dt) # solar gain state for timerange of file
nt = len(src_gs['times'])
nbands = src_gs['dcmattn'].shape[2]
if nbands != ref_gs['dcmattn'].shape[2]:
# Reference gain state is incompatible with this one, so set to src gain state (no correction will be applied)
ref_gs = src_gs
print 'GAINCAL2 Warning: Data and reference gain states are not compatible. No correction applied!'
antgain = np.zeros((15, 2, nbands, nt),
np.float32) # Antenna-based gains vs. band
for i in range(15):
for j in range(nbands):
antgain[i, 0, j] = src_gs['h1'][i] + src_gs['h2'][i] - ref_gs[
'h1'][i] - ref_gs['h2'][i] + src_gs['dcmattn'][
i, 0, j] - ref_gs['dcmattn'][i, 0, j]
antgain[i, 1, j] = src_gs['v1'][i] + src_gs['v2'][i] - ref_gs[
'v1'][i] - ref_gs['v2'][i] + src_gs['dcmattn'][
i, 1, j] - ref_gs['dcmattn'][i, 1, j]
cdata = copy.deepcopy(data)
# Frequency list is provided, so produce baseline-based gain table as well
# Create giant array of gains, translated to baselines and frequencies
fghz = data['fghz']
nf = len(fghz)
blist = (fghz * 2 - 1).astype(int) - 1
blgain = np.zeros((120, 4, nf, nt),
float) # Baseline-based gains vs. frequency
for i in range(14):
for j in range(i + 1, 15):
blgain[ri.bl2ord[i, j],
0] = 10**((antgain[i, 0, blist] + antgain[j, 0, blist]) /
20.)
blgain[ri.bl2ord[i, j],
1] = 10**((antgain[i, 1, blist] + antgain[j, 1, blist]) /
20.)
blgain[ri.bl2ord[i, j],
2] = 10**((antgain[i, 0, blist] + antgain[j, 1, blist]) /
20.)
blgain[ri.bl2ord[i, j],
3] = 10**((antgain[i, 1, blist] + antgain[j, 0, blist]) /
20.)
antgainf = 10**(antgain[:, :, blist] / 10.)
#idx1, idx2 = common_val_idx(data['time'],src_gs['times'].jd)
idx = nearest_val_idx(data['time'], src_gs['times'].jd)
# Apply corrections (some times may be eliminated from the data)
# Correct the cross-correlation data
cdata['x'] *= blgain[:, :, :, idx]
# Correct the power
cdata['p'][:15] *= antgainf[:, :, :, idx]
# Correct the autocorrelation
cdata['a'][:15, :2] *= antgainf[:, :, :, idx]
cross_fac = np.sqrt(antgainf[:, 0] * antgainf[:, 1])
cdata['a'][:15, 2] *= cross_fac[:, :, idx]
cdata['a'][:15, 3] *= cross_fac[:, :, idx]
# Correct the power-squared -- this should preserve SK
cdata['p2'][:15] *= antgainf[:, :, :, idx]**2
# Remove any uncorrected times before returning
#cdata['time'] = cdata['time'][idx1]
#cdata['p'] = cdata['p'][:,:,:,idx1]
#cdata['a'] = cdata['a'][:,:,:,idx1]
#cdata['p2'] = cdata['p2'][:,:,:,idx1]
#cdata['ha'] = cdata['ha'][idx1]
#cdata['m'] = cdata['m'][:,:,:,idx1]
return cdata
def tp_bgnd_all(tpdata):
''' Create time-variable background from ROACH inlet temperature
This version is far superior to the earlier, crude version, but
beware that it works best for a long timerange of data, especially
when there is a flare in the data.
Inputs:
tpdata dictionary returned by read_idb() NB: tpdata is not changed.
Returns:
bgnd The background fluctuation array of size (nf,nt) to be
subtracted from any antenna's total power (or mean of
antenna total powers)
'''
import dbutil as db
from util import Time, nearest_val_idx
outfghz = tpdata['fghz']
try:
outtime = tpdata['time']
trange = Time(outtime[[0, -1]], format='jd')
except:
outtime = tpdata['ut_mjd']
trange = Time(outtime[[0, -1]], format='mjd')
nt = len(outtime)
if nt < 1200:
print 'TP_BGND: Error, timebase too small. Must have at least 1200 time samples.'
return None
nf = len(outfghz)
outpd = Time(outtime, format='jd').plot_date
cursor = db.get_cursor()
data = db.get_dbrecs(cursor, dimension=8, timestamp=trange)
pd = Time(data['Timestamp'][:, 0].astype(int), format='lv').plot_date
inlet = data['Sche_Data_Roac_TempInlet'] # Inlet temperature variation
sinlet = np.sum(inlet.astype(float), 1)
# Eliminate 0 values in sinlet by replacing with nearest good value
bad, = np.where(sinlet == 0)
good, = np.where(sinlet != 0)
idx = nearest_val_idx(
bad, good) # Find locations of nearest good values to bad ones
sinlet[bad] = sinlet[good[idx]] # Overwrite bad values with good ones
sinlet -= np.mean(
sinlet) # Remove offset, to provide zero-mean fluctuation
sinlet = np.roll(
sinlet,
-110) # Shift phase of variation by 110 s earlier (seems to be needed)
# Interpolate sinlet values to the times in the data
sint = np.interp(outpd, pd, sinlet)
# sint = np.roll(sint,-90) # Shift phase of variation by 90 s earlier
# sint -= np.mean(sint) # Remove offset, to provide zero-mean fluctuation
sdev = np.std(sint)
sint_ok = np.abs(sint) < 2 * sdev
bgnd = np.zeros((13, 2, nf, nt), float)
for ant in range(13):
for pol in range(2):
for i in range(nf):
# Subtract smooth trend from data
nt = len(tpdata['p'][ant, pol, i])
wlen = min(nt, 2000)
if wlen % 2 != 0:
wlen -= 1
sig = tpdata['p'][ant, pol, i] - smooth(
tpdata['p'][ant, pol, i], wlen,
'blackman')[wlen / 2:-(wlen / 2 - 1)]
# Eliminate the worst outliers and repeat
stdev = np.nanstd(sig)
good, = np.where(np.abs(sig) < 2 * stdev)
if len(good) > nt * 0.1:
wlen = min(len(good), 2000)
if wlen % 2 != 0:
wlen -= 1
sig = tpdata['p'][ant, pol, i, good] - smooth(
tpdata['p'][ant, pol, i, good], wlen,
'blackman')[wlen / 2:-(wlen / 2 - 1)]
sint_i = sint[good]
stdev = np.std(sig)
# Final check for data quality
good, = np.where(
np.logical_and(sig < 2 * stdev, sint_ok[good]))
if len(good) > nt * 0.1:
p = np.polyfit(sint_i[good], sig[good], 1)
else:
p = [1., 0.]
# Apply correction for this frequency
bgnd[ant, pol, i] = sint * p[0] + p[1]
return bgnd
def get_fem_level(trange, dt=None):
''' Get FEM attenuation levels for a given timerange. Returns a dictionary
with keys as follows:
times: A Time object containing the array of times, size (nt)
hlev: The FEM attenuation level for HPol, size (nt, 15)
vlev: The FEM attenuation level for VPol, size (nt, 15)
dcmattn: The base DCM attenuations for 34 bands x 15 antennas x 2 Poln, size (34,30)
The order is Ant1 H, Ant1 V, Ant2 H, Ant2 V, etc.
dcmoff: If DPPoffset-on is 0, this is None (meaning there are no changes to the
above base attenuations).
If DPPoffset-on is 1, then dcmoff is a table of offsets to the
base attenuation, size (nt, 50). The offset applies to all
antennas/polarizations.
Optional keywords:
dt Seconds between entries to read from SQL stateframe database.
If omitted, 1 s is assumed.
'''
if dt is None:
tstart, tend = [str(i) for i in trange.lv]
else:
# Expand time by 1/2 of dt before and after
tstart = str(np.round(trange[0].lv - dt / 2))
tend = str(np.round(trange[1].lv + dt / 2))
cursor = db.get_cursor()
ver = db.find_table_version(cursor, trange[0].lv)
# Get front end attenuator states
query = 'select Timestamp,Ante_Fron_FEM_Clockms,' \
+'Ante_Fron_FEM_HPol_Regi_Level,Ante_Fron_FEM_VPol_Regi_Level from fV' \
+ver+'_vD15 where Timestamp >= '+tstart+' and Timestamp <= '+tend+' order by Timestamp'
data, msg = db.do_query(cursor, query)
if msg == 'Success':
if dt:
# If we want other than full cadence, get new array shapes and times
n = len(data['Timestamp']) # Original number of times
new_n = (
n / 15 / dt
) * 15 * dt # Truncated number of times equally divisible by dt
new_shape = (n / 15 / dt, dt, 15) # New shape of truncated arrays
times = Time(data['Timestamp'][:new_n].astype('int')[::15 * dt],
format='lv')
else:
times = Time(data['Timestamp'].astype('int')[::15], format='lv')
hlev = data['Ante_Fron_FEM_HPol_Regi_Level']
vlev = data['Ante_Fron_FEM_VPol_Regi_Level']
ms = data['Ante_Fron_FEM_Clockms']
nt = len(hlev) / 15
hlev.shape = (nt, 15)
vlev.shape = (nt, 15)
ms.shape = (nt, 15)
# Find any entries for which Clockms is zero, which indicates where no
# gain-state measurement is available.
for i in range(15):
bad, = np.where(ms[:, i] == 0)
if bad.size != 0 and bad.size != nt:
# Find nearest adjacent good value
good, = np.where(ms[:, i] != 0)
idx = nearest_val_idx(bad, good)
hlev[bad, i] = hlev[good[idx], i]
vlev[bad, i] = vlev[good[idx], i]
if dt:
# If we want other than full cadence, find mean over dt measurements
hlev = np.mean(hlev[:new_n / 15].reshape(new_shape), 1)
vlev = np.mean(vlev[:new_n / 15].reshape(new_shape), 1)
# Put results in canonical order [nant, nt]
hlev = hlev.T
vlev = vlev.T
else:
print 'Error reading FEM levels:', msg
return {}
# Get back end attenuator states
xml, buf = ch.read_cal(2, t=trange[0])
dcmattn = stf.extract(buf, xml['Attenuation'])
dcmattn.shape = (34, 15, 2)
# Put into canonical order [nant, npol, nband]
dcmattn = np.moveaxis(dcmattn, 0, 2)
# See if DPP offset is enabled
query = 'select Timestamp,DPPoffsetattn_on from fV' \
+ver+'_vD1 where Timestamp >= '+tstart+' and Timestamp <= '+tend+'order by Timestamp'
data, msg = db.do_query(cursor, query)
if msg == 'Success':
dppon = data['DPPoffsetattn_on']
if np.where(dppon > 0)[0].size == 0:
dcm_off = None
else:
query = 'select Timestamp,DCMoffset_attn from fV' \
+ver+'_vD50 where Timestamp >= '+tstart+' and Timestamp <= '+tend+' order by Timestamp'
data, msg = db.do_query(cursor, query)
if msg == 'Success':
otimes = Time(data['Timestamp'].astype('int')[::15],
format='lv')
dcmoff = data['DCMoffset_attn']
dcmoff.shape = (nt, 50)
# We now have a time-history of offsets, at least some of which are non-zero.
# Offsets by slot number do us no good, so we need to translate to band number.
# Get fseqfile name at mean of timerange, from stateframe SQL database
fseqfile = get_fseqfile(
Time(int(np.mean(trange.lv)), format='lv'))
if fseqfile is None:
print 'Error: No active fseq file.'
dcm_off = None
else:
# Get fseqfile from ACC and return bandlist
bandlist = fseqfile2bandlist(fseqfile)
# Use bandlist to covert nt x 50 array to nt x 34 band array of DCM attn offsets
# Note that this assumes DCM offset is the same for any multiply-sampled bands
# in the sequence.
dcm_off = np.zeros((nt, 34), float)
dcm_off[:, bandlist - 1] = dcmoff
# Put into canonical order [nband, nt]
dcm_off = dcm_off.T
if dt:
# If we want other than full cadence, find mean over dt measurements
new_nt = len(times)
dcm_off = dcm_off[:, :new_nt * dt]
dcm_off.shape = (34, dt, new_nt)
dcm_off = np.mean(dcm_off, 1)
else:
print 'Error reading DCM attenuations:', msg
dcm_off = None
else:
print 'Error reading DPPon state:', msg
dcm_off = None
cursor.close()
return {
'times': times,
'hlev': hlev.astype(int),
'vlev': vlev.astype(int),
'dcmattn': dcmattn,
'dcmoff': dcm_off
}
def autocorrect(out, ant_str='ant1-13'):
nt = len(out['time'])
nf = len(out['fghz'])
pfac1 = (out['p'][:, :, :, :-1] -
out['p'][:, :, :, 1:]) / out['p'][:, :, :, :-1]
trange = Time(out['time'][[0, -1]], format='jd')
src_lev = gc.get_fem_level(trange) # Read FEM levels from SQL
# Match times with data
tidx = nearest_val_idx(out['time'], src_lev['times'].jd)
# Find attenuation changes
for ant in range(13):
for pol in range(2):
if pol == 0:
lev = src_lev['hlev'][ant, tidx]
else:
lev = src_lev['vlev'][ant, tidx]
jidx, = np.where(abs(lev[:-1] - lev[1:]) == 1)
for freq in range(nf):
idx, = np.where(
np.logical_and(
abs(pfac1[ant, pol, freq]) > 0.05,
abs(pfac1[ant, pol, freq]) < 0.95))
for i in range(len(idx - 1)):
if idx[i] in jidx or idx[i] in jidx - 1:
out['p'][ant, pol, freq, idx[i] +
1:] /= (1 - pfac1[ant, pol, freq, idx[i]])
calfac = pc.get_calfac(trange[0])
tpcalfac = calfac['tpcalfac']
tpoffsun = calfac['tpoffsun']
hlev = src_lev['hlev'][:13, 0]
vlev = src_lev['vlev'][:13, 0]
attn_dict = ac.read_attncal(trange[0])[0] # Read GCAL attn from SQL
attn = np.zeros((13, 2, nf))
for i in range(13):
attn[i, 0] = attn_dict['attn'][hlev[i], 0, 0]
attn[i, 1] = attn_dict['attn'][vlev[i], 0, 1]
print 'Ant', i + 1, attn[i, 0, 20], attn[i, 1, 20]
attnfac = 10**(attn / 10.)
for i in range(13):
print attnfac[i, 0, 20], attnfac[i, 1, 20]
for i in range(nt):
out['p'][:13, :, :,
i] = (out['p'][:13, :, :, i] * attnfac - tpoffsun) * tpcalfac
antlist = ant_str2list(ant_str)
med = np.mean(np.median(out['p'][antlist], 0), 0)
bg = np.median(med[:, 0:300], 1).repeat(nt).reshape(nf, nt)
med -= bg
pdata = np.log10(med)
f, ax = plt.subplots(1, 1)
vmax = np.median(np.nanmax(pdata, 1))
im = ax.pcolormesh(Time(out['time'], format='jd').plot_date,
out['fghz'],
pdata,
vmin=1,
vmax=vmax)
plt.colorbar(im, ax=ax, label='Log Flux Density [sfu]')
ax.xaxis_date()
ax.xaxis.set_major_formatter(DateFormatter("%H:%M"))
ax.set_ylim(out['fghz'][0], out['fghz'][-1])
ax.set_xlabel('Time [UT]')
ax.set_ylabel('Frequency [GHz]')
return out, med
def get_attncal(trange, do_plot=False, dataonly=False):
''' Finds GAINCALTEST scans from FDB files corresponding to the days
present in trange Time() object (can be multiple days), calculates
the attenuation differences for the various FEMATTN states 1-8
relative to FEMATTN state 0, and optionally plots the results for
states 1 and 2 (the most commonly used). To analyze only a single
day, trange Time() object can have the same time repeated, or can
be a single time.
Returns a list of dictionaries, each pertaining to one of the days
in trange, with keys defined as follows:
'time': The start time of the GAINCALTEST scan, as a Time() object
'fghz': The list of frequencies [GHz] at which attenuations are measured
'attn': The array of attenuations [dB] of size (nattn, nant, npol, nf),
where nattn = 8, nant = 13, npol = 2, and nf is variable
'rcvr': The array of receiver noise level (raw units) of size
(nant, npol, nf), where nant = 13, npol = 2, and nf is variable
'rcvr_auto': Same as rcvr, but for auto-correlation (hence it is complex)
N.B.: Ignores days with other than one GAINCALTEST measurement, e.g. 0 or 2,
the first is obvious, while the second is because there is no way to
tell which of the 2 are good.
The dataonly parameter tells the routine to skip calculating the attenuation
and only return the IDB data from the (first) gaincal.
'''
from util import get_idbdir, fname2mjd, nearest_val_idx
import socket
import dbutil
if type(trange.mjd) == np.float:
# Interpret single time as both start and end time
mjd1 = int(trange.mjd)
mjd2 = mjd1
else:
mjd1, mjd2 = trange.mjd.astype(int)
if do_plot:
import matplotlib.pylab as plt
f, ax = plt.subplots(4, 13)
f.set_size_inches((14, 5))
ax[0, 0].set_ylabel('Atn1X [dB]')
ax[1, 0].set_ylabel('Atn1Y [dB]')
ax[2, 0].set_ylabel('Atn2X [dB]')
ax[3, 0].set_ylabel('Atn2Y [dB]')
for i in range(13):
ax[0, i].set_title('Ant ' + str(i + 1))
ax[3, i].set_xlabel('Freq [GHz]')
for j in range(2):
ax[j, i].set_ylim(1, 3)
ax[j + 2, i].set_ylim(3, 5)
outdict = []
for mjd in range(mjd1, mjd2 + 1):
fdb = dt.rd_fdb(Time(mjd, format='mjd'))
gcidx, = np.where(fdb['PROJECTID'] == 'GAINCALTEST')
if len(gcidx) == 1:
print fdb['FILE'][gcidx]
gcidx = gcidx[0]
else:
for i, fname in enumerate(fdb['FILE'][gcidx]):
print str(i) + ': GAINCALTEST File', fname
idex = input('There is more than one GAINCALTEST. Select: ' +
str(np.arange(len(gcidx))) + ':')
gcidx = gcidx[idex]
datadir = get_idbdir(Time(mjd, format='mjd'))
# Add date path if on pipeline
# if datadir.find('eovsa') != -1: datadir += fdb['FILE'][gcidx][3:11]+'/'
host = socket.gethostname()
if host == 'pipeline': datadir += fdb['FILE'][gcidx][3:11] + '/'
file = datadir + fdb['FILE'][gcidx]
out = ri.read_idb([file])
if dataonly:
return out
# Get time from filename and read 120 records of attn state from SQL database
filemjd = fname2mjd(fdb['FILE'][gcidx])
cursor = dbutil.get_cursor()
d15 = dbutil.get_dbrecs(cursor,
dimension=15,
timestamp=Time(filemjd, format='mjd'),
nrecs=120)
cursor.close()
# Find time indexes of the 62 dB attn state
# Uses only ant 1 assuming all are the same
dtot = (d15['Ante_Fron_FEM_HPol_Atte_Second'] +
d15['Ante_Fron_FEM_HPol_Atte_First'])[:, 0]
# Use system clock day number to identify bad SQL entries and eliminate them
good, = np.where(d15['Ante_Cont_SystemClockMJDay'][:, 0] != 0)
#import pdb; pdb.set_trace()
# Indexes into SQL records where a transition occurred.
transitions, = np.where(dtot[good] - np.roll(dtot[good], 1) != 0)
# Eliminate any zero-index transition (if it exists)
if transitions[0] == 0:
transitions = transitions[1:]
# These now have to be translated into indexes into the data, using the times
idx = nearest_val_idx(d15['Timestamp'][good, 0][transitions],
Time(out['time'], format='jd').lv)
#import pdb; pdb.set_trace()
vx = np.nanmedian(
out['p'][:13, :, :, np.arange(idx[0] + 1, idx[1] - 1)], 3)
va = np.mean(out['a'][:13, :2, :,
np.arange(idx[0] + 1, idx[1] - 1)], 3)
vals = []
attn = []
for i in range(1, 10):
vals.append(
np.nanmedian(
out['p'][:13, :, :,
np.arange(idx[i] + 1, idx[i + 1] - 1)], 3) - vx)
attn.append(np.log10(vals[0] / vals[-1]) * 10.)
#vals = []
#attna = []
#for i in range(1,10):
# vals.append(np.median(out['a'][:13,:2,:,np.arange(idx[i],idx[i+1])],3) - va)
# attna.append(np.log10(vals[0]/vals[-1])*10.)
if do_plot:
for i in range(13):
for j in range(2):
ax[j, i].plot(out['fghz'],
attn[1][i, j],
'.',
markersize=3)
#ax[j,i].plot(out['fghz'],attna[1][i,j],'.',markersize=1)
ax[j + 2, i].plot(out['fghz'],
attn[2][i, j],
'.',
markersize=3)
#ax[j+2,i].plot(out['fghz'],attna[2][i,j],'.',markersize=1)
outdict.append({
'time': Time(out['time'][0], format='jd'),
'fghz': out['fghz'],
'rcvr_auto': va, # 'attna': np.array(attna[1:]),
'rcvr': vx,
'attn': np.array(attn[1:])
})
return outdict
def apply_fem_level(data, gctime=None, skycal=None):
''' Applys the FEM level corrections to the given data dictionary.
Inputs:
data A dictionary such as that returned by readXdata().
gctime A Time() object whose date specifies which GAINCALTEST
measurements to use. If omitted, the date of the data
is used.
skycal Optional array of receiver noise from SKYCAL or GAINCAL
calibration. Only the receiver noise is applied (subtracted)
Output:
cdata A dictionary with the level-corrected data. The keys
p, x, p2, and a are all updated.
'''
import attncal as ac
from gaincal2 import get_fem_level
import copy
# Get timerange from data
trange = Time([data['time'][0], data['time'][-1]], format='jd')
if gctime is None:
gctime = trange[0]
# Get time cadence
dt = np.int(
np.round(np.nanmedian(data['time'][1:] - data['time'][:-1]) * 86400))
if dt == 1: dt = None
# Get the FEM levels of the requested timerange
src_lev = get_fem_level(trange,
dt) # solar gain state for timerange of file
nf = len(data['fghz'])
nt = len(src_lev['times'])
attn = ac.read_attncal(
gctime
)[0] # Reads attn from SQL database (returns a list, but use first, generally only, one)
# attn = ac.get_attncal(gctime)[0] # Analyzes GAINCALTEST (returns a list, but use first, generally only, one)
antgain = np.zeros((15, 2, nf, nt),
np.float32) # Antenna-based gains [dB] vs. frequency
# Find common frequencies of attn with data
idx1, idx2 = common_val_idx(data['fghz'], attn['fghz'], precision=4)
# Currently, GAINCALTEST measures 8 levels of attenuation (16 dB). I assumed this would be enough,
# but the flare of 2017-09-10 actually went to 10 levels (20 dB), so we have no choice but to extend
# to higher levels using only the nominal, 2 dB steps above the 8th level. This part of the code
# extends to the maximum 16 levels.
a = np.zeros((16, 13, 2, nf), float) # Extend attenuation to 14 levels
a[1:9, :, :, idx1] = attn[
'attn'][:, :13, :,
idx2] # Use GAINCALTEST results in levels 1-9 (bottom level is 0dB)
for i in range(8, 15):
# Extend to levels 9-15 by adding 2 dB to each previous level
a[i + 1] = a[i] + 2.
a[15] = 62. # Level 15 means 62 dB have been inserted.
#print 'Attn list (dB) for ant 1, pol xx, lowest frequency:',a[:,0,0,0]
if dt:
# For this case, src_lev is an array of dictionaries where keys are levels and
# values are the proportion of that level for the given integration
for i in range(13):
for k, j in enumerate(idx1):
for m in range(nt):
for lev, prop in src_lev['hlev'][i, m].items():
antgain[i, 0, j, m] += prop * a[lev, i, 0, idx2[k]]
for lev, prop in src_lev['vlev'][i, m].items():
antgain[i, 1, j, m] += prop * a[lev, i, 1, idx2[k]]
else:
# For this case, src_lev is just an array of levels
for i in range(13):
for k, j in enumerate(idx1):
antgain[i, 0, j] = a[src_lev['hlev'][i], i, 0, idx2[k]]
antgain[i, 1, j] = a[src_lev['vlev'][i], i, 1, idx2[k]]
cdata = copy.deepcopy(data)
nblant = 136
blgain = np.zeros((nf, nblant, 4, nt),
float) # Baseline-based gains vs. frequency
for i in range(14):
for j in range(i, 14):
k = bl2ord[i, j]
blgain[:, k, 0] = 10**((antgain[i, 0] + antgain[j, 0]) / 20.)
blgain[:, k, 1] = 10**((antgain[i, 1] + antgain[j, 1]) / 20.)
blgain[:, k, 2] = 10**((antgain[i, 0] + antgain[j, 1]) / 20.)
blgain[:, k, 3] = 10**((antgain[i, 1] + antgain[j, 0]) / 20.)
# Reorder antgain axes to put frequencies in first slot, to match data
antgain = np.swapaxes(np.swapaxes(antgain, 1, 2), 0, 1)
antgainf = 10**(antgain / 10.)
idx = nearest_val_idx(data['time'], src_lev['times'].jd)
nt = len(idx) # New number of times
# If a skycal dictionary exists, subtract auto-correlation receiver noise before scaling (clip to 0)
if skycal:
sna, snp, snf = skycal['rcvr_bgd_auto'].shape
bgd = skycal['rcvr_bgd_auto'].repeat(nt).reshape((sna, snp, snf, nt))
bgd = bgd[:, :, idx2] # Extract only frequencies matching the data
# Reorder axes
bgd = np.swapaxes(bgd, 0, 2)
# bslice = bgd[:,:,:,idx]
for i in range(13):
cdata['x'][:, bl2ord[i, i], 0] = np.clip(
cdata['x'][:, bl2ord[i, i], 0] - bgd[:, 0, i], 0,
None) #bslice[:,0,i],0,None)
cdata['x'][:, bl2ord[i, i], 1] = np.clip(
cdata['x'][:, bl2ord[i, i], 1] - bgd[:, 1, i], 0,
None) #bslice[:,1,i],0,None)
# Correct the auto- and cross-correlation data
cdata['x'] *= blgain[:, :, :, idx]
# Reshape px and py arrays
cdata['px'].shape = (nf, 16, 3, nt)
cdata['py'].shape = (nf, 16, 3, nt)
# If a skycal dictionary exists, subtract total power receiver noise before scaling (clip to 0)
# NB: This will break SK!
if skycal:
sna, snp, snf = skycal['rcvr_bgd'].shape
bgd = skycal['rcvr_bgd'].repeat(nt).reshape((sna, snp, snf, nt))
bgd = bgd[:, :, idx2] # Extract only frequencies matching the data
# Reorder axes
bgd = np.swapaxes(bgd, 0, 2)
#bslice = bgd[:,:,:,idx]
#bgnd = np.rollaxis(bslice,3)
cdata['px'][:, :13, 0] = np.clip(cdata['px'][:, :13, 0] - bgd[:, 0], 0,
None) #bslice[:,0],0,None)
cdata['py'][:, :13, 0] = np.clip(cdata['py'][:, :13, 0] - bgd[:, 1], 0,
None) #bslice[:,1],0,None)
# Correct the power
cdata['px'][:, :15, 0] *= antgainf[:, :, 0, idx]
cdata['py'][:, :15, 0] *= antgainf[:, :, 1, idx]
# Correct the power-squared
cdata['px'][:, :15, 1] *= antgainf[:, :, 0, idx]**2
cdata['py'][:, :15, 1] *= antgainf[:, :, 1, idx]**2
# Reshape px and py arrays back to original
cdata['px'].shape = (nf * 16 * 3, nt)
cdata['py'].shape = (nf * 16 * 3, nt)
return cdata
def rd_refcal(file, quackint=120., navg=3):
''' Reads a single UDB file representing a calibrator scan, and averages over the
bands in the file
'''
from read_idb import read_idb, bl2ord
from copy import deepcopy
import chan_util_bc as cu
import dbutil as db
out = read_idb([file], navg=navg, quackint=quackint)
bds = np.unique(out['band'])
nt = len(out['time'])
nbd = len(bds)
vis = np.zeros((15, 4, 34, nt), dtype=complex)
fghz = np.zeros(34)
# average over channels within each band
o = out['x'][bl2ord[13,:13]]
for bd in bds:
idx = np.where(out['band'] == bd)[0]
fghz[bd-1] = np.nanmean(out['fghz'][idx])
vis[:13,:,bd-1] = np.mean(o[:, :, idx], axis=2)
# Need to apply unrot to correct for feed rotation, before returning
xml, buf = ch.read_cal(11, Time(out['time'][0],format='jd'))
dph = extract(buf,xml['XYphase'])
xi_rot = extract(buf,xml['Xi_Rot'])
freq = extract(buf,xml['FGHz'])
freq = freq[np.where(freq != 0)]
band = []
for f in freq:
band.append(cu.freq2bdname(f))
bds, sidx = np.unique(band, return_index=True)
nbd = len(bds)
eidx = np.append(sidx[1:], len(band))
dxy = np.zeros((14, 34), dtype=np.float)
xi = np.zeros(34, dtype=np.float)
fghz = np.zeros(34)
# average dph and xi_rot frequencies within each band, to convert to 34-band representation
for b, bd in enumerate(bds):
fghz[bd - 1] = np.nanmean(freq[sidx[b]:eidx[b]])
xi[bd - 1] = np.nanmean(xi_rot[sidx[b]:eidx[b]])
for a in range(14):
dxy[a, bd - 1] = np.angle(np.sum(np.exp(1j * dph[a, sidx[b]:eidx[b]])))
# Read parallactic angles for this scan
trange = Time(out['time'][[0,-1]],format='jd')
times, chi = db.get_chi(trange)
tchi = times.jd
t = out['time']
if len(t) > 0:
vis2 = deepcopy(vis)
idx = nearest_val_idx(t, tchi)
pa = chi[idx] # Parallactic angle for the times of this refcal.
pa[:, [8, 9, 10, 12]] = 0.0
nt = len(idx) # Number of times in this refcal
# Apply X-Y delay phase correction
for a in range(13):
a1 = lobe(dxy[a] - dxy[13])
a2 = -dxy[13] - xi
a3 = dxy[a] - xi + np.pi
for j in range(nt):
vis2[a, 1, :, j] *= np.exp(1j * a1)
vis2[a, 2, :, j] *= np.exp(1j * a2)
vis2[a, 3, :, j] *= np.exp(1j * a3)
for j in range(nt):
for a in range(13):
vis[a, 0, :, j] = vis2[a, 0, :, j] * np.cos(pa[j, a]) + vis2[a, 3, :, j] * np.sin(pa[j, a])
vis[a, 2, :, j] = vis2[a, 2, :, j] * np.cos(pa[j, a]) + vis2[a, 1, :, j] * np.sin(pa[j, a])
vis[a, 3, :, j] = vis2[a, 3, :, j] * np.cos(pa[j, a]) - vis2[a, 0, :, j] * np.sin(pa[j, a])
vis[a, 1, :, j] = vis2[a, 1, :, j] * np.cos(pa[j, a]) - vis2[a, 2, :, j] * np.sin(pa[j, a])
# *******
return {'file': file, 'source': out['source'], 'vis': vis, 'bands': bds, 'fghz': fghz, 'times': out['time'], 'ha': out['ha'], 'dec': out['dec'], 'flag': np.zeros_like(vis, dtype=np.int)}
def apply_fem_level(data, gctime=None):
''' Applys the FEM level corrections to the given data dictionary.
Inputs:
data A dictionary such as that returned by read_idb().
gctime A Time() object whose date specifies which GAINCALTEST
measurements to use. If omitted, the date of the data
is used.
Output:
cdata A dictionary with the level-corrected data. The keys
p, x, p2, and a are all updated.
'''
from util import common_val_idx, nearest_val_idx
import attncal as ac
import copy
# Get timerange from data
trange = Time([data['time'][0], data['time'][-1]], format='jd')
if gctime is None:
gctime = trange[0]
# Get time cadence
dt = np.int(
np.round(np.median(data['time'][1:] - data['time'][:-1]) * 86400))
if dt == 1: dt = None
# Get the FEM levels of the requested timerange
src_lev = get_fem_level(trange,
dt) # solar gain state for timerange of file
nf = len(data['fghz'])
nt = len(src_lev['times'])
# First attempt to read from the SQL database. If that fails, read from the IDB file itself
try:
attn = ac.read_attncal(gctime)[0] # Attn from SQL
except:
attn = ac.get_attncal(
gctime
)[0] # Attn measured by GAINCALTEST (returns a list, but use first, generally only, one)
antgain = np.zeros((15, 2, nf, nt),
np.float32) # Antenna-based gains [dB] vs. frequency
# Find common frequencies of attn with data
idx1, idx2 = common_val_idx(data['fghz'], attn['fghz'], precision=4)
a = attn['attn']
for i in range(13):
for k, j in enumerate(idx1):
antgain[i, 0, j] = a[src_lev['hlev'][i], i, 0, idx2[k]]
antgain[i, 1, j] = a[src_lev['vlev'][i], i, 0, idx2[k]]
cdata = copy.deepcopy(data)
blgain = np.zeros((120, 4, nf, nt),
float) # Baseline-based gains vs. frequency
for i in range(14):
for j in range(i + 1, 15):
blgain[ri.bl2ord[i, j],
0] = 10**((antgain[i, 0] + antgain[j, 0]) / 20.)
blgain[ri.bl2ord[i, j],
1] = 10**((antgain[i, 1] + antgain[j, 1]) / 20.)
blgain[ri.bl2ord[i, j],
2] = 10**((antgain[i, 0] + antgain[j, 1]) / 20.)
blgain[ri.bl2ord[i, j],
3] = 10**((antgain[i, 1] + antgain[j, 0]) / 20.)
antgainf = 10**(antgain / 10.)
#idx1, idx2 = common_val_idx(data['time'],src_gs['times'].jd)
idx = nearest_val_idx(data['time'], src_lev['times'].jd)
# Apply corrections (some times may be eliminated from the data)
# Correct the cross-correlation data
cdata['x'] *= blgain[:, :, :, idx]
# Correct the power
cdata['p'][:15] *= antgainf[:, :, :, idx]
# Correct the autocorrelation
cdata['a'][:15, :2] *= antgainf[:, :, :, idx]
cross_fac = np.sqrt(antgainf[:, 0] * antgainf[:, 1])
cdata['a'][:15, 2] *= cross_fac[:, :, idx]
cdata['a'][:15, 3] *= cross_fac[:, :, idx]
# Correct the power-squared -- this should preserve SK
cdata['p2'][:15] *= antgainf[:, :, :, idx]**2
# Remove any uncorrected times before returning
#cdata['time'] = cdata['time'][idx1]
#cdata['p'] = cdata['p'][:,:,:,idx1]
#cdata['a'] = cdata['a'][:,:,:,idx1]
#cdata['p2'] = cdata['p2'][:,:,:,idx1]
#cdata['ha'] = cdata['ha'][idx1]
#cdata['m'] = cdata['m'][:,:,:,idx1]
return cdata
def apply_attn_corr(data, tref=None):
''' Applys the attenuator state corrections to the given data dictionary,
corrected to the gain-state at time given by Time() object tref.
Inputs:
data A dictionary returned by udb_util.py's readXdata().
tref A Time() object with the reference time, or if None,
the gain state of the nearest earlier REFCAL is
used.
Output:
cdata A dictionary with the gain-corrected data. The keys
px, py, and x, are updated.
NB: This is the same routine as in gaincal2.py, but modified
to handle the different ordering/format of data from udb_util.py's
readXdata() routine.
'''
from gaincal2 import get_gain_state
from util import common_val_idx, nearest_val_idx
import copy
if tref is None:
# No reference time specified, so get nearest earlier REFCAL
trange = Time(data['time'][[0,-1]],format='jd')
xml, buf = ch.read_cal(8,t=trange[0])
tref = Time(stateframe.extract(buf,xml['Timestamp']),format='lv')
# Get the gain state at the reference time (actually median over 1 minute)
trefrange = Time([tref.iso,Time(tref.lv+60,format='lv').iso])
ref_gs = get_gain_state(trefrange) # refcal gain state for 60 s
# Get median of refcal gain state (which should be constant anyway)
ref_gs['h1'] = np.median(ref_gs['h1'],1)
ref_gs['h2'] = np.median(ref_gs['h2'],1)
ref_gs['v1'] = np.median(ref_gs['v1'],1)
ref_gs['v2'] = np.median(ref_gs['v2'],1)
# Get timerange from data
trange = Time([data['time'][0],data['time'][-1]],format='jd')
# Get time cadence
dt = np.int(np.round(np.median(data['time'][1:] - data['time'][:-1]) * 86400))
if dt == 1: dt = None
# Get the gain state of the requested timerange
src_gs = get_gain_state(trange,dt) # solar gain state for timerange of file
nt = len(src_gs['times'])
antgain = np.zeros((15,2,34,nt),np.float32) # Antenna-based gains vs. band
for i in range(15):
for j in range(34):
antgain[i,0,j] = src_gs['h1'][i] + src_gs['h2'][i] - ref_gs['h1'][i] - ref_gs['h2'][i] + src_gs['dcmattn'][i,0,j] - ref_gs['dcmattn'][i,0,j]
antgain[i,1,j] = src_gs['v1'][i] + src_gs['v2'][i] - ref_gs['v1'][i] - ref_gs['v2'][i] + src_gs['dcmattn'][i,1,j] - ref_gs['dcmattn'][i,1,j]
cdata = copy.deepcopy(data)
# Create giant array of baseline-based gains, translated to baselines and frequencies
fghz = data['fghz']
nf = len(fghz)
blist = (fghz*2 - 1).astype(int) - 1 # Band list corresponding to frequencies in data
blgain = np.zeros((nf,136,4,nt),float) # Baseline-based gains vs. frequency
for k,bl in enumerate(get_bl_order()):
i, j = bl
if i < 15 and j < 15:
blgain[:,k,0] = 10**((antgain[i,0,blist] + antgain[j,0,blist])/20.)
blgain[:,k,1] = 10**((antgain[i,1,blist] + antgain[j,1,blist])/20.)
blgain[:,k,2] = 10**((antgain[i,0,blist] + antgain[j,1,blist])/20.)
blgain[:,k,3] = 10**((antgain[i,1,blist] + antgain[j,0,blist])/20.)
# Reorder antgain axes to put frequencies in first slot, to match data
antgain = np.swapaxes(np.swapaxes(antgain,1,2),0,1)
antgainf = 10**(antgain[blist]/10.)
idx = nearest_val_idx(data['time'],src_gs['times'].jd)
# Correct the auto- and cross-correlation data
cdata['x'] *= blgain[:,:,:,idx]
# Reshape px and py arrays
cdata['px'].shape = (134,16,3,nt)
cdata['py'].shape = (134,16,3,nt)
# Correct the power
cdata['px'][:,:15,0] *= antgainf[:,:,0,idx]
cdata['py'][:,:15,0] *= antgainf[:,:,1,idx]
# Correct the power-squared
cdata['px'][:,:15,1] *= antgainf[:,:,0,idx]**2
cdata['py'][:,:15,1] *= antgainf[:,:,1,idx]**2
# Reshape px and py arrays back to original
cdata['px'].shape = (134*16*3,nt)
cdata['py'].shape = (134*16*3,nt)
return cdata
def apply_gain_corr(data, tref=None):
''' Applys the gain_state() corrections to the given data dictionary,
corrected to the gain-state at time given by Time() object tref.
Inputs:
data A dictionary such as that returned by read_idb().
tref A Time() object with the reference time, or if None,
the gain state of the nearest earlier REFCAL is
used.
Output:
cdata A dictionary with the gain-corrected data. The keys
p, x, p2, and a are all updated.
'''
from util import common_val_idx, nearest_val_idx
import copy
if tref is None:
# No reference time specified, so get nearest earlier REFCAL
trange = Time(data['time'][[0, -1]], format='jd')
xml, buf = ch.read_cal(8, t=trange[0])
if xml == {}:
# No refcal for this date, so just use an early time as reference
tref = Time(trange[0].iso[:10] + ' 13:30')
else:
tref = Time(stf.extract(buf, xml['Timestamp']), format='lv')
# Get the gain state at the reference time (actually median over 1 minute)
trefrange = Time([tref.iso, Time(tref.lv + 61, format='lv').iso])
ref_gs = get_gain_state(trefrange) # refcal gain state for 60 s
# Get median of refcal gain state (which should be constant anyway)
ref_gs['h1'] = np.median(ref_gs['h1'], 1)
ref_gs['h2'] = np.median(ref_gs['h2'], 1)
ref_gs['v1'] = np.median(ref_gs['v1'], 1)
ref_gs['v2'] = np.median(ref_gs['v2'], 1)
# Get timerange from data
trange = Time([data['time'][0], data['time'][-1]], format='jd')
# Get time cadence
dt = np.int(
np.round(np.median(data['time'][1:] - data['time'][:-1]) * 86400))
if dt == 1: dt = None
# Get the gain state of the requested timerange
src_gs = get_gain_state(trange,
dt) # solar gain state for timerange of file
nt = len(src_gs['times'])
antgain = np.zeros((15, 2, 34, nt),
np.float32) # Antenna-based gains vs. band
for i in range(15):
for j in range(34):
antgain[i, 0, j] = src_gs['h1'][i] + src_gs['h2'][i] - ref_gs[
'h1'][i] - ref_gs['h2'][i] + src_gs['dcmattn'][
i, 0, j] - ref_gs['dcmattn'][i, 0, j]
antgain[i, 1, j] = src_gs['v1'][i] + src_gs['v2'][i] - ref_gs[
'v1'][i] - ref_gs['v2'][i] + src_gs['dcmattn'][
i, 1, j] - ref_gs['dcmattn'][i, 1, j]
cdata = copy.deepcopy(data)
# Frequency list is provided, so produce baseline-based gain table as well
# Create giant array of gains, translated to baselines and frequencies
fghz = data['fghz']
nf = len(fghz)
blist = (fghz * 2 - 1).astype(int) - 1
blgain = np.zeros((120, 4, nf, nt),
float) # Baseline-based gains vs. frequency
for i in range(14):
for j in range(i + 1, 15):
blgain[ri.bl2ord[i, j],
0] = 10**((antgain[i, 0, blist] + antgain[j, 0, blist]) /
20.)
blgain[ri.bl2ord[i, j],
1] = 10**((antgain[i, 1, blist] + antgain[j, 1, blist]) /
20.)
blgain[ri.bl2ord[i, j],
2] = 10**((antgain[i, 0, blist] + antgain[j, 1, blist]) /
20.)
blgain[ri.bl2ord[i, j],
3] = 10**((antgain[i, 1, blist] + antgain[j, 0, blist]) /
20.)
antgainf = 10**(antgain[:, :, blist] / 10.)
#idx1, idx2 = common_val_idx(data['time'],src_gs['times'].jd)
idx = nearest_val_idx(data['time'], src_gs['times'].jd)
# Apply corrections (some times may be eliminated from the data)
# Correct the cross-correlation data
cdata['x'] *= blgain[:, :, :, idx]
# Correct the power
cdata['p'][:15] *= antgainf[:, :, :, idx]
# Correct the autocorrelation
cdata['a'][:15, :2] *= antgainf[:, :, :, idx]
cross_fac = np.sqrt(antgainf[:, 0] * antgainf[:, 1])
cdata['a'][:15, 2] *= cross_fac[:, :, idx]
cdata['a'][:15, 3] *= cross_fac[:, :, idx]
# Correct the power-squared -- this should preserve SK
cdata['p2'][:15] *= antgainf[:, :, :, idx]**2
# Remove any uncorrected times before returning
#cdata['time'] = cdata['time'][idx1]
#cdata['p'] = cdata['p'][:,:,:,idx1]
#cdata['a'] = cdata['a'][:,:,:,idx1]
#cdata['p2'] = cdata['p2'][:,:,:,idx1]
#cdata['ha'] = cdata['ha'][idx1]
#cdata['m'] = cdata['m'][:,:,:,idx1]
return cdata
def calpntanal(t,ant_str='ant1-13',do_plot=True,ax=None):
''' Does a complete analysis of CALPNTCAL, reading information from the SQL
database, finding the corresponding Miriad IDB data, and doing the
gaussian fit to the beam, to return the beam and offset parameters.
t Time object with a time near the start of the desired scan (finds the scan
that starts closest time to the given time)
ax If specified as a two-element array of axes, the plots will be placed
in an already existing window, allowing reuse of the same window.
Otherwise, a new figure is created (if do_plot is True).
Returns a dictionary containing
Source name, Time, HA, Dec, RA offset and Dec offset
'''
import matplotlib.pyplot as plt
from matplotlib.transforms import Bbox
import read_idb
import dbutil as db
bl2ord = read_idb.bl2ord
tdate = t.iso.replace('-','')[:8]
fdir = '/data1/eovsa/fits/IDB/'+tdate+'/'
fdb = dump_tsys.rd_fdb(t)
scanidx, = np.where(fdb['PROJECTID'] == 'CALPNTCAL')
# Set offset coordinates appropriate to the type of PROJECTID found
if scanidx.size == 0:
# No CALPNTCAL scans, so search for CALPNT2M
scanidx, = np.where(fdb['PROJECTID'] == 'CALPNT2M')
if scanidx.size == 0:
print 'No CALPNTCAL or CALPNT2M project IDs found for date'+t.iso[:10]
return {}
else:
# Found CALPNT2M, so set offset coordinates to match calpnt2m.trj
rao = np.array([-5.00, -2.0, -1.0, -0.5, 0.00, 0.5, 1.0, 2.0])
deco = np.array([-2.0, -1.0, -0.5, 0.00, 0.5, 1.0, 2.0, 5.00])
pltfac = 10.
else:
# Found CALPNTCAL, so set offset coordinates to match calpnt.trj
rao = np.array([-1.00, -0.20, -0.10, -0.05, 0.00, 0.05, 0.10, 0.20])
deco = np.array([-0.20, -0.10, -0.05, 0.00, 0.05, 0.10, 0.20, 1.00])
pltfac = 1.
scans,sidx = np.unique(fdb['SCANID'][scanidx],return_index=True)
tlist = Time(fdb['ST_TS'][scanidx[sidx]].astype(float).astype(int),format='lv')
idx, = nearest_val_idx([t.jd],tlist.jd)
filelist = [fdir+f for f in fdb['FILE'][np.where(fdb['SCANID'] == scans[idx])]]
# Read pointing data (timerange t must be accurate)
out = read_idb.read_idb(filelist, navg=30)
# Determine wanted baselines with ant 14 from ant_str
idx = read_idb.p.ant_str2list(ant_str)
idx1 = idx[idx>7] # Ants > 8
idx2 = idx[idx<8] # Ants <= 8
# Determine parallactic angle for azel antennas (they are all the same, so find median).
# If 0 < abs(chi) < 30, use channel XX
# If 30 < abs(chi) < 60, use sum of channel XX and XY
# If 60 < abs(chi) < 90, use channel XY
midtime = Time((out['time'][0] + out['time'][-1])/2.,format='jd')
times, chi = db.get_chi(Time([midtime.lv+1,midtime.lv + 10],format='lv'))
abschi = abs(lobe(np.median(chi[0,0:8])))
if pltfac == 1.:
# Case of 27m antenna pointing
# Do appropriate sums over frequency, polarization and baseline
if abschi >= 0 and abschi < np.pi/6:
pntdata = np.sum(np.abs(np.sum(out['x'][bl2ord[idx,13],0,:,:48],1)),0) # Use only XX
elif abschi >= np.pi/6 and abschi < np.pi/3:
pntdata1 = np.sum(np.abs(np.sum(out['x'][bl2ord[idx1,13],0,:,:48],1)),0) # Use XX only for ants > 8
pntdata2 = np.sum(np.abs(np.sum(out['x'][bl2ord[idx2,13],:,:,:48],2)),0) # Use sum of XX and XY for ants <= 8
pntdata = pntdata1 + np.sum(pntdata2[np.array([0,2])],0)
else:
pntdata1 = np.sum(np.abs(np.sum(out['x'][bl2ord[idx1,13],0,:,:48],1)),0) # Use XX only for ants > 8
pntdata2 = np.sum(np.abs(np.sum(out['x'][bl2ord[idx2,13],2,:,:48],1)),0) # Use sum of XY for ants <= 8
pntdata = pntdata1 + pntdata2
# Measurements are 90 s long, hence 3 consecutive 30 s points, so do final
# sum over these
pntdata.shape = (16,3)
stdev = np.std(pntdata,1)
pntdata = np.sum(pntdata,1)
radat = pntdata[:8]
decdat = pntdata[8:]
plsqr, xr, yr = solpnt.gausfit(rao, radat)
plsqd, xd, yd = solpnt.gausfit(deco, decdat)
midtime = Time((out['time'][0] + out['time'][-1])/2.,format='jd')
if (do_plot):
if ax is None:
f, ax = plt.subplots(1,2)
f.set_size_inches(2.5,1.5,forward=True)
ax[0].errorbar(rao,radat,yerr=stdev[:8],fmt='.')
ax[0].plot(xr,yr)
ax[0].axvline(x=0,color='k')
ax[0].axvline(x=plsqr[1],linestyle='--')
ax[1].errorbar(deco,decdat,yerr=stdev[8:],fmt='.')
ax[1].plot(xd,yd)
ax[1].axvline(x=0,color='k')
ax[1].axvline(x=plsqd[1],linestyle='--')
for j in range(2):
ax[j].set_xlim(-0.3, 0.3)
ax[j].grid()
ax[0].text(0.05,0.9,'RAO :'+str(plsqr[1])[:5],transform=ax[0].transAxes)
ax[0].text(0.55,0.9,'FWHM:'+str(plsqr[2])[:5],transform=ax[0].transAxes)
ax[0].set_xlabel('RA Offset [deg]')
ax[1].text(0.05,0.9,'DECO:'+str(plsqd[1])[:5],transform=ax[1].transAxes)
ax[1].text(0.55,0.9,'FWHM:'+str(plsqd[2])[:5],transform=ax[1].transAxes)
ax[1].set_xlabel('Dec Offset [deg]')
if ax is None:
f.suptitle('Pointing on '+out['source']+' at '+midtime.iso)
else:
ax[0].set_title(out['source']+' at')
ax[1].set_title(midtime.iso[:16])
plt.pause(0.5)
return {'source':out['source'],'ha':out['ha'][24]*180./np.pi,'dec':out['dec']*180/np.pi,
'rao':plsqr[1],'deco':plsqd[1],'time':midtime,'antidx':13}
else:
# Case of 2m antenna pointing
# Do appropriate sum over frequency and polarization but not baseline
if abschi >= 0 and abschi < np.pi/6:
pntdata = np.abs(np.sum(out['x'][bl2ord[idx,13],0,:,:48],1)) # Use only XX
elif abschi >= np.pi/6 and abschi < np.pi/3:
pntdata1 = np.abs(np.sum(out['x'][bl2ord[idx1,13],0,:,:48],1)) # Use XX only for ants > 8
pntdata2 = np.abs(np.sum(out['x'][bl2ord[idx2,13],:,:,:48],2)) # Use sum of XX and XY for ants <= 8
pntdata = np.concatenate((pntdata1,np.sum(pntdata2[:,np.array([0,2])],1)),0)
else:
pntdata1 = np.abs(np.sum(out['x'][bl2ord[idx1,13],0,:,:48],1)) # Use XX only for ants > 8
pntdata2 = np.abs(np.sum(out['x'][bl2ord[idx2,13],2,:,:48],1)) # Use sum of XY for ants <= 8
pntdata = np.concatenate((pntdata1,pntdata2),0)
# Measurements are 90 s long, hence 3 consecutive 30 s points, so do final
# sum over these
pntdata.shape = (idx.size,16,3)
stdev = np.std(pntdata,2)
pntdata = np.sum(pntdata,2)
rao_fit = np.zeros(idx.size,float)
deco_fit = np.zeros(idx.size,float)
if (do_plot):
if ax is None:
f, ax = plt.subplots(1,2*idx.size)
f.set_size_inches(2.5*idx.size,1.5,forward=True)
plt.subplots_adjust(left=0.03, right=0.97, top=0.89, bottom=0.3, wspace=0.1, hspace=0.1)
for k in range(idx.size):
radat = pntdata[k,:8]
decdat = pntdata[k,8:]
plsqr, xr, yr = solpnt.gausfit(rao, radat)
plsqd, xd, yd = solpnt.gausfit(deco, decdat)
midtime = Time((out['time'][0] + out['time'][-1])/2.,format='jd')
if (do_plot):
ax[k*2+0].errorbar(rao,radat,yerr=stdev[k,:8],fmt='.')
ax[k*2+0].plot(xr,yr)
ax[k*2+0].axvline(x=0,color='k')
ax[k*2+0].axvline(x=plsqr[1],linestyle='--')
ax[k*2+1].errorbar(deco,decdat,yerr=stdev[k,8:],fmt='.')
ax[k*2+1].plot(xd,yd)
ax[k*2+1].axvline(x=0,color='k')
ax[k*2+1].axvline(x=plsqd[1],linestyle='--')
for j in range(2):
ax[k*2+j].set_xlim(-3.0, 3.0)
ax[k*2+j].grid()
ax[k*2+0].text(0.05,0.7,'RAO :'+str(plsqr[1])[:5],transform=ax[k*2+0].transAxes,fontsize=9)
ax[k*2+0].text(0.05,0.5,'FWHM:'+str(plsqr[2])[:5],transform=ax[k*2+0].transAxes,fontsize=9)
ax[k*2+0].set_xlabel('RAO [deg]',fontsize=9)
ax[k*2+1].text(-0.2,0.85,'Ant :'+str(idx[k]+1),transform=ax[k*2+1].transAxes,fontsize=9)
ax[k*2+1].text(0.05,0.7,'DECO:'+str(plsqd[1])[:5],transform=ax[k*2+1].transAxes,fontsize=9)
ax[k*2+1].text(0.05,0.5,'FWHM:'+str(plsqd[2])[:5],transform=ax[k*2+1].transAxes,fontsize=9)
ax[k*2+1].set_yticklabels([])
ax[k*2+1].set_xlabel('DECO [deg]',fontsize=9)
if k == idx.size-1:
ax[k+0].set_title(out['source']+' at',fontsize=9)
ax[k+1].set_title(midtime.iso[:16],fontsize=9)
# Adjust the pairs of subplots to group RA/Dec for each antenna together
ax[k*2+0].set_position(Bbox(ax[k*2+0].get_position().get_points() + np.array([[0.0025,0],[0.0025,0]])))
ax[k*2+1].set_position(Bbox(ax[k*2+1].get_position().get_points() + np.array([[-0.0025,0],[-0.0025,0]])))
# Set to the same amplitude scale
ymin = min(ax[k*2+0].get_ylim()[0],ax[k*2+1].get_ylim()[0])
ymax = max(ax[k*2+0].get_ylim()[1],ax[k*2+1].get_ylim()[1])
ax[k*2+0].set_ylim(ymin,ymax)
ax[k*2+1].set_ylim(ymin,ymax)
plt.pause(0.5)
rao_fit[k] = plsqr[1]
deco_fit[k] = plsqd[1]
return {'source':out['source'],'ha':out['ha'][24]*180./np.pi,'dec':out['dec']*180/np.pi,
'rao':rao_fit,'deco':deco_fit,'time':midtime,'antidx':idx}
def unrot(data, azeldict=None):
''' Apply the correction to differential feed rotation to data, and return
the corrected data.
Inputs:
data A dictionary returned by udb_util.py's readXdata().
azeldict The dictionary returned from get_sql_info(), or if None, the appropriate
get_sql_info() call is done internally.
Output:
cdata A dictionary with the phase-corrected data. Only the key
x is updated.
'''
import copy
from util import lobe
trange = Time(data['time'][[0,-1]],format='jd')
if azeldict is None:
azeldict = get_sql_info(trange)
chi = azeldict['ParallacticAngle'] # (nt, nant)
# Correct parallactic angle for equatorial mounts, relative to Ant14
for i in [8,9,10,12,13]:
chi[:,i] -= chi[:,13]
# Ensure that nearest valid parallactic angle is used for times in the data
good, = np.where(azeldict['ActualAzimuth'][0] != 0)
tidx = nearest_val_idx(data['time'],azeldict['Time'][good].jd)
# Read X-Y Delay phase from SQL database and get common frequencies
xml, buf = ch.read_cal(11,t=trange[0])
fghz = stateframe.extract(buf,xml['FGHz'])
good, = np.where(fghz != 0.)
fghz = fghz[good]
dph = stateframe.extract(buf,xml['XYphase'])
dph = dph[:,good]
fidx1, fidx2 = common_val_idx(data['fghz'],fghz,precision=4)
missing = np.setdiff1d(np.arange(len(data['fghz'])),fidx1)
nf, nbl, npol, nt = data['x'].shape
nf = len(fidx1)
# Correct data for X-Y delay phase
for k,bl in enumerate(get_bl_order()):
i, j = bl
if i < 14 and j < 14 and i != j:
a1 = lobe(dph[i,fidx2] - dph[j,fidx2])
a2 = -dph[j,fidx2] + np.pi/2
a3 = dph[i,fidx2] - np.pi/2
data['x'][fidx1,k,1] *= np.repeat(np.exp(1j*a1),nt).reshape(nf,nt)
data['x'][fidx1,k,2] *= np.repeat(np.exp(1j*a2),nt).reshape(nf,nt)
data['x'][fidx1,k,3] *= np.repeat(np.exp(1j*a3),nt).reshape(nf,nt)
# Correct data for differential feed rotation
cdata = copy.deepcopy(data)
for n in range(nt):
for k,bl in enumerate(get_bl_order()):
i, j = bl
if i < 14 and j < 14 and i != j:
dchi = chi[n,i] - chi[n,j]
cchi = np.cos(dchi)
schi = np.sin(dchi)
cdata['x'][:,k,0,n] = data['x'][:,k,0,n]*cchi + data['x'][:,k,3,n]*schi
cdata['x'][:,k,2,n] = data['x'][:,k,2,n]*cchi + data['x'][:,k,1,n]*schi
cdata['x'][:,k,3,n] = data['x'][:,k,3,n]*cchi - data['x'][:,k,0,n]*schi
cdata['x'][:,k,1,n] = data['x'][:,k,1,n]*cchi - data['x'][:,k,2,n]*schi
# Set flags for any missing frequencies (hopefully this also works when missing is np.array([]))
cdata[missing] = np.ma.masked
return cdata
def get_gain_state(trange, dt=None, relax=False):
''' Get all gain-state information for a given timerange. Returns a dictionary
with keys as follows:
times: A Time object containing the array of times, size (nt)
h1: The first HPol attenuator value for 15 antennas, size (nt, 15)
v1: The first VPol attenuator value for 15 antennas, size (nt, 15)
h2: The second HPol attenuator value for 15 antennas, size (nt, 15)
v2: The second VPol attenuator value for 15 antennas, size (nt, 15)
dcmattn: The base DCM attenuations for nbands x 15 antennas x 2 Poln, size (34 or 52,30)
The order is Ant1 H, Ant1 V, Ant2 H, Ant2 V, etc.
dcmoff: If DPPoffset-on is 0, this is None (meaning there are no changes to the
above base attenuations).
If DPPoffset-on is 1, then dcmoff is a table of offsets to the
base attenuation, size (nt, 50). The offset applies to all
antennas/polarizations.
Optional keywords:
dt Seconds between entries to read from SQL stateframe database.
If omitted, 1 s is assumed.
relax Used for gain of reference time, in case there are no SQL data for the
requested time. In that case it finds the data for the nearest later time.
'''
if dt is None:
tstart, tend = [str(i) for i in trange.lv]
else:
# Expand time by 1/2 of dt before and after
tstart = str(np.round(trange[0].lv - dt / 2))
tend = str(np.round(trange[1].lv + dt / 2))
cursor = db.get_cursor()
ver = db.find_table_version(cursor, trange[0].lv)
# Get front end attenuator states
# Attempt to solve the problem if there are no data
if relax:
# Special case of reference gain, where we want the first nt records after tstart, in case there
# are no data at time tstart
nt = int(float(tend) - float(tstart) - 1) * 15
query = 'select top '+str(nt)+' Timestamp,Ante_Fron_FEM_HPol_Atte_First,Ante_Fron_FEM_HPol_Atte_Second,' \
+'Ante_Fron_FEM_VPol_Atte_First,Ante_Fron_FEM_VPol_Atte_Second,Ante_Fron_FEM_Clockms from fV' \
+ver+'_vD15 where Timestamp >= '+tstart+' order by Timestamp'
else:
query = 'select Timestamp,Ante_Fron_FEM_HPol_Atte_First,Ante_Fron_FEM_HPol_Atte_Second,' \
+'Ante_Fron_FEM_VPol_Atte_First,Ante_Fron_FEM_VPol_Atte_Second,Ante_Fron_FEM_Clockms from fV' \
+ver+'_vD15 where Timestamp >= '+tstart+' and Timestamp < '+tend+' order by Timestamp'
#if dt:
# # If dt (seconds between measurements) is set, add appropriate SQL statement to query
# query += ' and (cast(Timestamp as bigint) % '+str(dt)+') = 0 '
data, msg = db.do_query(cursor, query)
if msg == 'Success':
if dt:
# If we want other than full cadence, get new array shapes and times
n = len(data['Timestamp']) # Original number of times
new_n = (
n / 15 / dt
) * 15 * dt # Truncated number of times equally divisible by dt
new_shape = (n / 15 / dt, dt, 15) # New shape of truncated arrays
times = Time(data['Timestamp'][:new_n].astype('int')[::15 * dt],
format='lv')
else:
times = Time(data['Timestamp'].astype('int')[::15], format='lv')
# Change tstart and tend to correspond to actual times from SQL
tstart, tend = [str(i) for i in times[[0, -1]].lv]
h1 = data['Ante_Fron_FEM_HPol_Atte_First']
h2 = data['Ante_Fron_FEM_HPol_Atte_Second']
v1 = data['Ante_Fron_FEM_VPol_Atte_First']
v2 = data['Ante_Fron_FEM_VPol_Atte_Second']
ms = data['Ante_Fron_FEM_Clockms']
nt = len(h1) / 15
h1.shape = (nt, 15)
h2.shape = (nt, 15)
v1.shape = (nt, 15)
v2.shape = (nt, 15)
ms.shape = (nt, 15)
# Find any entries for which Clockms is zero, which indicates where no
# gain-state measurement is available.
for i in range(15):
bad, = np.where(ms[:, i] == 0)
if bad.size != 0 and bad.size != nt:
# Find nearest adjacent good value
good, = np.where(ms[:, i] != 0)
idx = nearest_val_idx(bad, good)
h1[bad, i] = h1[good[idx], i]
h2[bad, i] = h2[good[idx], i]
v1[bad, i] = v1[good[idx], i]
v2[bad, i] = v2[good[idx], i]
if dt:
# If we want other than full cadence, find mean over dt measurements
h1 = np.mean(h1[:new_n / 15].reshape(new_shape), 1)
h2 = np.mean(h2[:new_n / 15].reshape(new_shape), 1)
v1 = np.mean(v1[:new_n / 15].reshape(new_shape), 1)
v2 = np.mean(v2[:new_n / 15].reshape(new_shape), 1)
# Put results in canonical order [nant, nt]
h1 = h1.T
h2 = h2.T
v1 = v1.T
v2 = v2.T
else:
print 'Error reading FEM attenuations:', msg
return {}
# Get back end attenuator states
xml, buf = ch.read_cal(2, t=trange[0])
dcmattn = stf.extract(buf, xml['Attenuation'])
nbands = dcmattn.shape[0]
dcmattn.shape = (nbands, 15, 2)
# Put into canonical order [nant, npol, nband]
dcmattn = np.moveaxis(dcmattn, 0, 2)
# See if DPP offset is enabled
query = 'select Timestamp,DPPoffsetattn_on from fV' \
+ver+'_vD1 where Timestamp >= '+tstart+' and Timestamp <= '+tend+'order by Timestamp'
data, msg = db.do_query(cursor, query)
if msg == 'Success':
dppon = data['DPPoffsetattn_on']
if np.where(dppon > 0)[0].size == 0:
dcm_off = None
else:
query = 'select Timestamp,DCMoffset_attn from fV' \
+ver+'_vD50 where Timestamp >= '+tstart+' and Timestamp <= '+tend
#if dt:
# # If dt (seconds between measurements) is set, add appropriate SQL statement to query
# query += ' and (cast(Timestamp as bigint) % '+str(dt)+') = 0 '
query += ' order by Timestamp'
data, msg = db.do_query(cursor, query)
if msg == 'Success':
otimes = Time(data['Timestamp'].astype('int')[::15],
format='lv')
dcmoff = data['DCMoffset_attn']
dcmoff.shape = (nt, 50)
# We now have a time-history of offsets, at least some of which are non-zero.
# Offsets by slot number do us no good, so we need to translate to band number.
# Get fseqfile name at mean of timerange, from stateframe SQL database
fseqfile = get_fseqfile(
Time(int(np.mean(trange.lv)), format='lv'))
if fseqfile is None:
print 'Error: No active fseq file.'
dcm_off = None
else:
# Get fseqfile from ACC and return bandlist
bandlist = fseqfile2bandlist(fseqfile)
nbands = len(bandlist)
# Use bandlist to covert nt x 50 array to nt x nbands array of DCM attn offsets
# Note that this assumes DCM offset is the same for any multiply-sampled bands
# in the sequence.
dcm_off = np.zeros((nt, nbands), float)
dcm_off[:, bandlist - 1] = dcmoff
# Put into canonical order [nband, nt]
dcm_off = dcm_off.T
if dt:
# If we want other than full cadence, find mean over dt measurements
new_nt = len(times)
dcm_off = dcm_off[:, :new_nt * dt]
dcm_off.shape = (nbands, dt, new_nt)
dcm_off = np.mean(dcm_off, 1)
else:
print 'Error reading DCM attenuations:', msg
dcm_off = None
else:
print 'Error reading DPPon state:', msg
dcm_off = None
cursor.close()
return {
'times': times,
'h1': h1,
'v1': v1,
'h2': h2,
'v2': v2,
'dcmattn': dcmattn,
'dcmoff': dcm_off
}
def apply_fem_level(data, skycal=None, gctime=None):
''' Applys the FEM level corrections to the given data dictionary.
Inputs:
data A dictionary such as that returned by read_idb().
skycal A dictionary returned by skycal_anal() in calibration.py. This is
used to subtract a small "receiver background" before scaling for
fem level, and then adding it back.
gctime A Time() object whose date specifies which GAINCALTEST
measurements to use. If omitted, the date of the data
is used.
Output:
cdata A dictionary with the level-corrected data. The keys
p, x, p2, and a are all updated.
'''
from util import common_val_idx, nearest_val_idx
import attncal as ac
import copy
# Get timerange from data
trange = Time([data['time'][0], data['time'][-1]], format='jd')
if gctime is None:
gctime = trange[0]
# Get time cadence
dt = np.int(
np.round(np.median(data['time'][1:] - data['time'][:-1]) * 86400))
if dt == 1: dt = None
# Get the FEM levels of the requested timerange
src_lev = get_fem_level(trange,
dt) # solar gain state for timerange of file
nf = len(data['fghz'])
nt = len(src_lev['times'])
# First attempt to read from the SQL database. If that fails, read from the IDB file itself
try:
attn = ac.read_attncal(gctime)[0] # Attn from SQL
if (gctime.mjd - attn['time'].mjd) > 1:
# SQL entry is too old, so analyze the GAINCALTEST
attn = ac.get_attncal(
gctime
)[0] # Attn measured by GAINCALTEST (returns a list, but use first, generally only, one)
ch.fem_attn_val2sql([attn]) # Go ahead and write it to SQL
except:
attn = ac.get_attncal(
gctime
)[0] # Attn measured by GAINCALTEST (returns a list, but use first, generally only, one)
antgain = np.zeros((15, 2, nf, nt),
np.float32) # Antenna-based gains [dB] vs. frequency
# Find common frequencies of attn with data
idx1, idx2 = common_val_idx(data['fghz'], attn['fghz'], precision=4)
# Currently, GAINCALTEST measures 8 levels of attenuation (16 dB). I assumed this would be enough,
# but the flare of 2017-09-10 actually went to 10 levels (20 dB), so we have no choice but to extend
# to higher levels using only the nominal, 2 dB steps above the 8th level. This part of the code
# extends to the maximum 16 levels.
a = np.zeros((16, 13, 2, nf), float) # Extend attenuation to 14 levels
a[1:9, :, :, idx1] = attn[
'attn'][:, :13, :,
idx2] # Use GAINCALTEST results in levels 1-9 (bottom level is 0dB)
for i in range(8, 15):
# Extend to levels 9-15 by adding 2 dB to each previous level
a[i + 1] = a[i] + 2.
a[15] = 62. # Level 15 means 62 dB have been inserted.
if dt:
# For this case, src_lev is an array of dictionaries where keys are levels and
# values are the proportion of that level for the given integration
for i in range(13):
for k, j in enumerate(idx1):
for m in range(nt):
for lev, prop in src_lev['hlev'][i, m].items():
antgain[i, 0, j, m] += prop * a[lev, i, 0, idx2[k]]
for lev, prop in src_lev['vlev'][i, m].items():
antgain[i, 1, j, m] += prop * a[lev, i, 1, idx2[k]]
else:
# For this case, src_lev is just an array of levels
for i in range(13):
for k, j in enumerate(idx1):
antgain[i, 0, j] = a[src_lev['hlev'][i], i, 0, idx2[k]]
antgain[i, 1, j] = a[src_lev['vlev'][i], i, 1, idx2[k]]
cdata = copy.deepcopy(data)
blgain = np.zeros((120, 4, nf, nt),
float) # Baseline-based gains vs. frequency
for i in range(14):
for j in range(i + 1, 15):
blgain[ri.bl2ord[i, j],
0] = 10**((antgain[i, 0] + antgain[j, 0]) / 20.)
blgain[ri.bl2ord[i, j],
1] = 10**((antgain[i, 1] + antgain[j, 1]) / 20.)
blgain[ri.bl2ord[i, j],
2] = 10**((antgain[i, 0] + antgain[j, 1]) / 20.)
blgain[ri.bl2ord[i, j],
3] = 10**((antgain[i, 1] + antgain[j, 0]) / 20.)
antgainf = 10**(antgain / 10.)
#idx1, idx2 = common_val_idx(data['time'],src_gs['times'].jd)
idx = nearest_val_idx(data['time'], src_lev['times'].jd)
# Apply corrections (some times may be eliminated from the data)
# Correct the cross-correlation data
cdata['x'] *= blgain[:, :, :, idx]
# If a skycal dictionary exists, subtract receiver noise before scaling
# NB: This will break SK!
if skycal:
sna, snp, snf = skycal['rcvr_bgd'].shape
bgd = skycal['rcvr_bgd'].repeat(nt).reshape((sna, snp, snf, nt))
bgd_auto = skycal['rcvr_bgd_auto'].repeat(nt).reshape(
(sna, snp, snf, nt))
cdata['p'][:13] -= bgd[:, :, :, idx]
cdata['a'][:13, :2] -= bgd_auto[:, :, :, idx]
# Correct the power,
cdata['p'][:15] *= antgainf[:, :, :, idx]
# Correct the autocorrelation
cdata['a'][:15, :2] *= antgainf[:, :, :, idx]
# If a skycal dictionary exists, add back the receiver noise
#if skycal:
# cdata['p'][:13] += bgd[:,:,:,idx]
# cdata['a'][:13,:2] += bgd_auto[:,:,:,idx]
cross_fac = np.sqrt(antgainf[:, 0] * antgainf[:, 1])
cdata['a'][:15, 2] *= cross_fac[:, :, idx]
cdata['a'][:15, 3] *= cross_fac[:, :, idx]
# Correct the power-squared -- this should preserve SK
cdata['p2'][:15] *= antgainf[:, :, :, idx]**2
# Remove any uncorrected times before returning
#cdata['time'] = cdata['time'][idx1]
#cdata['p'] = cdata['p'][:,:,:,idx1]
#cdata['a'] = cdata['a'][:,:,:,idx1]
#cdata['p2'] = cdata['p2'][:,:,:,idx1]
#cdata['ha'] = cdata['ha'][idx1]
#cdata['m'] = cdata['m'][:,:,:,idx1]
return cdata
def autocorrect(out, ant_str='ant1-13', brange=[0, 300]):
nt = len(out['time'])
nf = len(out['fghz'])
pfac1 = (out['p'][:, :, :, :-1] -
out['p'][:, :, :, 1:]) / out['p'][:, :, :, :-1]
trange = Time(out['time'][[0, -1]], format='jd')
src_lev = gc.get_fem_level(trange) # Read FEM levels from SQL
# Match times with data
tidx = nearest_val_idx(out['time'], src_lev['times'].jd)
# Find attenuation changes
for ant in range(13):
for pol in range(2):
if pol == 0:
lev = src_lev['hlev'][ant, tidx]
else:
lev = src_lev['vlev'][ant, tidx]
jidx, = np.where(abs(lev[:-1] - lev[1:]) == 1)
for freq in range(nf):
idx, = np.where(
np.logical_and(
abs(pfac1[ant, pol, freq]) > 0.05,
abs(pfac1[ant, pol, freq]) < 0.95))
for i in range(len(idx - 1)):
if idx[i] in jidx or idx[i] in jidx - 1:
out['p'][ant, pol, freq, idx[i] +
1:] /= (1 - pfac1[ant, pol, freq, idx[i]])
# Time of total power calibration is 20 UT on the date given
tptime = Time(np.floor(trange[0].mjd) + 20. / 24., format='mjd')
calfac = pc.get_calfac(tptime)
tpcalfac = calfac['tpcalfac']
tpoffsun = calfac['tpoffsun']
hlev = src_lev['hlev'][:13, 0]
vlev = src_lev['vlev'][:13, 0]
attn_dict = ac.read_attncal(trange[0])[0] # Read GCAL attn from SQL
attn = np.zeros((13, 2, nf))
for i in range(13):
attn[i, 0] = attn_dict['attn'][hlev[i], 0, 0]
attn[i, 1] = attn_dict['attn'][vlev[i], 0, 1]
print 'Ant', i + 1, attn[i, 0, 20], attn[i, 1, 20]
attnfac = 10**(attn / 10.)
for i in range(13):
print attnfac[i, 0, 20], attnfac[i, 1, 20]
for i in range(nt):
out['p'][:13, :, :,
i] = (out['p'][:13, :, :, i] * attnfac - tpoffsun) * tpcalfac
antlist = ant_str2list(ant_str)
bg = np.zeros_like(out['p'])
# Subtract background for each antenna/polarization
for ant in antlist:
for pol in range(2):
bg[ant,
pol] = np.median(out['p'][ant, pol, :, brange[0]:brange[1]],
1).repeat(nt).reshape(nf, nt)
#out['p'][ant,pol] -= bg
# Form median over antennas/pols
med = np.mean(np.median((out['p'] - bg)[antlist], 0), 0)
# Do background subtraction once more for good measure
bgd = np.median(med[:, brange[0]:brange[1]], 1).repeat(nt).reshape(nf, nt)
med -= bgd
pdata = np.log10(med)
f, ax = plt.subplots(1, 1)
vmax = np.median(np.nanmax(pdata, 1))
im = ax.pcolormesh(Time(out['time'], format='jd').plot_date,
out['fghz'],
pdata,
vmin=1,
vmax=vmax)
ax.axvspan(Time(out['time'][brange[0]], format='jd').plot_date,
Time(out['time'][brange[1]], format='jd').plot_date,
color='w',
alpha=0.3)
cbar = plt.colorbar(im, ax=ax)
cbar.set_label('Log Flux Density [sfu]')
ax.xaxis_date()
ax.xaxis.set_major_formatter(DateFormatter("%H:%M"))
ax.set_ylim(out['fghz'][0], out['fghz'][-1])
ax.set_xlabel('Time [UT]')
ax.set_ylabel('Frequency [GHz]')
return {'caldata': out, 'med_sub': med, 'bgd': bg}
