def SaveLoRX(self):
# Send saved Ant 14 delays as Ant 15 to the SQL database and the ACC (if the data source is not 'Simulation')
# Note, ONLY Ant 14 delay is changed, and it is ascribed to Ant 15. No other delay changes are made.
if self.data_source == 'Data':
if (Time.now().mjd - self.time.mjd) > 1.0:
question = "Warning: Data more than a day old. Are you sure you want to save delays to SQL and ACC?"
else:
question = "Save delays to SQL and ACC?"
import cal_header as ch
delays = self.delays[0] - self.delays
delays = np.append(delays,self.delays[0])
# Have to change the sign of Ant 14 Y-X delay, hence the minus sign
xydelays = np.append(self.xydelays,-float(self.dla14.get()))
# Do not change delays where both delays and xydelays are zero
# which is taken as a missing antenna
bad, = np.where(self.delays == 0)
bad2, = np.where(self.xydelays == 0)
idx1,idx2 = common_val_idx(bad,bad2)
delays[bad[idx1]] = 0.0
# Check that the delays to all antennas except Ant 14 are zero, or give warning if not
for i in range(13):
if delays[i] != 0.0:
question = "Some Ant1-13 delays are not 0, but will NOT be updated. Save anyway?"
break
if xydelays[i] != 0.0:
question = "Some Ant1-13 delays are not 0, but will NOT be updated. Save anyway?"
break
if askyesno("Write Delays",question):
# All Y-X delays need a sign flip, hence the minus sign
ch.dla_update2sql(delays,-xydelays,lorx=True)
#ch.dla_update2sql(-delays,xydelays) # 300 MHz design uses flipped signs!
ch.dla_censql2table()
def show(self):
delays_str = ' '
xydelays_str = ' '
delays = np.append(self.delays[0] - self.delays,self.delays[0])
# Do not change delays where both delays and xydelays are zero
# which is taken as a missing antenna
bad, = np.where(self.delays == 0)
bad2, = np.where(self.xydelays == 0)
idx1,idx2 = common_val_idx(bad,bad2)
delays[bad[idx1]] = 0.0
self.label1.configure(text='Ant 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14')
fmt = '{:5.1f},'*14
delays_str = fmt.format(*delays)
self.label2.configure(text=' '+delays_str[:-1])
xydelays = np.append(self.xydelays,-float(self.dla14.get()))
xydelays_str = fmt.format(*xydelays)
self.label3.configure(text=' '+xydelays_str[:-1])
def __init__(self,trange):
''' Create the object for the specified timerange specified by the
2-element Time() trange. The timerange is used to create a list
of Miriad database files to read, and the data are read.
'''
# Read data, assuming 16-element correlator if after 2016 May 1
if trange[0].lv > 3544905600.0:
out = dump_tsys.rd_miriad_tsys_16(trange)
else:
out = dump_tsys.rd_miriad_tsys(trange)
nant, npol, nf, nt = out['tsys'].shape
self.xdata = out['tsys'][:,0,:,:]
self.ydata = out['tsys'][:,1,:,:]
self.fghz = out['fghz']
self.time = Time(out['ut_mjd'],format='mjd')
self.tidx = [0,len(self.time)]
# Read calibration
fghz, self.calfac, self.offsun = offline.read_dbcalfac(trange[0])
# Make sure frequencies in data and calibration agree
calidx, dataidx = common_val_idx((fghz*1000).astype('int'),(self.fghz*1000).astype('int'))
self.bidx = [0,100]
self.fghz = self.fghz[dataidx]
self.xdata = self.xdata[:,dataidx,:]
self.ydata = self.ydata[:,dataidx,:]
if self.calfac is not None:
# Select frequencies and swap axes to put into standard form
self.calfac = np.rollaxis(self.calfac[:,calidx,:],2)
self.offsun = np.rollaxis(self.offsun[:,calidx,:],2)
fghz = fghz[calidx]
# Set frequency index range (fidx) to default to show only frequencies > 2.5 GHz
lowf, = np.where(self.fghz > 2.5)
self.fidx = [lowf[0],len(self.fghz)]
self.drange = [None,None]
self.antlist = range(nant)
self.cbar = True
self.showants = range(nant)
self.domedian = True
self.docal = True
self.dolog = False
self.dosub = True
self.ax = None
self.version = __version__
def Save(self):
# Send saved delays to the SQL database and the ACC (if the data source is not 'Simulation')
if self.data_source == 'Data':
if (Time.now().mjd - self.time.mjd) > 1.0:
question = "Warning: Data more than a day old. Are you sure you want to save delays to SQL and ACC?"
else:
question = "Save delays to SQL and ACC?"
import cal_header as ch
# Calculate delays relative to Ant 1 and tack Ant1-14 delay at end
delays = self.delays[0] - self.delays
delays = np.append(delays,self.delays[0])
# Have to change the sign of Ant 14 Y-X delay, hence the minus sign
xydelays = np.append(self.xydelays,-float(self.dla14.get()))
# Do not change delays where both delays and xydelays are zero
# which is taken as a missing antenna
bad, = np.where(self.delays == 0)
bad2, = np.where(self.xydelays == 0)
idx1,idx2 = common_val_idx(bad,bad2)
delays[bad[idx1]] = 0.0
if askyesno("Write Delays",question):
# All Y-X delays need a sign flip, hence the minus sign
ch.dla_update2sql(delays,-xydelays)
#ch.dla_update2sql(-delays,xydelays) # 300 MHz design uses flipped signs!
ch.dla_censql2table()
def unrot(data, azeldict=None):
''' Apply the correction to differential feed rotation to data, and return
the corrected data.
Inputs:
data A dictionary returned by udb_util.py's readXdata().
azeldict The dictionary returned from get_sql_info(), or if None, the appropriate
get_sql_info() call is done internally.
Output:
cdata A dictionary with the phase-corrected data. Only the key
x is updated.
'''
import copy
from util import lobe
trange = Time(data['time'][[0,-1]],format='jd')
if azeldict is None:
azeldict = get_sql_info(trange)
chi = azeldict['ParallacticAngle'] # (nt, nant)
# Correct parallactic angle for equatorial mounts, relative to Ant14
for i in [8,9,10,12,13]:
chi[:,i] -= chi[:,13]
# Ensure that nearest valid parallactic angle is used for times in the data
good, = np.where(azeldict['ActualAzimuth'][0] != 0)
tidx = nearest_val_idx(data['time'],azeldict['Time'][good].jd)
# Read X-Y Delay phase from SQL database and get common frequencies
xml, buf = ch.read_cal(11,t=trange[0])
fghz = stateframe.extract(buf,xml['FGHz'])
good, = np.where(fghz != 0.)
fghz = fghz[good]
dph = stateframe.extract(buf,xml['XYphase'])
dph = dph[:,good]
fidx1, fidx2 = common_val_idx(data['fghz'],fghz,precision=4)
missing = np.setdiff1d(np.arange(len(data['fghz'])),fidx1)
nf, nbl, npol, nt = data['x'].shape
nf = len(fidx1)
# Correct data for X-Y delay phase
for k,bl in enumerate(get_bl_order()):
i, j = bl
if i < 14 and j < 14 and i != j:
a1 = lobe(dph[i,fidx2] - dph[j,fidx2])
a2 = -dph[j,fidx2] + np.pi/2
a3 = dph[i,fidx2] - np.pi/2
data['x'][fidx1,k,1] *= np.repeat(np.exp(1j*a1),nt).reshape(nf,nt)
data['x'][fidx1,k,2] *= np.repeat(np.exp(1j*a2),nt).reshape(nf,nt)
data['x'][fidx1,k,3] *= np.repeat(np.exp(1j*a3),nt).reshape(nf,nt)
# Correct data for differential feed rotation
cdata = copy.deepcopy(data)
for n in range(nt):
for k,bl in enumerate(get_bl_order()):
i, j = bl
if i < 14 and j < 14 and i != j:
dchi = chi[n,i] - chi[n,j]
cchi = np.cos(dchi)
schi = np.sin(dchi)
cdata['x'][:,k,0,n] = data['x'][:,k,0,n]*cchi + data['x'][:,k,3,n]*schi
cdata['x'][:,k,2,n] = data['x'][:,k,2,n]*cchi + data['x'][:,k,1,n]*schi
cdata['x'][:,k,3,n] = data['x'][:,k,3,n]*cchi - data['x'][:,k,0,n]*schi
cdata['x'][:,k,1,n] = data['x'][:,k,1,n]*cchi - data['x'][:,k,2,n]*schi
# Set flags for any missing frequencies (hopefully this also works when missing is np.array([]))
cdata[missing] = np.ma.masked
return cdata
def get_reverseattn(trange_gaincal, trange_other=None, first_attn_base=5, second_attn_base=3, corrected_attns=False):
# This function finds and return the tsys, the noise corrected tsys, and the noise and attenuation
# tsys. The first parameter it takes should be a GAINCALTEST trange, which may be located with find_gaincal,
# The second parameter it takes should be a FEATTNTEST trange, which may be located with find_gaincal as
# well. It may or may not take a third parameter. Any form of file that was recorded by the system from
# from the antennas may be inserted here, whether a flare or just quiet sun data.
#
# PLEASE NOTE: ANY trange with data may be used as "trange_gaincal", use a trange from a GAINCALTEST and the other
# file as "trange_other" if you want the noise to be calibrated from the GAINCALTEST file, which will most likely
# be more recent than the FEATTNTEST file it would otherwise take the noise from.
tsys1, res1, noise_level, idx1a, idx3a, marker1, trange_feattncal, trange_feattncal2 = attn_noises(trange_gaincal)
if corrected_attns == True:
all_attns_avg = get_all_attns(trange_feattncal, test='FEATTNTEST', from_corrected_attns=True)
all_attns_avg1 = get_all_attns(trange_feattncal2, test='FEATTNTEST2', from_corrected_attns=True)
else:
if corrected_attns == False:
all_attns_avg = get_all_avg_attns(trange_feattncal, test='FEATTNTEST')
all_attns_avg1 = get_all_avg_attns(trange_feattncal2, test='FEATTNTEST2')
if trange_other == None:
tsys = tsys1
res = res1
idx1 = idx1a
idx3 = idx3a
marker = marker1
else:
s = sp.Spectrogram(trange_other)
s.docal = False
s.dosub = False
s.domedian = False
tsys, std = s.get_data()
cursor = dbutil.get_cursor()
res, msg = dbutil.do_query(cursor,'select Timestamp,Ante_Fron_FEM_HPol_Atte_First,Ante_Fron_FEM_HPol_Atte_Second from fV54_vD15 where Timestamp between '+str(trange_other[0].lv)+' and '+str(trange_other[1].lv))
cursor.close()
idx1, idx2 = util.common_val_idx(res['Timestamp'][0::15].astype('int'), (s.time.lv+0.5).astype('int'))
marker = -1
while idx1[-1] > idx2[-1]:
idx1 = np.delete(idx1, -1)
marker += 1
tsys = tsys[:, :, :, idx1]
idx3, idx4 = util.common_val_idx(res['Timestamp'].astype('int'), (s.time.lv+0.5).astype('int'))
res['Timestamp'] = res['Timestamp'][idx3]
if noise_level.shape[2] < tsys.shape[2]:
freqs_for_range = noise_level.shape[2]
else:
freqs_for_range = tsys.shape[2]
tsys_noise_corrected = []
for ant in range(tsys.shape[0]):
pol_corrected = []
for pol in range(tsys.shape[1]):
freq_corrected = []
for freq in range(freqs_for_range):
index_corrected = []
for index in range(tsys.shape[3]):
index_corrected.append(tsys[ant, pol, freq, index] - noise_level[ant, pol, freq])
freq_corrected.append(index_corrected)
pol_corrected.append(freq_corrected)
tsys_noise_corrected.append(pol_corrected)
tsys_noise_corrected= np.array(tsys_noise_corrected)
freqloopslist = [tsys_noise_corrected.shape[2], all_attns_avg.shape[3], all_attns_avg1.shape[3]]
freqloops = min(freqloopslist)
if tsys.shape[3] < len(res['Ante_Fron_FEM_HPol_Atte_Second'][0::15]):
indexloops = tsys.shape[3]
else:
if tsys.shape[3] >= len(res['Ante_Fron_FEM_HPol_Atte_Second'][0::15]):
indexloops = len(res['Ante_Fron_FEM_HPol_Atte_Second'][0::15])-1
if tsys.shape[3] < len(res['Ante_Fron_FEM_HPol_Atte_First'][0::15]):
indexloops1 = tsys.shape[3]
else:
if tsys.shape[3] >= len(res['Ante_Fron_FEM_HPol_Atte_First'][0::15]):
indexloops1 = len(res['Ante_Fron_FEM_HPol_Atte_First'][0::15])-1
idxstart = marker + (15-8)
xory = ['x' , 'y']
ant_postcorrected = []
for ant in range(tsys.shape[0]):
pol_postcorrected = []
for pol in range(tsys.shape[1]):
freq_postcorrected = []
for freq in range(freqloops):
indices_postcorrected = []
for indx in range(indexloops):
testlevel = res['Ante_Fron_FEM_HPol_Atte_Second'][ant::15][indx+idxstart]
if 0 <= testlevel <= 31:
pass
else:
print 'Problem with the attenuation of antenna ' + str(ant) + xory[pol] + ' at frequency channel ' + str(freq) + ' and time index ' + str(indx) + '. The attenuation is showing: ' + str(testlevel)
testlevel = 0
indices_postcorrected.append(10**((all_attns_avg[testlevel, ant, pol, freq]-all_attns_avg[second_attn_base, ant, pol, freq])/10)*tsys_noise_corrected[ant, pol, freq, indx])
indices_postcorrected1 = []
for indx in range(indexloops1):
testlevel = res['Ante_Fron_FEM_HPol_Atte_First'][ant::15][indx+idxstart]
if 0 <= testlevel <= 31:
pass
else:
print 'Problem with the attenuation of antenna ' + str(ant) + xory[pol] + ' at frequency channel ' + str(freq) + ' and time index ' + str(indx) + '. The attenuation is showing: ' + str(testlevel)
testlevel = 0
indices_postcorrected1.append(10**((all_attns_avg1[testlevel, ant, pol, freq]-all_attns_avg1[first_attn_base, ant, pol, freq])/10)*indices_postcorrected[indx])
freq_postcorrected.append(indices_postcorrected1)
pol_postcorrected.append(freq_postcorrected)
ant_postcorrected.append(pol_postcorrected)
tsys_attn_noise_corrected = np.array(ant_postcorrected)
return tsys_attn_noise_corrected, tsys_noise_corrected, tsys
def get_state_idx(trange, cycles=4, attenuator=1):
#This program creates an array of shape (6, 4) which contains the
# times in which each attenuator is in each state. 6 attenuators,
# 4 cycles.
firstorsecond = ['First', 'Second']
s = sp.Spectrogram(trange)
s.docal = False
s.dosub = False
s.domedian = False
cursor = dbutil.get_cursor()
res, msg = dbutil.do_query(cursor,'select Timestamp,Ante_Fron_FEM_HPol_Atte_First,Ante_Fron_FEM_HPol_Atte_Second from fV54_vD15 where Timestamp between '+str(trange[0].lv)+' and '+str(trange[1].lv))
cursor.close()
if msg == 'Success':
antlist = []
for i in [0, 1, 2, 4, 8, 16]:
statelist = []
for j in range(15):
state, = np.where(np.logical_and(res['Ante_Fron_FEM_HPol_Atte_' + firstorsecond[attenuator]][j::15].astype('int') == i,res['Ante_Fron_FEM_HPol_Atte_' + firstorsecond[attenuator-1]][j::15].astype('int') != 0))
statelist.append(state)
statelist = np.array(statelist)
antlist.append(statelist)
states = np.array(antlist)
states = np.rollaxis(states, 1)
for i in range(15):
for j in range(6):
states[i, j] = res['Timestamp'][i::15][states[i, j]]
else:
print 'failure'
return None
time_array = (s.time.lv+0.001).astype('int')
time_list = list(time_array)
attns = ['0', '1', '2', '4', '8', '16']
common_list = []
for j in range(6):
# Antenna 1 is used as the reference antenna here.
# Earlier versions had only indices which were shared
# for attenuations AND antennas, but because of a
# small timing error that occur between antennas
# during the scan itself, this older version would
# fail sometimes.
i1, i2 = util.common_val_idx(time_array,states[0,j])
if i1.shape == i2.shape:
common_ant_list = i2
else:
print 'There is a problem with antenna '+str(i)+' at attenuation '+attns[j]
common_list.append(common_ant_list)
final_indices = []
final_indices1 = []
for i in range(6):
index_list = []
for indxs in common_list[i]:
try:
index_list.append(time_list.index(states[0,i][indxs]))
except:
pass
final_indices1.append(index_list)
for i in range(6):
indices_array = np.array(final_indices1[i])
final_indices.append(indices_array)
final_indices = np.array(final_indices)
rolled_indices = []
for i in range(6):
rolled = np.roll(final_indices[i], -1)
rolled_indices.append(rolled)
subtracted_list = []
for j in range(6):
subtracted_list.append(rolled_indices[j] - final_indices[j])
break_lists = []
for k in range(6):
break_list = []
for indx in range(subtracted_list[k].shape[0]):
if np.absolute(subtracted_list[k][indx]) <= 2:
break_list.append(indx)
else:
break_list.append(-1)
break_lists.append(break_list)
for i in range(6):
for indx in range(int(len(break_lists[i]))-1):
try:
if break_lists[i][indx] == break_lists[i][indx-1]:
break_lists[i].pop(indx)
except:
pass
break_list = []
for j in range(6):
breaklist = np.array(break_lists[j])
break_list.append(breaklist)
break_spots = []
for i in range(6):
try:
break_spot = []
for indx in range(len(break_list[i])):
if break_list[i][indx] == -1:
break_spot.append(indx)
break_spots.append(break_spot)
except:
pass
split_lists = []
for k in range(6):
steps_list = [break_list[k][0:break_spots[k][0]]]
for j in range(cycles-1):
try:
steps_list.append(break_list[k][1 + break_spots[k][j]:break_spots[k][j+1]])
except:
pass
split_lists.append(steps_list)
split_lists = np.array(split_lists)
final_grouped_indices = []
for i in range(6):
grouped_indices = []
for j in range(cycles):
try:
indices_ = []
for indxs in split_lists[i][j]:
indices_.append(rolled_indices[i][indxs])
grouped_indices.append(indices_)
except:
pass
final_grouped_indices.append(grouped_indices)
final_grouped_indices = np.array(final_grouped_indices)
for i in range(6):
for j in range(cycles):
try:
for k in range(1,int(len(final_grouped_indices[i][j]))-1):
try:
for m in range(len(final_ped_indices[i][j])):
if (final_grouped_indices[i][j][k-1] + 3) <= final_grouped_indices[i][j][k]:
final_grouped_indices[i][j].pop(k-1)
if (final_grouped_indices[i][j][k+1]-3) >= final_grouped_indices[i][j][k]:
final_grouped_indices[i][j].pop(k+1)
except:
pass
except:
pass
return final_grouped_indices, res
def attn_noises(trange_gaincal):
# This function is used with "get_reverseattn." It find and returns background noise, and returns the "res"
# from dbutil.do_query and returns the "tsys" from spectrogram_fit and get_data().
# The first parameter it takes should be a GAINCALTEST trange, which may be located with find_gaincal,
# The second parameter it takes should be a FEATTNTEST trange, which may be located with find_gaincal as
# well.
#
# PLEASE NOTE: ANY trange with data may be used as "trange_gaincal", use a trange from a GAINCALTEST and the other
# file as "trange_other" if you want the noise to be calibrated from the GAINCALTEST file, which will most likely
# be more recent than the FEATTNTEST file it would otherwise take the noise from.
s = sp.Spectrogram(trange_gaincal)
s.docal = False
s.dosub = False
s.domedian = False
tsys, std = s.get_data()
trange_feattncal = find_gaincal()
if type(trange_feattncal) == list:
trange_feattncal = trange_feattncal[-1]
else:
pass
trange_feattncal2 = find_gaincal(t = Time('2015-07-21 00:00'), scan_length=5, findwhat='FEATTNTEST2')
if type(trange_feattncal2) == list:
trange_feattncal2 = trange_feattncal2[-1]
else:
pass
ratios, calfilenoise = show_dB_ratio(trange_feattncal)
ratios1, calfilenoise1 = show_dB_ratio(trange_feattncal2, test='FEATTNTEST2')
cursor = dbutil.get_cursor()
res, msg = dbutil.do_query(cursor,'select Timestamp,Ante_Fron_FEM_HPol_Atte_First,Ante_Fron_FEM_HPol_Atte_Second from fV54_vD15 where Timestamp between '+str(trange_gaincal[0].lv)+' and '+str(trange_gaincal[1].lv))
cursor.close()
idx1, idx2 = util.common_val_idx(res['Timestamp'][0::15].astype('int'), (s.time.lv+0.5).astype('int'))
idx3, idx4 = util.common_val_idx(res['Timestamp'].astype('int'), (s.time.lv+0.5).astype('int'))
marker = -1
while idx1[-1] > idx2[-1]:
idx1 = np.delete(idx1, -1)
marker += 1
tsys = tsys[:, :, :, idx1]
calfilenoise_ = []
for ant in range(calfilenoise.shape[0]):
calfilenoisepol = []
for pol in range(calfilenoise.shape[1]):
calfilenoisefreq = []
for freq in range(calfilenoise.shape[2]):
calfilenoisefreq.append(np.average(calfilenoise[ant, pol, freq, :]))
calfilenoisepol.append(calfilenoisefreq)
calfilenoise_.append(calfilenoisepol)
calfilenoise = np.array(calfilenoise_)
noise_level = []
for ant in range(tsys.shape[0]):
pol_noise = []
for pol in range(tsys.shape[1]):
freq_noise = []
state, = np.where(np.logical_and(res['Ante_Fron_FEM_HPol_Atte_Second'][ant::15].astype('int') == 31,res['Ante_Fron_FEM_HPol_Atte_First'][ant::15].astype('int') == 31))
for freq in range(tsys.shape[2]):
avg_noise = []
for index in state:
try:
if np.logical_and(tsys[ant, pol, freq, index] <= 0.005, index < tsys.shape[3]):
avg_noise.append(tsys[ant, pol, freq, index])
except:
pass
freq_noise.append(np.average(avg_noise))
pol_noise.append(freq_noise)
noise_level.append(pol_noise)
noise_level = np.array(noise_level)
for ant in range(tsys.shape[0]):
for pol in range(tsys.shape[1]):
for freq in range(tsys.shape[2]):
if np.isnan(noise_level[ant, pol, freq]) == False:
pass
else:
if np.isnan(noise_level[ant, pol, freq]) == True:
try:
noise_level[ant, pol, freq] = calfilenoise[ant, pol, freq]
except:
pass
return tsys, res, noise_level, idx1, idx3, marker, trange_feattncal, trange_feattncal2
def apply_fem_level(data, gctime=None):
''' Applys the FEM level corrections to the given data dictionary.
Inputs:
data A dictionary such as that returned by read_idb().
gctime A Time() object whose date specifies which GAINCALTEST
measurements to use. If omitted, the date of the data
is used.
Output:
cdata A dictionary with the level-corrected data. The keys
p, x, p2, and a are all updated.
'''
from util import common_val_idx, nearest_val_idx, bl2ord
import attncal as ac
from gaincal2 import get_fem_level
import copy
# Get timerange from data
trange = Time([data['time'][0], data['time'][-1]], format='jd')
if gctime is None:
gctime = trange[0]
# Get time cadence
dt = np.int(
np.round(np.median(data['time'][1:] - data['time'][:-1]) * 86400))
if dt == 1: dt = None
# Get the FEM levels of the requested timerange
src_lev = get_fem_level(trange,
dt) # solar gain state for timerange of file
nf = len(data['fghz'])
nt = len(src_lev['times'])
attn = ac.read_attncal(
gctime
)[0] # Reads attn from SQL database (returns a list, but use first, generally only, one)
# attn = ac.get_attncal(gctime)[0] # Analyzes GAINCALTEST (returns a list, but use first, generally only, one)
antgain = np.zeros((15, 2, nf, nt),
np.float32) # Antenna-based gains [dB] vs. frequency
# Find common frequencies of attn with data
idx1, idx2 = common_val_idx(data['fghz'], attn['fghz'], precision=4)
# Currently, GAINCALTEST measures 8 levels of attenuation (16 dB). I assumed this would be enough,
# but the flare of 2017-09-10 actually went to 10 levels (20 dB), so we have no choice but to extend
# to higher levels using only the nominal, 2 dB steps above the 8th level. This part of the code
# extends to the maximum 14 levels.
a = np.zeros((14, 13, 2, nf), float) # Extend attenuation to 14 levels
a[:8, :, :, idx1] = attn[
'attn'][:, :13, :,
idx2] # Use GAINCALTEST results in the first 8 levels
for i in range(7, 13):
# Extend to levels 9-14 by adding 2 dB to each previous level
a[i + 1] = a[i] + 2.
for i in range(13):
for k, j in enumerate(idx1):
antgain[i, 0, j] = a[src_lev['hlev'][i], i, 0, j]
antgain[i, 1, j] = a[src_lev['vlev'][i], i, 0, j]
cdata = copy.deepcopy(data)
nblant = 136
blgain = np.zeros((nf, nblant, 4, nt),
float) # Baseline-based gains vs. frequency
for i in range(14):
for j in range(i, 14):
k = bl2ord[i, j]
blgain[:, k, 0] = 10**((antgain[i, 0] + antgain[j, 0]) / 20.)
blgain[:, k, 1] = 10**((antgain[i, 1] + antgain[j, 1]) / 20.)
blgain[:, k, 2] = 10**((antgain[i, 0] + antgain[j, 1]) / 20.)
blgain[:, k, 3] = 10**((antgain[i, 1] + antgain[j, 0]) / 20.)
# Reorder antgain axes to put frequencies in first slot, to match data
antgain = np.swapaxes(np.swapaxes(antgain, 1, 2), 0, 1)
antgainf = 10**(antgain / 10.)
idx = nearest_val_idx(data['time'], src_lev['times'].jd)
nt = len(idx) # New number of times
# Correct the auto- and cross-correlation data
cdata['x'] *= blgain[:, :, :, idx]
# Reshape px and py arrays
cdata['px'].shape = (nf, 16, 3, nt)
cdata['py'].shape = (nf, 16, 3, nt)
# Correct the power
cdata['px'][:, :15, 0] *= antgainf[:, :, 0, idx]
cdata['py'][:, :15, 0] *= antgainf[:, :, 1, idx]
# Correct the power-squared
cdata['px'][:, :15, 1] *= antgainf[:, :, 0, idx]**2
cdata['py'][:, :15, 1] *= antgainf[:, :, 1, idx]**2
# Reshape px and py arrays back to original
cdata['px'].shape = (nf * 16 * 3, nt)
cdata['py'].shape = (nf * 16 * 3, nt)
return cdata
def unrot(data, azeldict=None):
''' Apply the correction to differential feed rotation to data, and return
the corrected data. This also applies flags to data whose antennas are
not tracking.
Inputs:
data A dictionary returned by udb_util.py's readXdata().
azeldict The dictionary returned from get_sql_info(), or if None, the appropriate
get_sql_info() call is done internally.
Output:
cdata A dictionary with the phase-corrected data. Only the key
x is updated.
'''
import copy
from util import lobe, bl2ord
trange = Time(data['time'][[0, -1]], format='jd')
if azeldict is None:
azeldict = get_sql_info(trange)
chi = azeldict['ParallacticAngle'] * np.pi / 180. # (nt, nant)
# Correct parallactic angle for equatorial mounts, relative to Ant14
chi[:,
[8, 9, 10, 12, 13]] = 0 # Currently 0, but can be measured and updated
# Which antennas are tracking
track = azeldict['TrackFlag'] # True if tracking
# Ensure that nearest valid parallactic angle is used for times in the data
good = np.where(azeldict['ActualAzimuth'] != 0)
tidx = [] # List of arrays of indexes for each antenna
for i in range(14):
gd = good[0][np.where(good[1] == i)]
tidx.append(nearest_val_idx(data['time'], azeldict['Time'][gd].jd))
# Read X-Y Delay phase from SQL database and get common frequencies
xml, buf = ch.read_cal(11, t=trange[0])
fghz = stateframe.extract(buf, xml['FGHz'])
good, = np.where(fghz != 0.)
fghz = fghz[good]
dph = stateframe.extract(buf, xml['XYphase'])
dph = dph[:, good]
xi_rot = stateframe.extract(buf, xml['Xi_Rot'])
xi_rot = xi_rot[good]
fidx1, fidx2 = common_val_idx(data['fghz'], fghz, precision=4)
missing = np.setdiff1d(np.arange(len(data['fghz'])), fidx1)
nf, nbl, npol, nt = data['x'].shape
nf = len(fidx1)
# Correct data for X-Y delay phase
for i in range(13):
for j in range(i + 1, 14):
k = bl2ord[i, j]
a1 = lobe(dph[i, fidx2] - dph[j, fidx2])
a2 = -dph[j, fidx2] - xi_rot[fidx2]
a3 = dph[i, fidx2] - xi_rot[fidx2] + np.pi
data['x'][fidx1, k, 1] *= np.repeat(np.exp(1j * a1),
nt).reshape(nf, nt)
data['x'][fidx1, k, 2] *= np.repeat(np.exp(1j * a2),
nt).reshape(nf, nt)
data['x'][fidx1, k, 3] *= np.repeat(np.exp(1j * a3),
nt).reshape(nf, nt)
# Correct data for differential feed rotation
cdata = copy.deepcopy(data)
for n in range(nt):
for i in range(13):
for j in range(i + 1, 14):
k = bl2ord[i, j]
ti = tidx[i][n]
tj = tidx[j][n]
if track[ti, i] and track[tj, j]:
dchi = chi[ti, i] - chi[tj, j]
cchi = np.cos(dchi)
schi = np.sin(dchi)
cdata['x'][:, k, 0,
n] = data['x'][:, k, 0,
n] * cchi + data['x'][:, k, 3,
n] * schi
cdata['x'][:, k, 2,
n] = data['x'][:, k, 2,
n] * cchi + data['x'][:, k, 1,
n] * schi
cdata['x'][:, k, 3,
n] = data['x'][:, k, 3,
n] * cchi - data['x'][:, k, 0,
n] * schi
cdata['x'][:, k, 1,
n] = data['x'][:, k, 1,
n] * cchi - data['x'][:, k, 2,
n] * schi
else:
cdata['x'][:, k, :, n] = np.ma.masked
# Set flags for any missing frequencies (hopefully this also works when "missing" is np.array([]))
cdata['x'][missing] = np.ma.masked
return cdata
def apply_calfac(data, calfac):
''' Applies calibration factors in calfac dictionary returned by get_calfac(),
to the data and returns the calibrated data in the same form. No calibration
can be applied for antennas/frequencies in data that are not included in
calfac, so for those the data are returned unchanged, except the correlated
data for missing frequencies are flagged.
Inputs:
data The data to be calibrated
calfac The calfac dictionary returned by a call to get_calfac().
Output:
cdata The calibrated data
N.B.: The offsun level needs to be subtracted to calibrate power and
auto-correlation, but this is not done here. Instead, one must
later read the calfac dictionary using get_calfac(), and then
subtract tpoffsun*tpcalfac from power, and acoffsun*accalfac from
auto-correlation.
'''
import copy
from util import common_val_idx, bl2ord
fghz = data['fghz']
nfin = len(fghz)
# Find common frequencies
idx1, idx2 = common_val_idx(fghz, calfac['fghz'], precision=4)
nf = len(idx1)
missing = np.setdiff1d(np.arange(len(fghz)), idx1)
nant = 16
nblant = nant * (nant - 1) / 2 + nant
blfac = np.ones(
(nf, nblant, 4),
float) # Factors for missing antennas/frequencies are set to unity
fac = calfac['accalfac'][:, :, idx2] # Extract only common frequencies
for i in range(13):
for j in range(i, 13):
blfac[idx1, bl2ord[i, j], 0] = np.sqrt(fac[i, 0] * fac[j, 0])
blfac[idx1, bl2ord[i, j], 1] = np.sqrt(fac[i, 1] * fac[j, 1])
blfac[idx1, bl2ord[i, j], 2] = np.sqrt(fac[i, 0] * fac[j, 1])
blfac[idx1, bl2ord[i, j], 3] = np.sqrt(fac[i, 1] * fac[j, 0])
antfac = np.ones(
(nf, nant, 2),
float) # Factors for missing antennas/frequencies are set to unity
for i in range(13):
for j in range(2):
antfac[idx1, i, j] = calfac['tpcalfac'][i, j, idx2]
cdata = copy.deepcopy(data)
nt = len(data['time'])
# Calibrate the auto- and cross-correlation data
for i in range(nt):
cdata['x'][idx1, :, :, i] *= blfac
# Reshape px and py arrays
cdata['px'].shape = (nfin, 16, 3, nt)
cdata['py'].shape = (nfin, 16, 3, nt)
for i in range(nt):
# Correct the power
cdata['px'][idx1, :, 0, i] *= antfac[:, :, 0]
cdata['py'][idx1, :, 0, i] *= antfac[:, :, 1]
# Correct the power-squared
cdata['px'][idx1, :, 1, i] *= antfac[:, :, 0]**2
cdata['py'][idx1, :, 1, i] *= antfac[:, :, 1]**2
# Reshape px and py arrays back to original
cdata['px'].shape = (nfin * 16 * 3, nt)
cdata['py'].shape = (nfin * 16 * 3, nt)
# Set flags for any missing frequencies (hopefully this also works when "missing" is np.array([]))
cdata['x'][missing] = np.ma.masked
return cdata
def apply_calfac(data, calfac):
''' Applies calibration factors in calfac dictionary returned by get_calfac(),
to the data and returns the calibrated data in the same form. No calibration
can be applied for antennas/frequencies in data that are not included in
calfac, so for those the data are returned unchanged, except the correlated
data for missing frequencies are flagged.
Inputs:
data The data to be calibrated
calfac The calfac dictionary returned by a call to get_calfac().
Output:
cdata The calibrated data
N.B.: Spectral Kurtosis is no longer valid after applying calibration
'''
import copy
fghz = data['fghz']
nfin = len(fghz)
# Find common frequencies
idx1, idx2 = common_val_idx(fghz, calfac['fghz'], precision=4)
nf = len(idx1)
missing = np.setdiff1d(np.arange(len(fghz)), idx1)
nant = 16
nblant = nant * (nant - 1) / 2 + nant
blfac = np.ones(
(nf, nblant, 4),
float) # Factors for missing antennas/frequencies are set to unity
bloff = np.zeros(
(nf, nblant, 4),
float) # Offsun values (will be zero except for auto-correlations)
fac = calfac['accalfac'][:, :, idx2] # Extract only common frequencies
acoffsun = calfac['acoffsun'][:, :,
idx2] # Extract only common frequencies
nsolant = 13 # Number of solar antennas
for i in range(nsolant):
for j in range(i, nsolant):
blfac[idx1, bl2ord[i, j], 0] = np.sqrt(fac[i, 0] * fac[j, 0])
blfac[idx1, bl2ord[i, j], 1] = np.sqrt(fac[i, 1] * fac[j, 1])
blfac[idx1, bl2ord[i, j], 2] = np.sqrt(fac[i, 0] * fac[j, 1])
blfac[idx1, bl2ord[i, j], 3] = np.sqrt(fac[i, 1] * fac[j, 0])
if i == j:
bloff[idx1, bl2ord[i, j], 0] = acoffsun[j, 0]
bloff[idx1, bl2ord[i, j], 1] = acoffsun[j, 1]
antfac = np.ones(
(nf, nant, 2),
float) # Factors for missing antennas/frequencies are set to unity
offsun = np.zeros(
(nf, nant, 2),
float) # Offsun for missing antennas/frequencies are set to zero
for i in range(nsolant):
for j in range(2):
antfac[idx1, i, j] = calfac['tpcalfac'][i, j, idx2]
offsun[idx1, i, j] = calfac['tpoffsun'][i, j, idx2]
cdata = copy.deepcopy(data)
nt = len(data['time'])
# Calibrate the auto- and cross-correlation data
# Note that bloff is non-zero only for the real part of auto-correlations
for i in range(nt):
cdata['x'][idx1, :, :, i] = (cdata['x'][idx1, :, :, i] - bloff) * blfac
# Reshape px and py arrays
cdata['px'].shape = (nfin, 16, 3, nt)
cdata['py'].shape = (nfin, 16, 3, nt)
for i in range(nt):
# Correct the power
cdata['px'][idx1, :, 0, i] = (cdata['px'][idx1, :, 0, i] -
offsun[:, :, 0]) * antfac[:, :, 0]
cdata['py'][idx1, :, 0, i] = (cdata['py'][idx1, :, 0, i] -
offsun[:, :, 1]) * antfac[:, :, 1]
# Zero-out the power-squared, since SK is no longer valid after applying calibration
cdata['px'][idx1, :, 1, i] = 0.0
cdata['py'][idx1, :, 1, i] = 0.0
# Reshape px and py arrays back to original
cdata['px'].shape = (nfin * 16 * 3, nt)
cdata['py'].shape = (nfin * 16 * 3, nt)
# Set flags for any missing frequencies (hopefully this also works when "missing" is np.array([]))
cdata['x'][missing] = np.ma.masked
return cdata
def apply_fem_level(data, gctime=None, skycal=None):
''' Applys the FEM level corrections to the given data dictionary.
Inputs:
data A dictionary such as that returned by readXdata().
gctime A Time() object whose date specifies which GAINCALTEST
measurements to use. If omitted, the date of the data
is used.
skycal Optional array of receiver noise from SKYCAL or GAINCAL
calibration. Only the receiver noise is applied (subtracted)
Output:
cdata A dictionary with the level-corrected data. The keys
p, x, p2, and a are all updated.
'''
import attncal as ac
from gaincal2 import get_fem_level
import copy
# Get timerange from data
trange = Time([data['time'][0], data['time'][-1]], format='jd')
if gctime is None:
gctime = trange[0]
# Get time cadence
dt = np.int(
np.round(np.nanmedian(data['time'][1:] - data['time'][:-1]) * 86400))
if dt == 1: dt = None
# Get the FEM levels of the requested timerange
src_lev = get_fem_level(trange,
dt) # solar gain state for timerange of file
nf = len(data['fghz'])
nt = len(src_lev['times'])
attn = ac.read_attncal(
gctime
)[0] # Reads attn from SQL database (returns a list, but use first, generally only, one)
# attn = ac.get_attncal(gctime)[0] # Analyzes GAINCALTEST (returns a list, but use first, generally only, one)
antgain = np.zeros((15, 2, nf, nt),
np.float32) # Antenna-based gains [dB] vs. frequency
# Find common frequencies of attn with data
idx1, idx2 = common_val_idx(data['fghz'], attn['fghz'], precision=4)
# Currently, GAINCALTEST measures 8 levels of attenuation (16 dB). I assumed this would be enough,
# but the flare of 2017-09-10 actually went to 10 levels (20 dB), so we have no choice but to extend
# to higher levels using only the nominal, 2 dB steps above the 8th level. This part of the code
# extends to the maximum 16 levels.
a = np.zeros((16, 13, 2, nf), float) # Extend attenuation to 14 levels
a[1:9, :, :, idx1] = attn[
'attn'][:, :13, :,
idx2] # Use GAINCALTEST results in levels 1-9 (bottom level is 0dB)
for i in range(8, 15):
# Extend to levels 9-15 by adding 2 dB to each previous level
a[i + 1] = a[i] + 2.
a[15] = 62. # Level 15 means 62 dB have been inserted.
#print 'Attn list (dB) for ant 1, pol xx, lowest frequency:',a[:,0,0,0]
if dt:
# For this case, src_lev is an array of dictionaries where keys are levels and
# values are the proportion of that level for the given integration
for i in range(13):
for k, j in enumerate(idx1):
for m in range(nt):
for lev, prop in src_lev['hlev'][i, m].items():
antgain[i, 0, j, m] += prop * a[lev, i, 0, idx2[k]]
for lev, prop in src_lev['vlev'][i, m].items():
antgain[i, 1, j, m] += prop * a[lev, i, 1, idx2[k]]
else:
# For this case, src_lev is just an array of levels
for i in range(13):
for k, j in enumerate(idx1):
antgain[i, 0, j] = a[src_lev['hlev'][i], i, 0, idx2[k]]
antgain[i, 1, j] = a[src_lev['vlev'][i], i, 1, idx2[k]]
cdata = copy.deepcopy(data)
nblant = 136
blgain = np.zeros((nf, nblant, 4, nt),
float) # Baseline-based gains vs. frequency
for i in range(14):
for j in range(i, 14):
k = bl2ord[i, j]
blgain[:, k, 0] = 10**((antgain[i, 0] + antgain[j, 0]) / 20.)
blgain[:, k, 1] = 10**((antgain[i, 1] + antgain[j, 1]) / 20.)
blgain[:, k, 2] = 10**((antgain[i, 0] + antgain[j, 1]) / 20.)
blgain[:, k, 3] = 10**((antgain[i, 1] + antgain[j, 0]) / 20.)
# Reorder antgain axes to put frequencies in first slot, to match data
antgain = np.swapaxes(np.swapaxes(antgain, 1, 2), 0, 1)
antgainf = 10**(antgain / 10.)
idx = nearest_val_idx(data['time'], src_lev['times'].jd)
nt = len(idx) # New number of times
# If a skycal dictionary exists, subtract auto-correlation receiver noise before scaling (clip to 0)
if skycal:
sna, snp, snf = skycal['rcvr_bgd_auto'].shape
bgd = skycal['rcvr_bgd_auto'].repeat(nt).reshape((sna, snp, snf, nt))
bgd = bgd[:, :, idx2] # Extract only frequencies matching the data
# Reorder axes
bgd = np.swapaxes(bgd, 0, 2)
# bslice = bgd[:,:,:,idx]
for i in range(13):
cdata['x'][:, bl2ord[i, i], 0] = np.clip(
cdata['x'][:, bl2ord[i, i], 0] - bgd[:, 0, i], 0,
None) #bslice[:,0,i],0,None)
cdata['x'][:, bl2ord[i, i], 1] = np.clip(
cdata['x'][:, bl2ord[i, i], 1] - bgd[:, 1, i], 0,
None) #bslice[:,1,i],0,None)
# Correct the auto- and cross-correlation data
cdata['x'] *= blgain[:, :, :, idx]
# Reshape px and py arrays
cdata['px'].shape = (nf, 16, 3, nt)
cdata['py'].shape = (nf, 16, 3, nt)
# If a skycal dictionary exists, subtract total power receiver noise before scaling (clip to 0)
# NB: This will break SK!
if skycal:
sna, snp, snf = skycal['rcvr_bgd'].shape
bgd = skycal['rcvr_bgd'].repeat(nt).reshape((sna, snp, snf, nt))
bgd = bgd[:, :, idx2] # Extract only frequencies matching the data
# Reorder axes
bgd = np.swapaxes(bgd, 0, 2)
#bslice = bgd[:,:,:,idx]
#bgnd = np.rollaxis(bslice,3)
cdata['px'][:, :13, 0] = np.clip(cdata['px'][:, :13, 0] - bgd[:, 0], 0,
None) #bslice[:,0],0,None)
cdata['py'][:, :13, 0] = np.clip(cdata['py'][:, :13, 0] - bgd[:, 1], 0,
None) #bslice[:,1],0,None)
# Correct the power
cdata['px'][:, :15, 0] *= antgainf[:, :, 0, idx]
cdata['py'][:, :15, 0] *= antgainf[:, :, 1, idx]
# Correct the power-squared
cdata['px'][:, :15, 1] *= antgainf[:, :, 0, idx]**2
cdata['py'][:, :15, 1] *= antgainf[:, :, 1, idx]**2
# Reshape px and py arrays back to original
cdata['px'].shape = (nf * 16 * 3, nt)
cdata['py'].shape = (nf * 16 * 3, nt)
return cdata
