def plt_gains(vis, nu, img_name='out.png', bad_chans=[]):
""" Plot grid of transit phases to check if both
fringestop and the calibration worked. If most
antennas end up with zero phase, then the calibration worked.
Parameters
----------
vis : np.ndarray[nfreq, ncorr, ntimes]
Visibility array
nu : int
Frequency index to plot up
"""
fig = plt.figure(figsize=(14, 14))
# Plot up 128 feeds correlated with antenna "ant"
ant = 1
# Try and estimate the residual phase error
# after calibration. Zero would be a perfect calibration.
angle_err = 0
# Go through 128 feeds plotting up their phase during transit
for i in range(32):
fig.add_subplot(16, 2, i + 1)
antj = 4 * i + 1
if i == ant:
# For autocorrelation plot the visibility rather
# than the phase. This gives a sense for when
# the source actually transits.
plt.plot(vis[nu, misc.feed_map(ant, antj + 1, 128)])
plt.xlim(0, len(vis[nu, 0]))
else:
angle_err += np.mean(
abs(np.angle(vis[nu,
misc.feed_map(ant, antj + 1, 128)]))) / 127.0
plt.plot((np.angle(vis[nu, misc.feed_map(ant, antj + 1, 128)])))
plt.axis('off')
plt.axhline(0.0, color='black')
oo = np.round(
np.std(
np.angle(vis[nu, misc.feed_map(ant, antj + 1, 128)]) *
180 / np.pi))
plt.title(np.str(oo) + ',' + np.str(antj))
if i in bad_chans:
plt.plot(np.angle(vis[nu,
misc.feed_map(ant, antj + 1, 128)]),
color='red')
plt.ylim(-np.pi, np.pi)
plt.title(np.str(180 / np.pi * angle_err))
del vis
print "\n Wrote to %s \n" % img_name
fig.savefig(img_name)
del fig
def remove_fpga_gains(vis, gains, nfeed=128, triu=False):
""" Remove fpga phases
"""
print "............ Removing FPGA gains, triu is %r ............ \n" % triu
# Get gain matrix for visibilites g_i \times g_j^*
#gains_corr = gains[:, :, np.newaxis] * np.conj(gains[:, np.newaxis])
# Take only upper triangle of gain matrix
#ind = np.triu_indices(nfeed)
#gains_mat = np.zeros([vis.shape[0], len(ind[0])], dtype=gains.dtype)
for nu in range(vis.shape[0]):
for i in range(nfeed):
for j in range(i, nfeed):
phi = np.angle(gains[nu, i] * np.conj(gains[nu, j]))
# CHIME seems to have a lowertriangle X-engine, should
# in general use Vij = np.conj(xi) * xj
if triu == True:
vis[nu, misc.feed_map(i, j, nfeed)] *= np.exp(-1j * phi)
elif triu == False:
vis[nu, misc.feed_map(i, j, nfeed)] *= np.exp(1j * phi)
return vis
def remove_fpga_gains(vis, gains, nfeed=128, triu=False):
""" Remove fpga phases
"""
print "............ Removing FPGA gains, triu is %r ............ \n" % triu
# Get gain matrix for visibilites g_i \times g_j^*
#gains_corr = gains[:, :, np.newaxis] * np.conj(gains[:, np.newaxis])
# Take only upper triangle of gain matrix
#ind = np.triu_indices(nfeed)
#gains_mat = np.zeros([vis.shape[0], len(ind[0])], dtype=gains.dtype)
for nu in range(vis.shape[0]):
for i in range(nfeed):
for j in range(i, nfeed):
phi = np.angle(gains[nu, i] * np.conj(gains[nu, j]))
# CHIME seems to have a lowertriangle X-engine, should
# in general use Vij = np.conj(xi) * xj
if triu==True:
vis[nu, misc.feed_map(i, j, nfeed)] *= np.exp(-1j * phi)
elif triu==False:
vis[nu, misc.feed_map(i, j, nfeed)] *= np.exp(1j * phi)
return vis
def plt_gains(vis, nu, img_name='out.png', bad_chans=[]):
""" Plot grid of transit phases to check if both
fringestop and the calibration worked. If most
antennas end up with zero phase, then the calibration worked.
Parameters
----------
vis : np.ndarray[nfreq, ncorr, ntimes]
Visibility array
nu : int
Frequency index to plot up
"""
fig = plt.figure(figsize=(14, 14))
# Plot up 128 feeds correlated with antenna "ant"
ant = 1
# Try and estimate the residual phase error
# after calibration. Zero would be a perfect calibration.
angle_err = 0
# Go through 128 feeds plotting up their phase during transit
for i in range(128):
fig.add_subplot(32, 4, i+1)
if i==ant:
# For autocorrelation plot the visibility rather
# than the phase. This gives a sense for when
# the source actually transits.
plt.plot(vis[nu, misc.feed_map(ant, i+1, 128)])
plt.xlim(0, len(vis[nu, 0]))
else:
angle_err += np.mean(abs(np.angle(vis[nu, misc.feed_map(ant, i+1, 128)]))) / 127.0
plt.plot((np.angle(vis[nu, misc.feed_map(ant, i+1, 128)])))
plt.axis('off')
plt.axhline(0.0, color='black')
oo = np.round(np.std(np.angle(vis[nu, misc.feed_map(ant, i+1, 128)]) * 180 / np.pi))
plt.title(np.str(oo) + ',' + np.str(i))
if i in bad_chans:
plt.plot(np.angle(vis[nu, misc.feed_map(ant, i+1, 128)]), color='red')
plt.ylim(-np.pi, np.pi)
plt.title(np.str(180 / np.pi * angle_err))
del vis
print "\n Wrote to %s \n" % img_name
fig.savefig(img_name)
del fig
def auto_corrs(nfeed):
autos = []
for ii in range(nfeed):
autos.append(misc.feed_map(ii, ii, nfeed))
return autos
def auto_corrs(nfeed):
autos = []
for ii in range(nfeed):
autos.append(misc.feed_map(ii, ii, nfeed))
return autos
def gen_inp(nfeed=256):
""" Generate input information for feeds
Parameters
----------
feeds : list
Feeds whose input info is needed
nfeeds : int
Number of feeds in total
Returns
-------
corrinput_real :
All 256 inputs
inpx :
Only x feeds
inpy :
Only y feeds
"""
# Assumes a standard layout for 128 feeds on each cyl
xfeeds = range(nfeed / 4) + range(2 * nfeed / 4, 3 * nfeed / 4)
yfeeds = range(nfeed / 4, 2 * nfeed / 4) + range(3 * nfeed / 4,
4 * nfeed / 4)
xcorrs = []
ycorrs = []
for ii in range(nfeed / 2):
for jj in range(ii, nfeed / 2):
xcorrs.append(misc.feed_map(xfeeds[ii], xfeeds[jj], nfeed))
ycorrs.append(misc.feed_map(yfeeds[ii], yfeeds[jj], nfeed))
corrinputs = tools.get_correlator_inputs(\
datetime.datetime(2015, 6, 1, 0, 0, 0), correlator='K7BP16-0004')
# Need to rearrange to match order in the correlated data
corrinput_real = rearrange_list(corrinputs, nfeeds=256)
inpx = []
inpy = []
for i in range(nfeed / 2):
inpx.append(corrinput_real[xfeeds[i]])
inpy.append(corrinput_real[yfeeds[i]])
return corrinput_real, inpx, inpy, xcorrs, ycorrs, xfeeds, yfeeds
def apply_pkl_gain(data, gain_pkl, nfeeds=256):
for ii in range(nfeeds):
for jj in range(ii, nfeeds):
data[:, misc.feed_map(ii, jj, nfeeds)] \
*= (gain_pkl[:, ii] * np.conj(gain_pkl[:, jj]))[..., np.newaxis]
return data
def gen_inp(nfeed=256):
""" Generate input information for feeds
Parameters
----------
feeds : list
Feeds whose input info is needed
nfeeds : int
Number of feeds in total
Returns
-------
corrinput_real :
All 256 inputs
inpx :
Only x feeds
inpy :
Only y feeds
"""
# Assumes a standard layout for 128 feeds on each cyl
xfeeds = range(nfeed/4) + range(2 * nfeed/4, 3 * nfeed/4)
yfeeds = range(nfeed/4, 2 * nfeed/4) + range(3 * nfeed/4, 4 * nfeed/4)
xcorrs = []
ycorrs = []
for ii in range(nfeed/2):
for jj in range(ii, nfeed/2):
xcorrs.append(misc.feed_map(xfeeds[ii], xfeeds[jj], nfeed))
ycorrs.append(misc.feed_map(yfeeds[ii], yfeeds[jj], nfeed))
corrinputs = tools.get_correlator_inputs(\
datetime.datetime(2015, 6, 1, 0, 0, 0), correlator='K7BP16-0004')
# Need to rearrange to match order in the correlated data
corrinput_real = rearrange_list(corrinputs, nfeeds=256)
inpx = []
inpy = []
for i in range(nfeed/2):
inpx.append(corrinput_real[xfeeds[i]])
inpy.append(corrinput_real[yfeeds[i]])
return corrinput_real, inpx, inpy, xcorrs, ycorrs, xfeeds, yfeeds
def apply_pkl_gain(data, gain_pkl, nfeeds=256):
for ii in range(nfeeds):
for jj in range(ii, nfeeds):
data[:, misc.feed_map(ii, jj, nfeeds)] \
*= (gain_pkl[:, ii] * np.conj(gain_pkl[:, jj]))[..., np.newaxis]
return data
def select_corrs(feeds, nfeed=256, feeds_cross=None):
autos = []
corrs = []
xycorrs = []
for ii, feedi in enumerate(feeds):
for jj, feedj in enumerate(feeds):
if ii==jj:
autos.append(misc.feed_map(feedi, feedj, nfeed))
if jj>ii:
corrs.append(misc.feed_map(feedi, feedj, nfeed))
if feeds_cross != None:
assert len(feeds) <= len(feeds_cross)
xycorrs.append(misc.feed_map(feedi, feeds_cross[jj], nfeed))
return autos, corrs, xycorrs
def select_corrs(feeds, nfeed=256, feeds_cross=None):
autos = []
corrs = []
xycorrs = []
for ii, feedi in enumerate(feeds):
for jj, feedj in enumerate(feeds):
if ii == jj:
autos.append(misc.feed_map(feedi, feedj, nfeed))
if jj > ii:
corrs.append(misc.feed_map(feedi, feedj, nfeed))
if feeds_cross != None:
assert len(feeds) <= len(feeds_cross)
xycorrs.append(misc.feed_map(feedi, feeds_cross[jj], nfeed))
return autos, corrs, xycorrs
def correct_dfs(dfs, Gains, nfeed=256):
""" Corrects fringestopped visibilities
"""
dfs_corrm = dfs.copy()
for i in range(nfeed):
for j in range(i, nfeed):
#dfs_corrm[:, misc.feed_map(i, j, 128)] *= np.conj(Gains[:, i] * np.conj(Gains[:, j]))
dfs_corrm[:, misc.feed_map(i, j, 128)] *= np.exp(-1j * (Gains[:, i] - Gains[:, j]))
return dfs_corrm
def correct_dfs(dfs, Gains, nfeed=256):
""" Corrects fringestopped visibilities
"""
dfs_corrm = dfs.copy()
for i in range(nfeed):
for j in range(i, nfeed):
#dfs_corrm[:, misc.feed_map(i, j, 128)] *= np.conj(Gains[:, i] * np.conj(Gains[:, j]))
dfs_corrm[:, misc.feed_map(i, j, 128)] *= np.exp(
-1j * (Gains[:, i] - Gains[:, j]))
return dfs_corrm
def fs_and_correct_gains(fn_h5, fn_gain, src, freq=305, \
del_t = 900, transposed=True,\
remove_fpga_phase=True, remove_instr=True,
remove_fpga=False):
dfs = pc.fs_from_file(fn_h5, freq, src,
del_t=1800, transposed=transposed)
ntimes = dfs.shape[0]
fg = h5py.File(fn_gain, 'r')
gx = fg['gainsx']
gy = fg['gainsy']
gain_mat = construct_gain_mat(gx, gy, 64)[freq]
f = h5py.File(fn_h5, 'r')
feeds = range(256)
# Skip loading the gains and applying them if both are False
if (remove_fpga is False) and \
(remove_instr is False) and (remove_fpga_phase is False):
return dfs
if transposed is True:
g = f['gain_coeff'][freq, :, 0]
Gh5 = g['r'] + 1j * g['i']
else:
g = f['gain_coeff'][0, freq]
Gh5 = g['r'] + 1j * g['i']
print np.isnan(Gh5).sum() / np.float(len(Gh5.flatten()))
print "doing the big loop"
print gain_mat.sum()
for fi, nu in enumerate(freq):
for i in range(len(feeds)):
for j in range(i, len(feeds)):
# If solved with triu=False then
# remove_fpga should be +1j and instr should be -1j
if remove_fpga is True:
dfs[:, misc.feed_map(i, j, 256)] \
/= np.conj((Gh5[fi, i]) * np.conj(Gh5[fi, j]))
if remove_fpga_phase is True:
# Remove fpga phases written in .h5 file
dfs[:, misc.feed_map(i, j, 256)] *= \
np.exp(+1j * np.angle(Gh5[fi, i] * np.conj(Gh5[fi, j])))
if remove_instr is True:
# Apply gains solved for
dfs[:, misc.feed_map(i, j, 256)] \
*= np.exp(-1j * np.angle(gain_mat[fi, i]\
* np.conj(gain_mat[fi, j])))
print "Summed h5 gains: ", Gh5.sum()
return dfs
nchunks = len(list) / file_chunk
print "Total of %i files" % len(list)
jj = comm.rank
print "Starting chunk %i of %i" % (jj+1, nchunks)
print "Getting", file_chunk*jj, ":", file_chunk*(jj+1)
x=7
y=3
feeds = np.arange(nfeeds)
nfeeds = len(feeds)
feeds = [3, 7]
corrs = [misc.feed_map(i, x, nfeeds) for i in feeds] + [misc.feed_map(i, y, nfeeds) for i in feeds]
corrs_auto = [misc.feed_map(i, i, nfeeds) for i in feeds]
corrs = range(nfeeds * (nfeeds+1)/2)
ncorr = len(corrs)
if args.use_chime_autos:
feeds = np.arange(nfeeds)
corrs = corrs_auto
ncorr = len(corrs)
if jj==0:
print "RA, dec, DM, period:", RA_src, dec, DM, p1
print "Using correlations", corrs
print "with autos:", corrs_auto
dat_name = args.data_dir[-16:]
list = glob.glob(args.data_dir + '/*h5*')
list.sort()
list = list[0:0 + file_chunk * nnodes]
nchunks = len(list) / file_chunk
print "Total of %i files" % len(list)
jj = comm.rank
print "Starting chunk %i of %i" % (jj+1, nchunks)
print "Getting", file_chunk*jj, ":", file_chunk*(jj+1)
corrs = [misc.feed_map(i, i, 16) for i in range(16)] #+ [misc.feed_map(i, 3, 16) for i in range(16)]
data_arr, time_full, RA, fpga_count = misc.get_data(list[file_chunk*jj:file_chunk*(jj+1)])[1:]
data_arr = data_arr[:, corrs, :]
ntimes = len(time_full)
time = time_full
time_int = args.time_int
freq_int = args.freq_int
fpga_tag = ''
if args.use_fpga==1:
time = (fpga_count - fpga_count[0]) * (np.diff(time_full)[0]) / np.diff(fpga_count)[0]
print "We're going with the fpga counts:", np.median(np.diff(fpga_count))
file_list[ii] = args.data_dir + file_list[ii]
print ""
print "Reading in Data:", file_list
print ""
dataobj = ch_util.andata.Reader(file_list)
X = dataobj.read()
vis, times, nprod = X.vis, X.timestamp, X.nprod
RA = ch_util.ephemeris.transit_RA(times)
nant = int((-1 + (1 + 4 * 2 * nprod)**0.5) / 2)
m = 3
if args.product_set=='autos':
corrs = [misc.feed_map(i, i, nant) for i in range(nant)]
elif args.product_set=='26m':
corrs = [misc.feed_map(i, m, nant) for i in range(nant)]
else:
corrs = [0]
print "RA range of data:", RA.min(),":",RA.max()
print "for correlations", corrs
beam_params = fm.beam_fit(vis, RA, dec = celestial_object[src][2], correlations=corrs) # This is where the actual fit is done.
transit_time = ch_util.ephemeris.datetime.fromtimestamp(times[0])
date_str = transit_time.strftime('%Y_%m_%d')
filename = args.outdir + src + date_str + '.hdf5'
print "Writing beam parameters file to %s" % filename
psr_arr = []
psr_arrx = []
psr_arr26 = []
psr_auto_i = []
psr_auto_j = []
nfeed = 16
corrs = range(nfeed*(nfeed+1)/2)
autos_i = []
autos_j = []
for i in range(nfeed):
for j in range(i, nfeed):
autos_i.append(misc.feed_map(i, i, 16))
autos_j.append(misc.feed_map(j, j, 16))
for nu in range(nfreq):
if nu % 128 == 0:
print "Freq %d" % nu
f = h5py.File(args.data_file + np.str(nu) + '.hdf5', 'r')
datax = f['folded_arr'][corrs[rank], 2000:9000]
data_auto_i = f['folded_arr'][[autos_i[rank]], 2000:9000]
data_auto_j = f['folded_arr'][[autos_j[rank]], 2000:9000]
data26m = f['folded_arr'][autos, 2000:9000]
psr_arrx.append(datax[np.newaxis])
def fs_and_correct_gains(fn_h5, fn_gain, src, freq=305, \
del_t = 900, transposed=True,\
remove_fpga_phase=True, remove_instr=True,
remove_fpga=False):
dfs = pc.fs_from_file(fn_h5, freq, src,
del_t=1800, transposed=transposed)
ntimes = dfs.shape[0]
fg = h5py.File(fn_gain, 'r')
gx = fg['gainsx']
gy = fg['gainsy']
gain_mat = construct_gain_mat(gx, gy, 64)[freq]
f = h5py.File(fn_h5, 'r')
feeds = range(256)
# Skip loading the gains and applying them if both are False
if (remove_fpga is False) and \
(remove_instr is False) and (remove_fpga_phase is False):
return dfs
if transposed is True:
g = f['gain_coeff'][freq, :, 0]
Gh5 = g['r'] + 1j * g['i']
else:
g = f['gain_coeff'][0, freq]
Gh5 = g['r'] + 1j * g['i']
print np.isnan(Gh5).sum() / np.float(len(Gh5.flatten()))
print "doing the big loop"
print gain_mat.sum()
for fi, nu in enumerate(freq):
for i in range(len(feeds)):
for j in range(i, len(feeds)):
# If solved with triu=False then
# remove_fpga should be +1j and instr should be -1j
if remove_fpga is True:
dfs[:, misc.feed_map(i, j, 256)] \
/= np.conj((Gh5[fi, i]) * np.conj(Gh5[fi, j]))
if remove_fpga_phase is True:
# Remove fpga phases written in .h5 file
dfs[:, misc.feed_map(i, j, 256)] *= \
np.exp(+1j * np.angle(Gh5[fi, i] * np.conj(Gh5[fi, j])))
if remove_instr is True:
# Apply gains solved for
dfs[:, misc.feed_map(i, j, 256)] \
*= np.exp(-1j * np.angle(gain_mat[fi, i]\
* np.conj(gain_mat[fi, j])))
print "Summed h5 gains: ", Gh5.sum()
return dfs
nchunks = len(list) / file_chunk
print "Total of %i files" % len(list)
jj = comm.rank
print "Starting chunk %i of %i" % (jj+1, nchunks)
print "Getting", file_chunk*jj, ":", file_chunk*(jj+1)
x = 1
y = 0
feeds = np.arange(1, nfeeds)
# The following chooses only correlations between feed i and the 26m feeds
corrs_x = [misc.feed_map(i, x, nfeeds) for i in feeds if i!=x] \
# + [misc.feed_map(i, y, nfeeds) for i in feeds if i!=x]
corrs_auto = [misc.feed_map(i, i, nfeeds) for i in feeds]
# Or you can just select random correlation products
corrs = corrs_x + corrs_auto
ncorr = len(corrs)
corrs.sort()
if jj == 0:
print "RA, dec, DM, period:", RA_src, dec, DM, p1
print "Using correlations", corrs
data_reader_obj = ch_util.andata.Reader(list[file_chunk*jj:file_chunk*(jj+1)])
