def plt_gains(vis, nu, img_name='out.png', bad_chans=[]): """ Plot grid of transit phases to check if both fringestop and the calibration worked. If most antennas end up with zero phase, then the calibration worked. Parameters ---------- vis : np.ndarray[nfreq, ncorr, ntimes] Visibility array nu : int Frequency index to plot up """ fig = plt.figure(figsize=(14, 14)) # Plot up 128 feeds correlated with antenna "ant" ant = 1 # Try and estimate the residual phase error # after calibration. Zero would be a perfect calibration. angle_err = 0 # Go through 128 feeds plotting up their phase during transit for i in range(32): fig.add_subplot(16, 2, i + 1) antj = 4 * i + 1 if i == ant: # For autocorrelation plot the visibility rather # than the phase. This gives a sense for when # the source actually transits. plt.plot(vis[nu, misc.feed_map(ant, antj + 1, 128)]) plt.xlim(0, len(vis[nu, 0])) else: angle_err += np.mean( abs(np.angle(vis[nu, misc.feed_map(ant, antj + 1, 128)]))) / 127.0 plt.plot((np.angle(vis[nu, misc.feed_map(ant, antj + 1, 128)]))) plt.axis('off') plt.axhline(0.0, color='black') oo = np.round( np.std( np.angle(vis[nu, misc.feed_map(ant, antj + 1, 128)]) * 180 / np.pi)) plt.title(np.str(oo) + ',' + np.str(antj)) if i in bad_chans: plt.plot(np.angle(vis[nu, misc.feed_map(ant, antj + 1, 128)]), color='red') plt.ylim(-np.pi, np.pi) plt.title(np.str(180 / np.pi * angle_err)) del vis print "\n Wrote to %s \n" % img_name fig.savefig(img_name) del fig
def remove_fpga_gains(vis, gains, nfeed=128, triu=False): """ Remove fpga phases """ print "............ Removing FPGA gains, triu is %r ............ \n" % triu # Get gain matrix for visibilites g_i \times g_j^* #gains_corr = gains[:, :, np.newaxis] * np.conj(gains[:, np.newaxis]) # Take only upper triangle of gain matrix #ind = np.triu_indices(nfeed) #gains_mat = np.zeros([vis.shape[0], len(ind[0])], dtype=gains.dtype) for nu in range(vis.shape[0]): for i in range(nfeed): for j in range(i, nfeed): phi = np.angle(gains[nu, i] * np.conj(gains[nu, j])) # CHIME seems to have a lowertriangle X-engine, should # in general use Vij = np.conj(xi) * xj if triu == True: vis[nu, misc.feed_map(i, j, nfeed)] *= np.exp(-1j * phi) elif triu == False: vis[nu, misc.feed_map(i, j, nfeed)] *= np.exp(1j * phi) return vis
def remove_fpga_gains(vis, gains, nfeed=128, triu=False): """ Remove fpga phases """ print "............ Removing FPGA gains, triu is %r ............ \n" % triu # Get gain matrix for visibilites g_i \times g_j^* #gains_corr = gains[:, :, np.newaxis] * np.conj(gains[:, np.newaxis]) # Take only upper triangle of gain matrix #ind = np.triu_indices(nfeed) #gains_mat = np.zeros([vis.shape[0], len(ind[0])], dtype=gains.dtype) for nu in range(vis.shape[0]): for i in range(nfeed): for j in range(i, nfeed): phi = np.angle(gains[nu, i] * np.conj(gains[nu, j])) # CHIME seems to have a lowertriangle X-engine, should # in general use Vij = np.conj(xi) * xj if triu==True: vis[nu, misc.feed_map(i, j, nfeed)] *= np.exp(-1j * phi) elif triu==False: vis[nu, misc.feed_map(i, j, nfeed)] *= np.exp(1j * phi) return vis
def plt_gains(vis, nu, img_name='out.png', bad_chans=[]): """ Plot grid of transit phases to check if both fringestop and the calibration worked. If most antennas end up with zero phase, then the calibration worked. Parameters ---------- vis : np.ndarray[nfreq, ncorr, ntimes] Visibility array nu : int Frequency index to plot up """ fig = plt.figure(figsize=(14, 14)) # Plot up 128 feeds correlated with antenna "ant" ant = 1 # Try and estimate the residual phase error # after calibration. Zero would be a perfect calibration. angle_err = 0 # Go through 128 feeds plotting up their phase during transit for i in range(128): fig.add_subplot(32, 4, i+1) if i==ant: # For autocorrelation plot the visibility rather # than the phase. This gives a sense for when # the source actually transits. plt.plot(vis[nu, misc.feed_map(ant, i+1, 128)]) plt.xlim(0, len(vis[nu, 0])) else: angle_err += np.mean(abs(np.angle(vis[nu, misc.feed_map(ant, i+1, 128)]))) / 127.0 plt.plot((np.angle(vis[nu, misc.feed_map(ant, i+1, 128)]))) plt.axis('off') plt.axhline(0.0, color='black') oo = np.round(np.std(np.angle(vis[nu, misc.feed_map(ant, i+1, 128)]) * 180 / np.pi)) plt.title(np.str(oo) + ',' + np.str(i)) if i in bad_chans: plt.plot(np.angle(vis[nu, misc.feed_map(ant, i+1, 128)]), color='red') plt.ylim(-np.pi, np.pi) plt.title(np.str(180 / np.pi * angle_err)) del vis print "\n Wrote to %s \n" % img_name fig.savefig(img_name) del fig
def auto_corrs(nfeed): autos = [] for ii in range(nfeed): autos.append(misc.feed_map(ii, ii, nfeed)) return autos
def gen_inp(nfeed=256): """ Generate input information for feeds Parameters ---------- feeds : list Feeds whose input info is needed nfeeds : int Number of feeds in total Returns ------- corrinput_real : All 256 inputs inpx : Only x feeds inpy : Only y feeds """ # Assumes a standard layout for 128 feeds on each cyl xfeeds = range(nfeed / 4) + range(2 * nfeed / 4, 3 * nfeed / 4) yfeeds = range(nfeed / 4, 2 * nfeed / 4) + range(3 * nfeed / 4, 4 * nfeed / 4) xcorrs = [] ycorrs = [] for ii in range(nfeed / 2): for jj in range(ii, nfeed / 2): xcorrs.append(misc.feed_map(xfeeds[ii], xfeeds[jj], nfeed)) ycorrs.append(misc.feed_map(yfeeds[ii], yfeeds[jj], nfeed)) corrinputs = tools.get_correlator_inputs(\ datetime.datetime(2015, 6, 1, 0, 0, 0), correlator='K7BP16-0004') # Need to rearrange to match order in the correlated data corrinput_real = rearrange_list(corrinputs, nfeeds=256) inpx = [] inpy = [] for i in range(nfeed / 2): inpx.append(corrinput_real[xfeeds[i]]) inpy.append(corrinput_real[yfeeds[i]]) return corrinput_real, inpx, inpy, xcorrs, ycorrs, xfeeds, yfeeds
def apply_pkl_gain(data, gain_pkl, nfeeds=256): for ii in range(nfeeds): for jj in range(ii, nfeeds): data[:, misc.feed_map(ii, jj, nfeeds)] \ *= (gain_pkl[:, ii] * np.conj(gain_pkl[:, jj]))[..., np.newaxis] return data
def gen_inp(nfeed=256): """ Generate input information for feeds Parameters ---------- feeds : list Feeds whose input info is needed nfeeds : int Number of feeds in total Returns ------- corrinput_real : All 256 inputs inpx : Only x feeds inpy : Only y feeds """ # Assumes a standard layout for 128 feeds on each cyl xfeeds = range(nfeed/4) + range(2 * nfeed/4, 3 * nfeed/4) yfeeds = range(nfeed/4, 2 * nfeed/4) + range(3 * nfeed/4, 4 * nfeed/4) xcorrs = [] ycorrs = [] for ii in range(nfeed/2): for jj in range(ii, nfeed/2): xcorrs.append(misc.feed_map(xfeeds[ii], xfeeds[jj], nfeed)) ycorrs.append(misc.feed_map(yfeeds[ii], yfeeds[jj], nfeed)) corrinputs = tools.get_correlator_inputs(\ datetime.datetime(2015, 6, 1, 0, 0, 0), correlator='K7BP16-0004') # Need to rearrange to match order in the correlated data corrinput_real = rearrange_list(corrinputs, nfeeds=256) inpx = [] inpy = [] for i in range(nfeed/2): inpx.append(corrinput_real[xfeeds[i]]) inpy.append(corrinput_real[yfeeds[i]]) return corrinput_real, inpx, inpy, xcorrs, ycorrs, xfeeds, yfeeds
def select_corrs(feeds, nfeed=256, feeds_cross=None): autos = [] corrs = [] xycorrs = [] for ii, feedi in enumerate(feeds): for jj, feedj in enumerate(feeds): if ii==jj: autos.append(misc.feed_map(feedi, feedj, nfeed)) if jj>ii: corrs.append(misc.feed_map(feedi, feedj, nfeed)) if feeds_cross != None: assert len(feeds) <= len(feeds_cross) xycorrs.append(misc.feed_map(feedi, feeds_cross[jj], nfeed)) return autos, corrs, xycorrs
def select_corrs(feeds, nfeed=256, feeds_cross=None): autos = [] corrs = [] xycorrs = [] for ii, feedi in enumerate(feeds): for jj, feedj in enumerate(feeds): if ii == jj: autos.append(misc.feed_map(feedi, feedj, nfeed)) if jj > ii: corrs.append(misc.feed_map(feedi, feedj, nfeed)) if feeds_cross != None: assert len(feeds) <= len(feeds_cross) xycorrs.append(misc.feed_map(feedi, feeds_cross[jj], nfeed)) return autos, corrs, xycorrs
def correct_dfs(dfs, Gains, nfeed=256): """ Corrects fringestopped visibilities """ dfs_corrm = dfs.copy() for i in range(nfeed): for j in range(i, nfeed): #dfs_corrm[:, misc.feed_map(i, j, 128)] *= np.conj(Gains[:, i] * np.conj(Gains[:, j])) dfs_corrm[:, misc.feed_map(i, j, 128)] *= np.exp(-1j * (Gains[:, i] - Gains[:, j])) return dfs_corrm
def correct_dfs(dfs, Gains, nfeed=256): """ Corrects fringestopped visibilities """ dfs_corrm = dfs.copy() for i in range(nfeed): for j in range(i, nfeed): #dfs_corrm[:, misc.feed_map(i, j, 128)] *= np.conj(Gains[:, i] * np.conj(Gains[:, j])) dfs_corrm[:, misc.feed_map(i, j, 128)] *= np.exp( -1j * (Gains[:, i] - Gains[:, j])) return dfs_corrm
def fs_and_correct_gains(fn_h5, fn_gain, src, freq=305, \ del_t = 900, transposed=True,\ remove_fpga_phase=True, remove_instr=True, remove_fpga=False): dfs = pc.fs_from_file(fn_h5, freq, src, del_t=1800, transposed=transposed) ntimes = dfs.shape[0] fg = h5py.File(fn_gain, 'r') gx = fg['gainsx'] gy = fg['gainsy'] gain_mat = construct_gain_mat(gx, gy, 64)[freq] f = h5py.File(fn_h5, 'r') feeds = range(256) # Skip loading the gains and applying them if both are False if (remove_fpga is False) and \ (remove_instr is False) and (remove_fpga_phase is False): return dfs if transposed is True: g = f['gain_coeff'][freq, :, 0] Gh5 = g['r'] + 1j * g['i'] else: g = f['gain_coeff'][0, freq] Gh5 = g['r'] + 1j * g['i'] print np.isnan(Gh5).sum() / np.float(len(Gh5.flatten())) print "doing the big loop" print gain_mat.sum() for fi, nu in enumerate(freq): for i in range(len(feeds)): for j in range(i, len(feeds)): # If solved with triu=False then # remove_fpga should be +1j and instr should be -1j if remove_fpga is True: dfs[:, misc.feed_map(i, j, 256)] \ /= np.conj((Gh5[fi, i]) * np.conj(Gh5[fi, j])) if remove_fpga_phase is True: # Remove fpga phases written in .h5 file dfs[:, misc.feed_map(i, j, 256)] *= \ np.exp(+1j * np.angle(Gh5[fi, i] * np.conj(Gh5[fi, j]))) if remove_instr is True: # Apply gains solved for dfs[:, misc.feed_map(i, j, 256)] \ *= np.exp(-1j * np.angle(gain_mat[fi, i]\ * np.conj(gain_mat[fi, j]))) print "Summed h5 gains: ", Gh5.sum() return dfs
nchunks = len(list) / file_chunk print "Total of %i files" % len(list) jj = comm.rank print "Starting chunk %i of %i" % (jj+1, nchunks) print "Getting", file_chunk*jj, ":", file_chunk*(jj+1) x=7 y=3 feeds = np.arange(nfeeds) nfeeds = len(feeds) feeds = [3, 7] corrs = [misc.feed_map(i, x, nfeeds) for i in feeds] + [misc.feed_map(i, y, nfeeds) for i in feeds] corrs_auto = [misc.feed_map(i, i, nfeeds) for i in feeds] corrs = range(nfeeds * (nfeeds+1)/2) ncorr = len(corrs) if args.use_chime_autos: feeds = np.arange(nfeeds) corrs = corrs_auto ncorr = len(corrs) if jj==0: print "RA, dec, DM, period:", RA_src, dec, DM, p1 print "Using correlations", corrs print "with autos:", corrs_auto
dat_name = args.data_dir[-16:] list = glob.glob(args.data_dir + '/*h5*') list.sort() list = list[0:0 + file_chunk * nnodes] nchunks = len(list) / file_chunk print "Total of %i files" % len(list) jj = comm.rank print "Starting chunk %i of %i" % (jj+1, nchunks) print "Getting", file_chunk*jj, ":", file_chunk*(jj+1) corrs = [misc.feed_map(i, i, 16) for i in range(16)] #+ [misc.feed_map(i, 3, 16) for i in range(16)] data_arr, time_full, RA, fpga_count = misc.get_data(list[file_chunk*jj:file_chunk*(jj+1)])[1:] data_arr = data_arr[:, corrs, :] ntimes = len(time_full) time = time_full time_int = args.time_int freq_int = args.freq_int fpga_tag = '' if args.use_fpga==1: time = (fpga_count - fpga_count[0]) * (np.diff(time_full)[0]) / np.diff(fpga_count)[0] print "We're going with the fpga counts:", np.median(np.diff(fpga_count))
file_list[ii] = args.data_dir + file_list[ii] print "" print "Reading in Data:", file_list print "" dataobj = ch_util.andata.Reader(file_list) X = dataobj.read() vis, times, nprod = X.vis, X.timestamp, X.nprod RA = ch_util.ephemeris.transit_RA(times) nant = int((-1 + (1 + 4 * 2 * nprod)**0.5) / 2) m = 3 if args.product_set=='autos': corrs = [misc.feed_map(i, i, nant) for i in range(nant)] elif args.product_set=='26m': corrs = [misc.feed_map(i, m, nant) for i in range(nant)] else: corrs = [0] print "RA range of data:", RA.min(),":",RA.max() print "for correlations", corrs beam_params = fm.beam_fit(vis, RA, dec = celestial_object[src][2], correlations=corrs) # This is where the actual fit is done. transit_time = ch_util.ephemeris.datetime.fromtimestamp(times[0]) date_str = transit_time.strftime('%Y_%m_%d') filename = args.outdir + src + date_str + '.hdf5' print "Writing beam parameters file to %s" % filename
psr_arr = [] psr_arrx = [] psr_arr26 = [] psr_auto_i = [] psr_auto_j = [] nfeed = 16 corrs = range(nfeed*(nfeed+1)/2) autos_i = [] autos_j = [] for i in range(nfeed): for j in range(i, nfeed): autos_i.append(misc.feed_map(i, i, 16)) autos_j.append(misc.feed_map(j, j, 16)) for nu in range(nfreq): if nu % 128 == 0: print "Freq %d" % nu f = h5py.File(args.data_file + np.str(nu) + '.hdf5', 'r') datax = f['folded_arr'][corrs[rank], 2000:9000] data_auto_i = f['folded_arr'][[autos_i[rank]], 2000:9000] data_auto_j = f['folded_arr'][[autos_j[rank]], 2000:9000] data26m = f['folded_arr'][autos, 2000:9000] psr_arrx.append(datax[np.newaxis])
nchunks = len(list) / file_chunk print "Total of %i files" % len(list) jj = comm.rank print "Starting chunk %i of %i" % (jj+1, nchunks) print "Getting", file_chunk*jj, ":", file_chunk*(jj+1) x = 1 y = 0 feeds = np.arange(1, nfeeds) # The following chooses only correlations between feed i and the 26m feeds corrs_x = [misc.feed_map(i, x, nfeeds) for i in feeds if i!=x] \ # + [misc.feed_map(i, y, nfeeds) for i in feeds if i!=x] corrs_auto = [misc.feed_map(i, i, nfeeds) for i in feeds] # Or you can just select random correlation products corrs = corrs_x + corrs_auto ncorr = len(corrs) corrs.sort() if jj == 0: print "RA, dec, DM, period:", RA_src, dec, DM, p1 print "Using correlations", corrs data_reader_obj = ch_util.andata.Reader(list[file_chunk*jj:file_chunk*(jj+1)])