def test_transit_delta_t(): t_start = 21351.34 t1 = ephemeris.transit_times(123.0, t_start)[0] t2 = ephemeris.transit_times(125.0, t_start)[0] delta = (t2 - t1) % 86400 assert delta == approx(8.0 * 60.0 * ephemeris.STELLAR_S, abs=0.1)
def test_transit_sources(): # Check at an early time (this is close to the UNIX epoch) t_start = 12315.123 t1 = ephemeris.transit_times(350.8664, t_start)[0] # This is CasA's RA t2 = ephemeris.transit_times(ephemeris.CasA, t_start)[0] # Due to precession of the polar axis this only matches within ~30s assert t1 == approx(t2, abs=30) # Check at an early time (this is close to the UNIX epoch) t_start = datetime(2001, 1, 1) t1 = ephemeris.transit_times(350.8664, t_start)[0] # This is CasA's RA t2 = ephemeris.transit_times(ephemeris.CasA, t_start)[0] # This is the J2000 epoch and so the precession should be ~0. assert t1 == approx(t2, abs=0.1)
def print_timestamp(fn): src_dict = {'CasA': eph.CasA, 'TauA': eph.TauA, 'CygA': eph.CygA, 'VirA': eph.VirA, '1929': 292.0, '0329': 54.0} try: f = h5py.File(fn, 'r') except IOError: print "File couldn't be opened" return times = f['index_map']['time'].value['ctime'][:] print "-------------------------------------" print "start time %s in PT\n" % (eph.unix_to_datetime(times[0])) print "RA range %f : %f" % (np.round(eph.transit_RA(times[0]), 1), np.round(eph.transit_RA(times[-1]),1)) print "-------------------------------------" srcz = [] for src in src_dict: if eph.transit_times(src_dict[src], times[0])[0] < times[-1]: print "%s is in this file" % src srcz.append(src) return srcz
def test_transit_against_transit_ra(): t_start = 65422.2 ra = 234.54234 t = ephemeris.transit_times(ra, t_start)[0] ra_back_calculated = ephemeris.lsa(t) assert ra == approx(ra_back_calculated, abs=1e-2)
def fs_from_file(filename, frq, src, del_t=900, transposed=True, subtract_avg=False): f = h5py.File(filename, 'r') times = f['index_map']['time'].value['ctime'] + 10.6 src_trans = eph.transit_times(src, times[0]) # try to account for differential arrival time from cylinder rotation. del_phi = (src._dec - np.radians(eph.CHIMELATITUDE)) * np.sin(np.radians(1.988)) del_phi *= (24 * 3600.0) / (2 * np.pi) # Adjust the transit time accordingly src_trans += del_phi # Select +- del_t of transit, accounting for the mispointing t_range = np.where((times < src_trans + del_t) & (times > src_trans - del_t))[0] times = times[t_range[0]:t_range[-1]]#[offp::2] test print "Time range:", times[0], times[-1] print "\n...... This data is from %s starting at RA: %f ...... \n" \ % (eph.unix_to_datetime(times[0]), eph.transit_RA(times[0])) if transposed is True: v = f['vis'][frq[0]:frq[-1]+1, :] v = v[..., t_range[0]:t_range[-1]] vis = v['r'] + 1j * v['i'] del v # Read in time and freq slice if data has not yet been transposed if transposed is False: v = f['vis'][t_range[0]:t_range[-1], frq[0]:frq[-1]+1, :] vis = v['r'][:] + 1j * v['i'][:] del v vis = np.transpose(vis, (1, 2, 0)) inp = gen_inp()[0] # Remove offset from galaxy if subtract_avg is True: vis -= 0.5 * (vis[..., 0] + vis[..., -1])[..., np.newaxis] freq_MHZ = 800.0 - np.array(frq) / 1024.0 * 400. print len(inp) baddies = np.where(np.isnan(tools.get_feed_positions(inp)[:, 0]))[0] # Fringestop to location of "src" data_fs = tools.fringestop_pathfinder(vis, eph.transit_RA(times), freq_MHZ, inp, src) # data_fs = fringestop_pathfinder(vis, eph.transit_RA(times), freq_MHZ, inp, src) return data_fs
def transit_flag(name, time, nsig=1.0, freq=400.0, pol='X'): flag = np.zeros(time.size, dtype=np.bool) if name.lower() == 'sun': window_sec = nsig * utils.get_window(freq, pol=pol, dec=np.radians(23.45), deg=False) ttrans = ephemeris.solar_transit(time[0] - 24.0 * 3600.0, time[-1] + 24.0 * 3600.0) else: src = FluxCatalog[name].skyfield window_sec = nsig * utils.get_window(freq, pol=pol, dec=np.radians(FluxCatalog[name].dec), deg=False) ttrans = ephemeris.transit_times(src, time[0] - 24.0 * 3600.0, time[-1] + 24.0 * 3600.0) for tt in ttrans: # Flag +/- window_sec around peak location begin = tt - window_sec end = tt + window_sec flag |= ((time >= begin) & (time <= end)) return flag
def print_timestamp(fn): src_dict = { 'CasA': eph.CasA, 'TauA': eph.TauA, 'CygA': eph.CygA, 'VirA': eph.VirA, '1929': 292.0, '0329': 54.0 } try: f = h5py.File(fn, 'r') except IOError: print "File couldn't be opened" return times = f['index_map']['time'].value['ctime'][:] print "-------------------------------------" print "start time %s in PT\n" % (eph.unix_to_datetime(times[0])) print "RA range %f : %f" % (np.round(eph.transit_RA( times[0]), 1), np.round(eph.transit_RA(times[-1]), 1)) print "-------------------------------------" srcz = [] for src in src_dict: if eph.transit_times(src_dict[src], times[0])[0] < times[-1]: print "%s is in this file" % src srcz.append(src) return srcz
def get_sunfree_srcs(self, srcs=None): """This method uses the attributes 'night_acq_list' and 'acq_list' to determine the srcs that transit in the available data. If these attributes do not exist, then the method 'set_acq_list' is called. If srcs is not specified, then it defaults to the brightest four radio point sources in the sky: CygA, CasA, TauA, and VirA. """ if self.acq_list is None: self.set_acq_list() if srcs is None: srcs = [ ephemeris.CygA, ephemeris.CasA, ephemeris.TauA, ephemeris.VirA ] Ns = len(srcs) clr = [False] * Ns ntt = [False] * Ns # night transit for ii, src in enumerate(srcs): night_transit = np.array([]) for acq in self.night_acq_list: night_transit = np.append( night_transit, ephemeris.transit_times(src, *acq[1])) if night_transit.size: ntt[ii] = True if src.name in ["CygA", "CasA"]: transit = np.array([]) for acq in self.acq_list: transit = np.append(transit, ephemeris.transit_times(src, *acq[1])) if transit.size: clr[ii] = True else: clr[ii] = ntt[ii] return clr, ntt, srcs
def test_ha_src(self): times = self.times ha_src = pv.ha_src(times, self.RA_src) diff_true = np.round(np.mod(eph.CHIMELONGITUDE - (-78.0730), 360)) trans_time = eph.transit_times(self.RA_src + diff_true, times[0]) ha_src_trans = np.mod(np.round(pv.ha_src(trans_time, self.RA_src)), 360) ha_src_pretrans = pv.ha_src(trans_time - 1000, self.RA_src) assert ha_src_trans == 0.0 assert (ha_src < 360).all() assert ha_src_pretrans < 0.0
def noise_src_phase(fn_ns_sol, fn_sky_sol, src=eph.CasA, trb=10): """ Get noise source phase at the time sky solution was calculated (e.g. time of CasA transit) and use that as baseline. Parameters ---------- fn_ns_sol : np.str file name with noise source solution fn_sky_sol : np.str file name with sky solution src : ephemeris.Object object that gave sky solution in fn_sky_sol tbr : np.int time rebin factor Returns ------- phase solution : array_like (nfeed, nt) real array of phases """ f = h5py.File(fn_ns_sol, 'r') fsky = h5py.File(fn_sky_sol, 'r') gx = fsky['gainsx'][:] gy = fsky['gainsy'][:] gains_ns = f['gains'][:] feeds = f['channels'][:] freq = f['freq_bins'][:] toff = f['timestamp_off'] nt = len(toff) nfeed = len(feeds) nfreq = len(freq) toff = toff[::10] gains_sky = cp.construct_gain_mat(gx, gy, 64) gains_sky = gains_sky[freq, feeds] gains_ns = gains_ns[..., :nt // trb * trb].reshape(nfreq, nfeed, -1, trb).mean(-1) src_trans = eph.transit_times(src, toff[0]) trans_pix = np.argmin(abs(src_trans - toff)) phase = np.angle(gains_ns) - np.angle(gains_ns[..., trans_pix, np.newaxis]) return np.angle(gains_sky)[:, np.newaxis] + phase
def find_transit_file(dir_nm, unix_time=None, src=None, trans=True, verbose=True): flist = glob.glob(dir_nm) tdel = [] print "Looking at %d files" % len(flist) for ii in range(len(flist)): if trans is True: try: r = andata.Reader(flist[ii]) if ii == 0 and src != None: time_trans = eph.transit_times(src, r.time[0]) elif unix_time != None: time_trans = unix_time else: continue tdel.append(np.min(abs(time_trans - r.time))) if verbose is True: print flist[ii] print "%f hours away \n" % np.min( abs(time_trans - r.time)) / 3600.0 except (KeyError, IOError): print "That one didn't work" elif trans is False: try: f = h5py.File(flist[ii], 'r') times = f['index_map']['time'].value['ctime'][0] f.close() time_trans = unix_time tdel.append(np.min(abs(time_trans - times))) except (ValueError, IOError): pass tdel = np.array(tdel) return flist[np.argmin(tdel)], tdel.min()
def find_transit_file(dir_nm, unix_time=None, src=None, trans=True, verbose=True): flist = glob.glob(dir_nm) tdel = [] print "Looking at %d files" % len(flist) for ii in range(len(flist)): if trans is True: try: r = andata.Reader(flist[ii]) if ii==0 and src != None: time_trans = eph.transit_times(src, r.time[0]) elif unix_time != None: time_trans = unix_time else: continue tdel.append(np.min(abs(time_trans - r.time))) if verbose is True: print flist[ii] print "%f hours away \n" % np.min(abs(time_trans - r.time)) / 3600.0 except (KeyError, IOError): print "That one didn't work" elif trans is False: try: f = h5py.File(flist[ii], 'r') times = f['index_map']['time'].value['ctime'][0] f.close() time_trans = unix_time tdel.append(np.min(abs(time_trans - times))) except (ValueError, IOError): pass tdel = np.array(tdel) return flist[np.argmin(tdel)], tdel.min()
def read_data(reader_obj, src, prod_sel, freq_sel=None, del_t=50): """ Use andata tools to select freq, products, and times. """ R = reader_obj # Figure out when calibration source transits src_trans = eph.transit_times(src, R.time[0]) # Select +-100 seconds of transit time_range = np.where((R.time < src_trans + del_t) & (R.time > src_trans - del_t))[0] R.time_sel = [time_range[0], time_range[-1]] R.prod_sel = prod_sel R.freq_sel = freq_sel and_obj = R.read() return and_obj
def find_transit(time0='Now', src=eph.CasA): import time if time0 == 'Now': time0 = time.time() # Get reference datetime from unixtime dt_now = eph.unix_to_datetime(time0) dt_now = dt_now.isoformat() # Only use relevant characters in datetime string dt_str = dt_now.replace("-", "")[:7] dirnm = '/mnt/gong/archive/' + dt_str filelist = glob.glob(dirnm + '*/*h5') # Step through each file and find that day's transit for ff in filelist: try: andataReader = andata.Reader(ff) acqtimes = andataReader.time trans_time = eph.transit_times(src, time0) #print ff # print eph.transit_RA(trans_time), eph.transit_RA(acqtimes[0]) del andataReader if np.abs(acqtimes - trans_time[0]).min() < 1000.0: # print "On ", eph.unix_to_datetime(trans_time[0]) # print "foundit in %s \n" % ff return ff break except (KeyError, ValueError, IOError): pass return None
def fringestop_and_sum(fn, feeds, freq, src, transposed=True, return_unfs=True, meridian=False, del_t=1500, frick=None): """ Take an input file fn and a set of feeds and return a formed beam on src. """ if transposed is True: r = andata.Reader(fn) r.freq_sel = freq X = r.read() times = r.time else: f = h5py.File(fn, 'r') times = f['index_map']['time'].value['ctime'] print "Read in data" # Get transit time for source src_trans = eph.transit_times(src, times[0]) del_phi = 1.30 * (src._dec - np.radians(eph.CHIMELATITUDE)) * np.sin(np.radians(1.988)) del_phi *= (24 * 3600.0) / (2 * np.pi) # Adjust the transit time accordingly src_trans += del_phi # Select +- del_t of transit, accounting for the mispointing t_range = np.where((times < src_trans + del_t) & (times > src_trans - del_t))[0] times = times[t_range[0]:t_range[-1]]#[offp::2] test print "Time range:", times[0], times[-1] # Generate correctly ordered corrinputs corrinput_real = gen_inp()[0] inp = np.array(corrinput_real) # Ensure vis array is in correct order (freq, prod, time) if transposed is True: data = X.vis[:, :, t_range[0]:t_range[-1]] freq = X.freq else: v = f['vis'][t_range[0]:t_range[-1], freq, :] vis = v['r'] + 1j * v['i'] data = vis.transpose()[np.newaxis] del vis freq = 800 - 400.0 / 1024 * freq freq = np.array([freq]) # autos = auto_corrs(256) # offp = (abs(data[:, autos, 0::2]).mean() > (abs(data[:, autos, 1::2]).mean())).astype(int) # data = data[..., offp::2] test data_unfs = sum_corrs(data.copy(), feeds) ra_ = eph.transit_RA(times) ra_2 = nolan_ra(times) #ra_ = ra_2.copy() if meridian is True: ra = np.ones_like(ra_) * np.degrees(src._ra) else: ra = ra_ print len(inp) dfs = tools.fringestop_pathfinder(data.copy(), ra, freq, inp, src) #dfs = fringestop_pathfinder(data.copy(), ra_2, freq, inp, src, frick=frick) # dfs = fringestop_pathfinder(data.copy(), ra_1, freq, inp, src, frick=frick) # fp = np.loadtxt('/home/connor/feed_layout_decrease.txt') # PH = fill_nolan(times, src._ra * 180.0 / np.pi, src._dec * 180.0 / np.pi, fp) dfs_sum = sum_corrs(dfs, feeds) if return_unfs is True: return dfs_sum, ra_, dfs, data else: return dfs_sum, ra_
def process(self, tstream): """Apply the timing correction to the input timestream. Parameters ---------- tstream : andata.CorrData, containers.TimeStream, or containers.SiderealStream Apply the timing correction to the visibilities stored in this container. Returns ------- tstream_corr : same as `tstream` Timestream with corrected visibilities. """ # Determine times if "time" in tstream.index_map: timestamp = tstream.time else: csd = (tstream.attrs["lsd"] if "lsd" in tstream.attrs else tstream.attrs["csd"]) timestamp = ephemeris.csd_to_unix(csd + tstream.ra / 360.0) # Extract local frequencies tstream.redistribute("freq") nfreq = tstream.vis.local_shape[0] sfreq = tstream.vis.local_offset[0] efreq = sfreq + nfreq freq = tstream.freq[sfreq:efreq] # If requested, extract the input flags input_flags = tstream.input_flags[:] if self.use_input_flags else None # Check needs_timing_correction flags time ranges to see if input timestream # falls within the interval needs_timing_correction = False for flag in self.flags: if (timestamp[0] >= flag["start_time"] and timestamp[0] <= flag["finish_time"]): if timestamp[-1] >= flag["finish_time"]: raise PipelineRuntimeError( f"Data covering {timestamp[0]} to {timestamp[-1]} partially overlaps " f"needs_timing_correction DataFlag covering {ephemeris.unix_to_datetime(flag['start_time']).strftime('%Y%m%dT%H%M%SZ')} " f"to {ephemeris.unix_to_datetime(flag['finish_time']).strftime('%Y%m%dT%H%M%SZ')}." ) else: self.log.info( f"Data covering {timestamp[0]} to {timestamp[-1]} flagged by " f"needs_timing_correction DataFlag covering {ephemeris.unix_to_datetime(flag['start_time']).strftime('%Y%m%dT%H%M%SZ')} " f"to {ephemeris.unix_to_datetime(flag['finish_time']).strftime('%Y%m%dT%H%M%SZ')}. Timing correction will be applied." ) needs_timing_correction = True break if not needs_timing_correction: self.log.info( f"Data in span {ephemeris.unix_to_datetime(timestamp[0]).strftime('%Y%m%dT%H%M%SZ')} to {ephemeris.unix_to_datetime(timestamp[-1]).strftime('%Y%m%dT%H%M%SZ')} does not need timing correction" ) return tstream # Find the right timing correction for tcorr in self.tcorr: if timestamp[0] >= tcorr.time[0] and timestamp[-1] <= tcorr.time[ -1]: break else: msg = ( "Could not find timing correction file covering " "range of timestream data (%s to %s)." % tuple( ephemeris.unix_to_datetime([timestamp[0], timestamp[-1]]))) if self.pass_if_missing: self.log.warning(msg + " Doing nothing.") return tstream raise RuntimeError(msg) self.log.info("Using correction file %s" % tcorr.attrs["tag"]) # If requested, reference the timing correct with respect to source transit time if self.refer_to_transit: # First check for transit_time attribute in file ttrans = tstream.attrs.get("transit_time", None) if ttrans is None: source = tstream.attrs["source_name"] ttrans = ephemeris.transit_times( ephemeris.source_dictionary[source], tstream.time[0], tstream.time[-1], ) if ttrans.size != 1: raise RuntimeError( "Found %d transits of %s in timestream. " "Require single transit." % (ttrans.size, source)) else: ttrans = ttrans[0] self.log.info( "Referencing timing correction to %s (RA=%0.1f deg)." % ( ephemeris.unix_to_datetime(ttrans).strftime( "%Y%m%dT%H%M%SZ"), ephemeris.lsa(ttrans), )) tcorr.set_global_reference_time(ttrans, window=self.transit_window, interpolate=True, interp="linear") # Apply the timing correction tcorr.apply_timing_correction(tstream, time=timestamp, freq=freq, input_flags=input_flags, copy=False) return tstream
def solve_ps_transit(filename, corrs, feeds, inp, src, nfreq=1024, transposed=False, nfeed=128): """ Function that fringestops time slice where point source is in the beam, takes all correlations for a given polarization, and then eigendecomposes the correlation matrix freq by freq after removing the fpga phases. It will also plot intermediate steps to verify the phase solution. Parameters ---------- filename : np.str Full-path filename corrs : list List of correlations to use in solver feeds : list List of feeds to use inp : Correlator inputs (output of ch_util.tools.get_correlator_inputs) src : ephem.FixedBody Source to calibrate off of. e.g. ch_util.ephemeris.TauA Returns ------- Gains : np.array Complex gain array (nfreq, nfeed) """ nsplit = 32 # Number of freq chunks to divide nfreq into del_t = 800 f = h5py.File(filename, 'r') # Add half an integration time to each. Hack. times = f['index_map']['time'].value['ctime'] + 10.50 src_trans = eph.transit_times(src, times[0]) # try to account for differential arrival time from # cylinder rotation. del_phi = (src._dec - np.radians(eph.CHIMELATITUDE)) \ * np.sin(np.radians(1.988)) del_phi *= (24 * 3600.0) / (2 * np.pi) # Adjust the transit time accordingly src_trans += del_phi # Select +- del_t of transit, accounting for the mispointing t_range = np.where((times < src_trans + del_t) & (times > src_trans - del_t))[0] print "\n...... This data is from %s starting at RA: %f ...... \n" \ % (eph.unix_to_datetime(times[0]), eph.transit_RA(times[0])) assert (len(t_range) > 0), "Source is not in this acq" # Create gains array to fill in solution Gains = np.zeros([nfreq, nfeed], np.complex128) print "Starting the solver" times = times[t_range[0]:t_range[-1]] k=0 # Start at a strong freq channel that can be plotted # and from which we can find the noise source on-sample for i in range(12, nsplit) + range(0, 12): k+=1 # Divides the arrays up into nfreq / nsplit freq chunks and solves those frq = range(i * nfreq // nsplit, (i+1) * nfreq // nsplit) print " %d:%d \n" % (frq[0], frq[-1]) # Read in time and freq slice if data has already been transposed if transposed is True: v = f['vis'][frq[0]:frq[-1]+1, corrs, :] v = v[..., t_range[0]:t_range[-1]] vis = v['r'] + 1j * v['i'] if k==1: autos = auto_corrs(nfeed) offp = (abs(vis[:, autos, 0::2]).mean() > \ (abs(vis[:, autos, 1::2]).mean())).astype(int) times = times[offp::2] vis = vis[..., offp::2] gg = f['gain_coeff'][frq[0]:frq[-1]+1, feeds, t_range[0]:t_range[-1]][..., offp::2] gain_coeff = gg['r'] + 1j * gg['i'] del gg # Read in time and freq slice if data has not yet been transposed if transposed is False: print "TRANSPOSED V OF CODE DOESN'T WORK YET!" v = f['vis'][t_range[0]:t_range[-1]:2, frq[0]:frq[-1]+1, corrs] vis = v['r'][:] + 1j * v['i'][:] del v gg = f['gain_coeff'][0, frq[0]:frq[-1]+1, feeds] gain_coeff = gg['r'][:] + 1j * gg['i'][:] vis = vis[..., offp::2] vis = np.transpose(vis, (1, 2, 0)) # Remove fpga gains from data vis = remove_fpga_gains(vis, gain_coeff, nfeed=nfeed, triu=False) # Remove offset from galaxy vis -= 0.5 * (vis[..., 0] + vis[..., -1])[..., np.newaxis] # Get physical freq for fringestopper freq_MHZ = 800.0 - np.array(frq) / 1024.0 * 400. baddies = np.where(np.isnan(tools.get_feed_positions(inp)[:, 0]))[0] a, b, c = select_corrs(baddies, nfeed=128) vis[:, a + b] = 0.0 # Fringestop to location of "src" data_fs = tools.fringestop_pathfinder(vis, eph.transit_RA(times), freq_MHZ, inp, src) del vis dr, sol_arr = solve_gain(data_fs) # Find index of point source transit drlist = np.argmax(dr, axis=-1) # If multiple freq channels are zerod, the trans_pix # will end up being 0. This is bad, so ensure that # you are only looking for non-zero transit pixels. drlist = [x for x in drlist if x != 0] trans_pix = np.argmax(np.bincount(drlist)) assert trans_pix != 0.0 Gains[frq] = sol_arr[..., trans_pix-3:trans_pix+4].mean(-1) zz = h5py.File('data' + str(i) + '.hdf5','w') zz.create_dataset('data', data=dr) zz.close() print "%f, %d Nans out of %d" % (np.isnan(sol_arr).sum(), np.isnan(Gains[frq]).sum(), np.isnan(Gains[frq]).sum()) print trans_pix, sol_arr[..., trans_pix-3:trans_pix+4].mean(-1).sum(), sol_arr.mean(-1).sum() # Plot up post-fs phases to see if everything has been fixed if frq[0] == 12 * nsplit: print "======================" print " Plotting up freq: %d" % frq[0] print "======================" img_nm = './phs_plots/dfs' + np.str(frq[17]) + np.str(np.int(time.time())) +'.png' img_nmcorr = './phs_plots/dfs' + np.str(frq[17]) + np.str(np.int(time.time())) +'corr.png' plt_gains(data_fs, 0, img_name=img_nm, bad_chans=baddies) dfs_corr = correct_dfs(data_fs, np.angle(Gains[frq])[..., np.newaxis], nfeed=128) plt_gains(dfs_corr, 0, img_name=img_nmcorr, bad_chans=baddies) del dfs_corr del data_fs, a return Gains
def __init__( self, vis1, vis2, tm1, tm2, src1, src2, freqs, prods, inputs, pstns0, bsipts=None, ): """Basic initialization method""" self.adj_freqs = False # frequencies adjacent in data? (used for some tests) self.VERBOSE = False self.PATH = True # True if Pathfinder, False if CHIME self.vis1 = vis1 self.vis2 = vis2 self.tm1 = tm1 self.tm2 = tm2 self.freqs = freqs self.prods = prods self.inputs = inputs self.inprds = np.array([[self.inputs[prod[0]], self.inputs[prod[1]]] for prod in self.prods]) self.pstns0 = pstns0 self.bsipts = bsipts self.Nfr = len(freqs) self.Npr = len(prods) self.Nip = len(inputs) self.c_bslns0, self.bslns0 = self.set_bslns0() self.source1 = src1 self.source2 = src2 self.dec1 = self.source1._dec if self.source2 is not None: self.dec2 = self.source2._dec self.tt1 = (ephemeris.transit_times(self.source1, self.tm1[0], self.tm1[-1])[0] + TTCORR[self.source1.name]) if self.source2 is not None: self.tt2 = (ephemeris.transit_times(self.source2, self.tm2[0], self.tm2[-1])[0] + TTCORR[self.source2.name]) # Results place holders: self.pass_xd1, self.pass_xd2 = None, None self.pass_cont1, self.pass_cont2 = None, None self.good_prods = None self.good_freqs = None self.good_prods_cons = None self.good_freqs_cons = None # TODO: delete all incidences of self.dists_computed. Subs by # initializing here to self.xdists1 = None, etc.. self.dists_computed = False # TODO: this might not be used... # Possible to scramble expected baselines. For debug purposes only: self.bslins0_scrambled = False
def fringestop_and_sum(fn, feeds, freq, src, transposed=True, return_unfs=True, meridian=False, del_t=1500, frick=None): """ Take an input file fn and a set of feeds and return a formed beam on src. """ if transposed is True: r = andata.Reader(fn) r.freq_sel = freq X = r.read() times = r.time else: f = h5py.File(fn, 'r') times = f['index_map']['time'].value['ctime'] print "Read in data" # Get transit time for source src_trans = eph.transit_times(src, times[0]) del_phi = 1.30 * (src._dec - np.radians(eph.CHIMELATITUDE)) * np.sin( np.radians(1.988)) del_phi *= (24 * 3600.0) / (2 * np.pi) # Adjust the transit time accordingly src_trans += del_phi # Select +- del_t of transit, accounting for the mispointing t_range = np.where((times < src_trans + del_t) & (times > src_trans - del_t))[0] times = times[t_range[0]:t_range[-1]] #[offp::2] test print "Time range:", times[0], times[-1] # Generate correctly ordered corrinputs corrinput_real = gen_inp()[0] inp = np.array(corrinput_real) # Ensure vis array is in correct order (freq, prod, time) if transposed is True: data = X.vis[:, :, t_range[0]:t_range[-1]] freq = X.freq else: v = f['vis'][t_range[0]:t_range[-1], freq, :] vis = v['r'] + 1j * v['i'] data = vis.transpose()[np.newaxis] del vis freq = 800 - 400.0 / 1024 * freq freq = np.array([freq]) # autos = auto_corrs(256) # offp = (abs(data[:, autos, 0::2]).mean() > (abs(data[:, autos, 1::2]).mean())).astype(int) # data = data[..., offp::2] test data_unfs = sum_corrs(data.copy(), feeds) ra_ = eph.transit_RA(times) ra_2 = nolan_ra(times) #ra_ = ra_2.copy() if meridian is True: ra = np.ones_like(ra_) * np.degrees(src._ra) else: ra = ra_ print len(inp) dfs = tools.fringestop_pathfinder(data.copy(), ra, freq, inp, src) #dfs = fringestop_pathfinder(data.copy(), ra_2, freq, inp, src, frick=frick) # dfs = fringestop_pathfinder(data.copy(), ra_1, freq, inp, src, frick=frick) # fp = np.loadtxt('/home/connor/feed_layout_decrease.txt') # PH = fill_nolan(times, src._ra * 180.0 / np.pi, src._dec * 180.0 / np.pi, fp) dfs_sum = sum_corrs(dfs, feeds) if return_unfs is True: return dfs_sum, ra_, dfs, data else: return dfs_sum, ra_
def main(config_file=None, logging_params=DEFAULT_LOGGING): # Load config config = DEFAULTS.deepcopy() if config_file is not None: config.merge(NameSpace(load_yaml_config(config_file))) # Setup logging log.setup_logging(logging_params) logger = log.get_logger(__name__) ## Load data for flagging # Load fpga restarts time_fpga_restart = [] if config.fpga_restart_file is not None: with open(config.fpga_restart_file, 'r') as handler: for line in handler: time_fpga_restart.append( ephemeris.datetime_to_unix( ephemeris.timestr_to_datetime(line.split('_')[0]))) time_fpga_restart = np.array(time_fpga_restart) # Load housekeeping flag if config.housekeeping_file is not None: ftemp = TempData.from_acq_h5(config.housekeeping_file, datasets=["time_flag"]) else: ftemp = None # Load jump data if config.jump_file is not None: with h5py.File(config.jump_file, 'r') as handler: jump_time = handler["time"][:] jump_size = handler["jump_size"][:] else: jump_time = None jump_size = None # Load rain data if config.rain_file is not None: with h5py.File(config.rain_file, 'r') as handler: rain_ranges = handler["time_range_conservative"][:] else: rain_ranges = [] # Load data flags data_flags = {} if config.data_flags: finder.connect_database() flag_types = finder.DataFlagType.select() possible_data_flags = [] for ft in flag_types: possible_data_flags.append(ft.name) if ft.name in config.data_flags: new_data_flags = finder.DataFlag.select().where( finder.DataFlag.type == ft) data_flags[ft.name] = list(new_data_flags) # Set desired range of time start_time = (ephemeris.datetime_to_unix( datetime.datetime( *config.start_date)) if config.start_date is not None else None) end_time = (ephemeris.datetime_to_unix(datetime.datetime( *config.end_date)) if config.end_date is not None else None) ## Find gain files files = {} for src in config.sources: files[src] = sorted( glob.glob( os.path.join(config.directory, src.lower(), "%s_%s_lsd_*.h5" % ( config.prefix, src.lower(), )))) csd = {} for src in config.sources: csd[src] = np.array( [int(os.path.splitext(ff)[0][-4:]) for ff in files[src]]) for src in config.sources: logger.info("%s: %d files" % (src, len(csd[src]))) ## Remove files that occur during flag csd_flag = {} for src in config.sources: body = ephemeris.source_dictionary[src] csd_flag[src] = np.ones(csd[src].size, dtype=np.bool) for ii, cc in enumerate(csd[src][:]): ttrans = ephemeris.transit_times(body, ephemeris.csd_to_unix(cc))[0] if (start_time is not None) and (ttrans < start_time): csd_flag[src][ii] = False continue if (end_time is not None) and (ttrans > end_time): csd_flag[src][ii] = False continue # If requested, remove daytime transits if not config.include_daytime.get( src, config.include_daytime.default) and daytime_flag( ttrans)[0]: logger.info("%s CSD %d: daytime transit" % (src, cc)) csd_flag[src][ii] = False continue # Remove transits during HKP drop out if ftemp is not None: itemp = np.flatnonzero( (ftemp.time[:] >= (ttrans - config.transit_window)) & (ftemp.time[:] <= (ttrans + config.transit_window))) tempflg = ftemp['time_flag'][itemp] if (tempflg.size == 0) or ((np.sum(tempflg, dtype=np.float32) / float(tempflg.size)) < 0.50): logger.info("%s CSD %d: no housekeeping" % (src, cc)) csd_flag[src][ii] = False continue # Remove transits near jumps if jump_time is not None: njump = np.sum((jump_size > config.min_jump_size) & (jump_time > (ttrans - config.jump_window)) & (jump_time < ttrans)) if njump > config.max_njump: logger.info("%s CSD %d: %d jumps before" % (src, cc, njump)) csd_flag[src][ii] = False continue # Remove transits near rain for rng in rain_ranges: if (((ttrans - config.transit_window) <= rng[1]) and ((ttrans + config.transit_window) >= rng[0])): logger.info("%s CSD %d: during rain" % (src, cc)) csd_flag[src][ii] = False break # Remove transits during data flag for name, flag_list in data_flags.items(): if csd_flag[src][ii]: for flg in flag_list: if (((ttrans - config.transit_window) <= flg.finish_time) and ((ttrans + config.transit_window) >= flg.start_time)): logger.info("%s CSD %d: %s flag" % (src, cc, name)) csd_flag[src][ii] = False break # Print number of files left after flagging for src in config.sources: logger.info("%s: %d files (after flagging)" % (src, np.sum(csd_flag[src]))) ## Construct pair wise differences npair = len(config.diff_pair) shift = [nd * 24.0 * 3600.0 for nd in config.nday_shift] calmap = [] calpair = [] for (tsrc, csrc), sh in zip(config.diff_pair, shift): body_test = ephemeris.source_dictionary[tsrc] body_cal = ephemeris.source_dictionary[csrc] for ii, cc in enumerate(csd[tsrc]): if csd_flag[tsrc][ii]: test_transit = ephemeris.transit_times( body_test, ephemeris.csd_to_unix(cc))[0] cal_transit = ephemeris.transit_times(body_cal, test_transit + sh)[0] cal_csd = int(np.fix(ephemeris.unix_to_csd(cal_transit))) ttrans = np.sort([test_transit, cal_transit]) if cal_csd in csd[csrc]: jj = list(csd[csrc]).index(cal_csd) if csd_flag[csrc][jj] and not np.any( (time_fpga_restart >= ttrans[0]) & (time_fpga_restart <= ttrans[1])): calmap.append([ii, jj]) calpair.append([tsrc, csrc]) calmap = np.array(calmap) calpair = np.array(calpair) ntransit = calmap.shape[0] logger.info("%d total transit pairs" % ntransit) for ii in range(ntransit): t1 = ephemeris.transit_times( ephemeris.source_dictionary[calpair[ii, 0]], ephemeris.csd_to_unix(csd[calpair[ii, 0]][calmap[ii, 0]]))[0] t2 = ephemeris.transit_times( ephemeris.source_dictionary[calpair[ii, 1]], ephemeris.csd_to_unix(csd[calpair[ii, 1]][calmap[ii, 1]]))[0] logger.info("%s (%d) - %s (%d): %0.1f hr" % (calpair[ii, 0], csd_flag[calpair[ii, 0]][calmap[ii, 0]], calpair[ii, 1], csd_flag[calpair[ii, 1]][calmap[ii, 1]], (t1 - t2) / 3600.0)) # Determine unique diff pairs diff_name = np.array(['%s/%s' % tuple(cp) for cp in calpair]) uniq_diff, lbl_diff, cnt_diff = np.unique(diff_name, return_inverse=True, return_counts=True) ndiff = uniq_diff.size for ud, udcnt in zip(uniq_diff, cnt_diff): logger.info("%s: %d transit pairs" % (ud, udcnt)) ## Load gains inputmap = tools.get_correlator_inputs(datetime.datetime.utcnow(), correlator='chime') ninput = len(inputmap) nfreq = 1024 # Set up gain arrays gain = np.zeros((2, nfreq, ninput, ntransit), dtype=np.complex64) weight = np.zeros((2, nfreq, ninput, ntransit), dtype=np.float32) input_sort = np.zeros((2, ninput, ntransit), dtype=np.int) kcsd = np.zeros((2, ntransit), dtype=np.float32) timestamp = np.zeros((2, ntransit), dtype=np.float64) is_daytime = np.zeros((2, ntransit), dtype=np.bool) for tt in range(ntransit): for kk, (src, ind) in enumerate(zip(calpair[tt], calmap[tt])): body = ephemeris.source_dictionary[src] filename = files[src][ind] logger.info("%s: %s" % (src, filename)) temp = containers.StaticGainData.from_file(filename) freq = temp.freq[:] inputs = temp.input[:] isort = reorder_inputs(inputmap, inputs) inputs = inputs[isort] gain[kk, :, :, tt] = temp.gain[:, isort] weight[kk, :, :, tt] = temp.weight[:, isort] input_sort[kk, :, tt] = isort kcsd[kk, tt] = temp.attrs['lsd'] timestamp[kk, tt] = ephemeris.transit_times( body, ephemeris.csd_to_unix(kcsd[kk, tt]))[0] is_daytime[kk, tt] = daytime_flag(timestamp[kk, tt])[0] if np.any(isort != np.arange(isort.size)): logger.info("Input ordering has changed: %s" % ephemeris.unix_to_datetime( timestamp[kk, tt]).strftime("%Y-%m-%d")) logger.info("") inputs = np.array([(inp.id, inp.input_sn) for inp in inputmap], dtype=[('chan_id', 'u2'), ('correlator_input', 'S32')]) ## Load input flags inpflg = np.ones((2, ninput, ntransit), dtype=np.bool) min_flag_time = np.min(timestamp) - 7.0 * 24.0 * 60.0 * 60.0 max_flag_time = np.max(timestamp) + 7.0 * 24.0 * 60.0 * 60.0 flaginput_files = sorted( glob.glob( os.path.join(config.flaginput_dir, "*" + config.flaginput_suffix, "*.h5"))) if flaginput_files: logger.info("Found %d flaginput files." % len(flaginput_files)) tmp = andata.FlagInputData.from_acq_h5(flaginput_files, datasets=()) start, stop = [ int(yy) for yy in np.percentile( np.flatnonzero((tmp.time[:] >= min_flag_time) & (tmp.time[:] <= max_flag_time)), [0, 100]) ] cont = andata.FlagInputData.from_acq_h5(flaginput_files, start=start, stop=stop, datasets=['flag']) for kk in range(2): inpflg[kk, :, :] = cont.resample('flag', timestamp[kk], transpose=True) logger.info("Flaginput time offsets in minutes (pair %d):" % kk) logger.info( str( np.fix((cont.time[cont.search_update_time(timestamp[kk])] - timestamp[kk]) / 60.0).astype(np.int))) # Sort flags so they are in same order for tt in range(ntransit): for kk in range(2): inpflg[kk, :, tt] = inpflg[kk, input_sort[kk, :, tt], tt] # Do not apply input flag to phase reference for ii in config.index_phase_ref: inpflg[:, ii, :] = True ## Flag out gains with high uncertainty and frequencies with large fraction of data flagged frac_err = tools.invert_no_zero(np.sqrt(weight) * np.abs(gain)) flag = np.all((weight > 0.0) & (np.abs(gain) > 0.0) & (frac_err < config.max_uncertainty), axis=0) freq_flag = ((np.sum(flag, axis=(1, 2), dtype=np.float32) / float(np.prod(flag.shape[1:]))) > config.freq_threshold) if config.apply_rfi_mask: freq_flag &= np.logical_not(rfi.frequency_mask(freq)) flag = flag & freq_flag[:, np.newaxis, np.newaxis] good_freq = np.flatnonzero(freq_flag) logger.info("Number good frequencies %d" % good_freq.size) ## Generate flags with more conservative cuts on frequency c_flag = flag & np.all(frac_err < config.conservative.max_uncertainty, axis=0) c_freq_flag = ((np.sum(c_flag, axis=(1, 2), dtype=np.float32) / float(np.prod(c_flag.shape[1:]))) > config.conservative.freq_threshold) if config.conservative.apply_rfi_mask: c_freq_flag &= np.logical_not(rfi.frequency_mask(freq)) c_flag = c_flag & c_freq_flag[:, np.newaxis, np.newaxis] c_good_freq = np.flatnonzero(c_freq_flag) logger.info("Number good frequencies (conservative thresholds) %d" % c_good_freq.size) ## Apply input flags flag &= np.all(inpflg[:, np.newaxis, :, :], axis=0) ## Update flags based on beam flag if config.beam_flag_file is not None: dbeam = andata.BaseData.from_acq_h5(config.beam_flag_file) db_csd = np.floor(ephemeris.unix_to_csd(dbeam.index_map['time'][:])) for ii, name in enumerate(config.beam_flag_datasets): logger.info("Applying %s beam flag." % name) if not ii: db_flag = dbeam.flags[name][:] else: db_flag &= dbeam.flags[name][:] cnt = 0 for ii, dbc in enumerate(db_csd): this_csd = np.flatnonzero(np.any(kcsd == dbc, axis=0)) if this_csd.size > 0: logger.info("Beam flag for %d matches %s." % (dbc, str(kcsd[:, this_csd]))) flag[:, :, this_csd] &= db_flag[np.newaxis, :, ii, np.newaxis] cnt += 1 logger.info("Applied %0.1f percent of the beam flags" % (100.0 * cnt / float(db_csd.size), )) ## Flag inputs with large amount of missing data input_frac_flagged = ( np.sum(flag[good_freq, :, :], axis=(0, 2), dtype=np.float32) / float(good_freq.size * ntransit)) input_flag = input_frac_flagged > config.input_threshold for ii in config.index_phase_ref: logger.info("Phase reference %d has %0.3f fraction of data flagged." % (ii, input_frac_flagged[ii])) input_flag[ii] = True good_input = np.flatnonzero(input_flag) flag = flag & input_flag[np.newaxis, :, np.newaxis] logger.info("Number good inputs %d" % good_input.size) ## Calibrate gaincal = gain[0] * tools.invert_no_zero(gain[1]) frac_err_cal = np.sqrt(frac_err[0]**2 + frac_err[1]**2) count = np.sum(flag, axis=-1, dtype=np.int) stat_flag = count > config.min_num_transit ## Calculate phase amp = np.abs(gaincal) phi = np.angle(gaincal) ## Calculate polarisation groups pol_dict = {'E': 'X', 'S': 'Y'} cyl_dict = {2: 'A', 3: 'B', 4: 'C', 5: 'D'} if config.group_by_cyl: group_id = [ (inp.pol, inp.cyl) if tools.is_chime(inp) and (ii in good_input) else None for ii, inp in enumerate(inputmap) ] else: group_id = [ inp.pol if tools.is_chime(inp) and (ii in good_input) else None for ii, inp in enumerate(inputmap) ] ugroup_id = sorted([uidd for uidd in set(group_id) if uidd is not None]) ngroup = len(ugroup_id) group_list_noref = [ np.array([ gg for gg, gid in enumerate(group_id) if (gid == ugid) and gg not in config.index_phase_ref ]) for ugid in ugroup_id ] group_list = [ np.array([gg for gg, gid in enumerate(group_id) if gid == ugid]) for ugid in ugroup_id ] if config.group_by_cyl: group_str = [ "%s-%s" % (pol_dict[pol], cyl_dict[cyl]) for pol, cyl in ugroup_id ] else: group_str = [pol_dict[pol] for pol in ugroup_id] index_phase_ref = [] for gstr, igroup in zip(group_str, group_list): candidate = [ii for ii in config.index_phase_ref if ii in igroup] if len(candidate) != 1: index_phase_ref.append(None) else: index_phase_ref.append(candidate[0]) logger.info( "Phase reference: %s" % ', '.join(['%s = %s' % tpl for tpl in zip(group_str, index_phase_ref)])) ## Apply thermal correction to amplitude if config.amp_thermal.enabled: logger.info("Applying thermal correction.") # Load the temperatures tdata = TempData.from_acq_h5(config.amp_thermal.filename) index = tdata.search_sensors(config.amp_thermal.sensor)[0] temp = tdata.datasets[config.amp_thermal.field][index] temp_func = scipy.interpolate.interp1d(tdata.time, temp, **config.amp_thermal.interp) itemp = temp_func(timestamp) dtemp = itemp[0] - itemp[1] flag_func = scipy.interpolate.interp1d( tdata.time, tdata.datasets['flag'][index].astype(np.float32), **config.amp_thermal.interp) dtemp_flag = np.all(flag_func(timestamp) == 1.0, axis=0) flag &= dtemp_flag[np.newaxis, np.newaxis, :] for gstr, igroup in zip(group_str, group_list): pstr = gstr[0] thermal_coeff = np.polyval(config.amp_thermal.coeff[pstr], freq) gthermal = 1.0 + thermal_coeff[:, np.newaxis, np.newaxis] * dtemp[ np.newaxis, np.newaxis, :] amp[:, igroup, :] *= tools.invert_no_zero(gthermal) ## Compute common mode if config.subtract_common_mode_before: logger.info("Calculating common mode amplitude and phase.") cmn_amp, flag_cmn_amp = compute_common_mode(amp, flag, group_list_noref, median=False) cmn_phi, flag_cmn_phi = compute_common_mode(phi, flag, group_list_noref, median=False) # Subtract common mode (from phase only) logger.info("Subtracting common mode phase.") group_flag = np.zeros((ngroup, ninput), dtype=np.bool) for gg, igroup in enumerate(group_list): group_flag[gg, igroup] = True phi[:, igroup, :] = phi[:, igroup, :] - cmn_phi[:, gg, np.newaxis, :] for iref in index_phase_ref: if (iref is not None) and (iref in igroup): flag[:, iref, :] = flag_cmn_phi[:, gg, :] ## If requested, determine and subtract a delay template if config.fit_delay_before: logger.info("Fitting delay template.") omega = timing.FREQ_TO_OMEGA * freq tau, tau_flag, _ = construct_delay_template( omega, phi, c_flag & flag, min_num_freq_for_delay_fit=config.min_num_freq_for_delay_fit) # Compute residuals logger.info("Subtracting delay template.") phi = phi - tau[np.newaxis, :, :] * omega[:, np.newaxis, np.newaxis] ## Normalize by median over time logger.info("Calculating median amplitude and phase.") med_amp = np.zeros((nfreq, ninput, ndiff), dtype=amp.dtype) med_phi = np.zeros((nfreq, ninput, ndiff), dtype=phi.dtype) count_by_diff = np.zeros((nfreq, ninput, ndiff), dtype=np.int) stat_flag_by_diff = np.zeros((nfreq, ninput, ndiff), dtype=np.bool) def weighted_mean(yy, ww, axis=-1): return np.sum(ww * yy, axis=axis) * tools.invert_no_zero( np.sum(ww, axis=axis)) for dd in range(ndiff): this_diff = np.flatnonzero(lbl_diff == dd) this_flag = flag[:, :, this_diff] this_amp = amp[:, :, this_diff] this_amp_err = this_amp * frac_err_cal[:, :, this_diff] * this_flag.astype( np.float32) this_phi = phi[:, :, this_diff] this_phi_err = frac_err_cal[:, :, this_diff] * this_flag.astype( np.float32) count_by_diff[:, :, dd] = np.sum(this_flag, axis=-1, dtype=np.int) stat_flag_by_diff[:, :, dd] = count_by_diff[:, :, dd] > config.min_num_transit if config.weighted_mean == 2: logger.info("Calculating inverse variance weighted mean.") med_amp[:, :, dd] = weighted_mean(this_amp, tools.invert_no_zero(this_amp_err**2), axis=-1) med_phi[:, :, dd] = weighted_mean(this_phi, tools.invert_no_zero(this_phi_err**2), axis=-1) elif config.weighted_mean == 1: logger.info("Calculating uniform weighted mean.") med_amp[:, :, dd] = weighted_mean(this_amp, this_flag.astype(np.float32), axis=-1) med_phi[:, :, dd] = weighted_mean(this_phi, this_flag.astype(np.float32), axis=-1) else: logger.info("Calculating median value.") for ff in range(nfreq): for ii in range(ninput): if np.any(this_flag[ff, ii, :]): med_amp[ff, ii, dd] = wq.median( this_amp[ff, ii, :], this_flag[ff, ii, :].astype(np.float32)) med_phi[ff, ii, dd] = wq.median( this_phi[ff, ii, :], this_flag[ff, ii, :].astype(np.float32)) damp = np.zeros_like(amp) dphi = np.zeros_like(phi) for dd in range(ndiff): this_diff = np.flatnonzero(lbl_diff == dd) damp[:, :, this_diff] = amp[:, :, this_diff] * tools.invert_no_zero( med_amp[:, :, dd, np.newaxis]) - 1.0 dphi[:, :, this_diff] = phi[:, :, this_diff] - med_phi[:, :, dd, np.newaxis] # Compute common mode if not config.subtract_common_mode_before: logger.info("Calculating common mode amplitude and phase.") cmn_amp, flag_cmn_amp = compute_common_mode(damp, flag, group_list_noref, median=True) cmn_phi, flag_cmn_phi = compute_common_mode(dphi, flag, group_list_noref, median=True) # Subtract common mode (from phase only) logger.info("Subtracting common mode phase.") group_flag = np.zeros((ngroup, ninput), dtype=np.bool) for gg, igroup in enumerate(group_list): group_flag[gg, igroup] = True dphi[:, igroup, :] = dphi[:, igroup, :] - cmn_phi[:, gg, np.newaxis, :] for iref in index_phase_ref: if (iref is not None) and (iref in igroup): flag[:, iref, :] = flag_cmn_phi[:, gg, :] ## Compute RMS logger.info("Calculating RMS of amplitude and phase.") mad_amp = np.zeros((nfreq, ninput), dtype=amp.dtype) std_amp = np.zeros((nfreq, ninput), dtype=amp.dtype) mad_phi = np.zeros((nfreq, ninput), dtype=phi.dtype) std_phi = np.zeros((nfreq, ninput), dtype=phi.dtype) mad_amp_by_diff = np.zeros((nfreq, ninput, ndiff), dtype=amp.dtype) std_amp_by_diff = np.zeros((nfreq, ninput, ndiff), dtype=amp.dtype) mad_phi_by_diff = np.zeros((nfreq, ninput, ndiff), dtype=phi.dtype) std_phi_by_diff = np.zeros((nfreq, ninput, ndiff), dtype=phi.dtype) for ff in range(nfreq): for ii in range(ninput): this_flag = flag[ff, ii, :] if np.any(this_flag): std_amp[ff, ii] = np.std(damp[ff, ii, this_flag]) std_phi[ff, ii] = np.std(dphi[ff, ii, this_flag]) mad_amp[ff, ii] = 1.48625 * wq.median( np.abs(damp[ff, ii, :]), this_flag.astype(np.float32)) mad_phi[ff, ii] = 1.48625 * wq.median( np.abs(dphi[ff, ii, :]), this_flag.astype(np.float32)) for dd in range(ndiff): this_diff = this_flag & (lbl_diff == dd) if np.any(this_diff): std_amp_by_diff[ff, ii, dd] = np.std(damp[ff, ii, this_diff]) std_phi_by_diff[ff, ii, dd] = np.std(dphi[ff, ii, this_diff]) mad_amp_by_diff[ff, ii, dd] = 1.48625 * wq.median( np.abs(damp[ff, ii, :]), this_diff.astype(np.float32)) mad_phi_by_diff[ff, ii, dd] = 1.48625 * wq.median( np.abs(dphi[ff, ii, :]), this_diff.astype(np.float32)) ## Construct delay template if not config.fit_delay_before: logger.info("Fitting delay template.") omega = timing.FREQ_TO_OMEGA * freq tau, tau_flag, _ = construct_delay_template( omega, dphi, c_flag & flag, min_num_freq_for_delay_fit=config.min_num_freq_for_delay_fit) # Compute residuals logger.info("Subtracting delay template from phase.") resid = (dphi - tau[np.newaxis, :, :] * omega[:, np.newaxis, np.newaxis]) * flag.astype(np.float32) else: resid = dphi tau_count = np.sum(tau_flag, axis=-1, dtype=np.int) tau_stat_flag = tau_count > config.min_num_transit tau_count_by_diff = np.zeros((ninput, ndiff), dtype=np.int) tau_stat_flag_by_diff = np.zeros((ninput, ndiff), dtype=np.bool) for dd in range(ndiff): this_diff = np.flatnonzero(lbl_diff == dd) tau_count_by_diff[:, dd] = np.sum(tau_flag[:, this_diff], axis=-1, dtype=np.int) tau_stat_flag_by_diff[:, dd] = tau_count_by_diff[:, dd] > config.min_num_transit ## Calculate statistics of residuals std_resid = np.zeros((nfreq, ninput), dtype=phi.dtype) mad_resid = np.zeros((nfreq, ninput), dtype=phi.dtype) std_resid_by_diff = np.zeros((nfreq, ninput, ndiff), dtype=phi.dtype) mad_resid_by_diff = np.zeros((nfreq, ninput, ndiff), dtype=phi.dtype) for ff in range(nfreq): for ii in range(ninput): this_flag = flag[ff, ii, :] if np.any(this_flag): std_resid[ff, ii] = np.std(resid[ff, ii, this_flag]) mad_resid[ff, ii] = 1.48625 * wq.median( np.abs(resid[ff, ii, :]), this_flag.astype(np.float32)) for dd in range(ndiff): this_diff = this_flag & (lbl_diff == dd) if np.any(this_diff): std_resid_by_diff[ff, ii, dd] = np.std(resid[ff, ii, this_diff]) mad_resid_by_diff[ff, ii, dd] = 1.48625 * wq.median( np.abs(resid[ff, ii, :]), this_diff.astype(np.float32)) ## Calculate statistics of delay template mad_tau = np.zeros((ninput, ), dtype=phi.dtype) std_tau = np.zeros((ninput, ), dtype=phi.dtype) mad_tau_by_diff = np.zeros((ninput, ndiff), dtype=phi.dtype) std_tau_by_diff = np.zeros((ninput, ndiff), dtype=phi.dtype) for ii in range(ninput): this_flag = tau_flag[ii] if np.any(this_flag): std_tau[ii] = np.std(tau[ii, this_flag]) mad_tau[ii] = 1.48625 * wq.median(np.abs(tau[ii]), this_flag.astype(np.float32)) for dd in range(ndiff): this_diff = this_flag & (lbl_diff == dd) if np.any(this_diff): std_tau_by_diff[ii, dd] = np.std(tau[ii, this_diff]) mad_tau_by_diff[ii, dd] = 1.48625 * wq.median( np.abs(tau[ii]), this_diff.astype(np.float32)) ## Define output res = { "timestamp": { "data": timestamp, "axis": ["div", "time"] }, "is_daytime": { "data": is_daytime, "axis": ["div", "time"] }, "csd": { "data": kcsd, "axis": ["div", "time"] }, "pair_map": { "data": lbl_diff, "axis": ["time"] }, "pair_count": { "data": cnt_diff, "axis": ["pair"] }, "gain": { "data": gaincal, "axis": ["freq", "input", "time"] }, "frac_err": { "data": frac_err_cal, "axis": ["freq", "input", "time"] }, "flags/gain": { "data": flag, "axis": ["freq", "input", "time"], "flag": True }, "flags/gain_conservative": { "data": c_flag, "axis": ["freq", "input", "time"], "flag": True }, "flags/count": { "data": count, "axis": ["freq", "input"], "flag": True }, "flags/stat": { "data": stat_flag, "axis": ["freq", "input"], "flag": True }, "flags/count_by_pair": { "data": count_by_diff, "axis": ["freq", "input", "pair"], "flag": True }, "flags/stat_by_pair": { "data": stat_flag_by_diff, "axis": ["freq", "input", "pair"], "flag": True }, "med_amp": { "data": med_amp, "axis": ["freq", "input", "pair"] }, "med_phi": { "data": med_phi, "axis": ["freq", "input", "pair"] }, "flags/group_flag": { "data": group_flag, "axis": ["group", "input"], "flag": True }, "cmn_amp": { "data": cmn_amp, "axis": ["freq", "group", "time"] }, "cmn_phi": { "data": cmn_phi, "axis": ["freq", "group", "time"] }, "amp": { "data": damp, "axis": ["freq", "input", "time"] }, "phi": { "data": dphi, "axis": ["freq", "input", "time"] }, "std_amp": { "data": std_amp, "axis": ["freq", "input"] }, "std_amp_by_pair": { "data": std_amp_by_diff, "axis": ["freq", "input", "pair"] }, "mad_amp": { "data": mad_amp, "axis": ["freq", "input"] }, "mad_amp_by_pair": { "data": mad_amp_by_diff, "axis": ["freq", "input", "pair"] }, "std_phi": { "data": std_phi, "axis": ["freq", "input"] }, "std_phi_by_pair": { "data": std_phi_by_diff, "axis": ["freq", "input", "pair"] }, "mad_phi": { "data": mad_phi, "axis": ["freq", "input"] }, "mad_phi_by_pair": { "data": mad_phi_by_diff, "axis": ["freq", "input", "pair"] }, "tau": { "data": tau, "axis": ["input", "time"] }, "flags/tau": { "data": tau_flag, "axis": ["input", "time"], "flag": True }, "flags/tau_count": { "data": tau_count, "axis": ["input"], "flag": True }, "flags/tau_stat": { "data": tau_stat_flag, "axis": ["input"], "flag": True }, "flags/tau_count_by_pair": { "data": tau_count_by_diff, "axis": ["input", "pair"], "flag": True }, "flags/tau_stat_by_pair": { "data": tau_stat_flag_by_diff, "axis": ["input", "pair"], "flag": True }, "std_tau": { "data": std_tau, "axis": ["input"] }, "std_tau_by_pair": { "data": std_tau_by_diff, "axis": ["input", "pair"] }, "mad_tau": { "data": mad_tau, "axis": ["input"] }, "mad_tau_by_pair": { "data": mad_tau_by_diff, "axis": ["input", "pair"] }, "resid_phi": { "data": resid, "axis": ["freq", "input", "time"] }, "std_resid_phi": { "data": std_resid, "axis": ["freq", "input"] }, "std_resid_phi_by_pair": { "data": std_resid_by_diff, "axis": ["freq", "input", "pair"] }, "mad_resid_phi": { "data": mad_resid, "axis": ["freq", "input"] }, "mad_resid_phi_by_pair": { "data": mad_resid_by_diff, "axis": ["freq", "input", "pair"] }, } ## Create the output container logger.info("Creating StabilityData container.") data = StabilityData() data.create_index_map( "div", np.array(["numerator", "denominator"], dtype=np.string_)) data.create_index_map("pair", np.array(uniq_diff, dtype=np.string_)) data.create_index_map("group", np.array(group_str, dtype=np.string_)) data.create_index_map("freq", freq) data.create_index_map("input", inputs) data.create_index_map("time", timestamp[0, :]) logger.info("Writing datsets to container.") for name, dct in res.iteritems(): is_flag = dct.get('flag', False) if is_flag: dset = data.create_flag(name.split('/')[-1], data=dct['data']) else: dset = data.create_dataset(name, data=dct['data']) dset.attrs['axis'] = np.array(dct['axis'], dtype=np.string_) data.attrs['phase_ref'] = np.array( [iref for iref in index_phase_ref if iref is not None]) # Determine the output filename and save results start_time, end_time = ephemeris.unix_to_datetime( np.percentile(timestamp, [0, 100])) tfmt = "%Y%m%d" night_str = 'night_' if not np.any(is_daytime) else '' output_file = os.path.join( config.output_dir, "%s_%s_%sraw_stability_data.h5" % (start_time.strftime(tfmt), end_time.strftime(tfmt), night_str)) logger.info("Saving results to %s." % output_file) data.save(output_file)
def offline_point_source_calibration(file_list, source, inputmap=None, start=None, stop=None, physical_freq=None, tcorr=None, logging_params=DEFAULT_LOGGING, **kwargs): # Load config config = DEFAULTS.deepcopy() config.merge(NameSpace(kwargs)) # Setup logging log.setup_logging(logging_params) mlog = log.get_logger(__name__) mlog.info("ephemeris file: %s" % ephemeris.__file__) # Set the model to use fitter_function = utils.fit_point_source_transit model_function = utils.model_point_source_transit farg = inspect.getargspec(fitter_function) defaults = { key: val for key, val in zip(farg.args[-len(farg.defaults):], farg.defaults) } poly_deg_amp = kwargs.get('poly_deg_amp', defaults['poly_deg_amp']) poly_deg_phi = kwargs.get('poly_deg_phi', defaults['poly_deg_phi']) poly_type = kwargs.get('poly_type', defaults['poly_type']) param_name = ([ '%s_poly_amp_coeff%d' % (poly_type, cc) for cc in range(poly_deg_amp + 1) ] + [ '%s_poly_phi_coeff%d' % (poly_type, cc) for cc in range(poly_deg_phi + 1) ]) model_kwargs = [('poly_deg_amp', poly_deg_amp), ('poly_deg_phi', poly_deg_phi), ('poly_type', poly_type)] model_name = '.'.join( [getattr(model_function, key) for key in ['__module__', '__name__']]) tval = {} # Set where to evaluate gain ha_eval_str = ['raw_transit'] if config.multi_sample: ha_eval_str += ['transit', 'peak'] ha_eval = [0.0, None] fitslc = slice(1, 3) ind_eval = ha_eval_str.index(config.evaluate_gain_at) # Determine dimensions direction = ['amp', 'phi'] nparam = len(param_name) ngain = len(ha_eval_str) ndir = len(direction) # Determine frequencies data = andata.CorrData.from_acq_h5(file_list, datasets=(), start=start, stop=stop) freq = data.freq if physical_freq is not None: index_freq = np.array( [np.argmin(np.abs(ff - freq)) for ff in physical_freq]) freq_sel = utils.convert_to_slice(index_freq) freq = freq[index_freq] else: index_freq = np.arange(freq.size) freq_sel = None nfreq = freq.size # Compute flux of source inv_rt_flux_density = tools.invert_no_zero( np.sqrt(FluxCatalog[source].predict_flux(freq))) # Read in the eigenvaluess for all frequencies data = andata.CorrData.from_acq_h5(file_list, datasets=['erms', 'eval'], freq_sel=freq_sel, start=start, stop=stop) # Determine source coordinates this_csd = np.floor(ephemeris.unix_to_csd(np.median(data.time))) timestamp0 = ephemeris.transit_times(FluxCatalog[source].skyfield, ephemeris.csd_to_unix(this_csd))[0] src_ra, src_dec = ephemeris.object_coords(FluxCatalog[source].skyfield, date=timestamp0, deg=True) ra = ephemeris.lsa(data.time) ha = ra - src_ra ha = ha - (ha > 180.0) * 360.0 + (ha < -180.0) * 360.0 ha = np.radians(ha) itrans = np.argmin(np.abs(ha)) window = 0.75 * np.max(np.abs(ha)) off_source = np.abs(ha) > window mlog.info("CSD %d" % this_csd) mlog.info("Hour angle at transit (%d of %d): %0.2f deg " % (itrans, len(ha), np.degrees(ha[itrans]))) mlog.info("Hour angle off source: %0.2f deg" % np.median(np.abs(np.degrees(ha[off_source])))) src_dec = np.radians(src_dec) lat = np.radians(ephemeris.CHIMELATITUDE) # Determine division of frequencies ninput = data.ninput ntime = data.ntime nblock_freq = int(np.ceil(nfreq / float(config.nfreq_per_block))) # Determine bad inputs eps = 10.0 * np.finfo(data['erms'].dtype).eps good_freq = np.flatnonzero(np.all(data['erms'][:] > eps, axis=-1)) ind_sub_freq = good_freq[slice(0, good_freq.size, max(int(good_freq.size / 10), 1))] tmp_data = andata.CorrData.from_acq_h5(file_list, datasets=['evec'], freq_sel=ind_sub_freq, start=start, stop=stop) eps = 10.0 * np.finfo(tmp_data['evec'].dtype).eps bad_input = np.flatnonzero( np.all(np.abs(tmp_data['evec'][:, 0]) < eps, axis=(0, 2))) input_axis = tmp_data.input.copy() del tmp_data # Query layout database for correlator inputs if inputmap is None: inputmap = tools.get_correlator_inputs( datetime.datetime.utcfromtimestamp(data.time[itrans]), correlator='chime') inputmap = tools.reorder_correlator_inputs(input_axis, inputmap) tools.change_chime_location(rotation=config.telescope_rotation) # Determine x and y pol index xfeeds = np.array([ idf for idf, inp in enumerate(inputmap) if (idf not in bad_input) and tools.is_array_x(inp) ]) yfeeds = np.array([ idf for idf, inp in enumerate(inputmap) if (idf not in bad_input) and tools.is_array_y(inp) ]) nfeed = xfeeds.size + yfeeds.size pol = [yfeeds, xfeeds] polstr = ['Y', 'X'] npol = len(pol) neigen = min(max(npol, config.neigen), data['eval'].shape[1]) phase_ref = config.phase_reference_index phase_ref_by_pol = [ pol[pp].tolist().index(phase_ref[pp]) for pp in range(npol) ] # Calculate dynamic range eval0_off_source = np.median(data['eval'][:, 0, off_source], axis=-1) dyn = data['eval'][:, 1, :] * tools.invert_no_zero( eval0_off_source[:, np.newaxis]) # Determine frequencies to mask not_rfi = np.ones((nfreq, 1), dtype=np.bool) if config.mask_rfi is not None: for frng in config.mask_rfi: not_rfi[:, 0] &= ((freq < frng[0]) | (freq > frng[1])) mlog.info("%0.1f percent of frequencies available after masking RFI." % (100.0 * np.sum(not_rfi, dtype=np.float32) / float(nfreq), )) #dyn_flg = utils.contiguous_flag(dyn > config.dyn_rng_threshold, centre=itrans) if source in config.dyn_rng_threshold: dyn_rng_threshold = config.dyn_rng_threshold[source] else: dyn_rng_threshold = config.dyn_rng_threshold.default mlog.info("Dynamic range threshold set to %0.1f." % dyn_rng_threshold) dyn_flg = dyn > dyn_rng_threshold # Calculate fit flag fit_flag = np.zeros((nfreq, npol, ntime), dtype=np.bool) for pp in range(npol): mlog.info("Dynamic Range Nsample, Pol %d: %s" % (pp, ','.join([ "%d" % xx for xx in np.percentile(np.sum(dyn_flg, axis=-1), [25, 50, 75, 100]) ]))) if config.nsigma1 is None: fit_flag[:, pp, :] = dyn_flg & not_rfi else: fit_window = config.nsigma1 * np.radians( utils.get_window(freq, pol=polstr[pp], dec=src_dec, deg=True)) win_flg = np.abs(ha)[np.newaxis, :] <= fit_window[:, np.newaxis] fit_flag[:, pp, :] = (dyn_flg & win_flg & not_rfi) # Calculate base error base_err = data['erms'][:, np.newaxis, :] # Check for sign flips ref_resp = andata.CorrData.from_acq_h5(file_list, datasets=['evec'], input_sel=config.eigen_reference, freq_sel=freq_sel, start=start, stop=stop)['evec'][:, 0:neigen, 0, :] sign0 = 1.0 - 2.0 * (ref_resp.real < 0.0) # Check that we have the correct reference feed if np.any(np.abs(ref_resp.imag) > 0.0): ValueError("Reference feed %d is incorrect." % config.eigen_reference) del ref_resp # Save index_map results = {} results['model'] = model_name results['param'] = param_name results['freq'] = data.index_map['freq'][:] results['input'] = input_axis results['eval'] = ha_eval_str results['dir'] = direction for key, val in model_kwargs: results[key] = val # Initialize numpy arrays to hold results if config.return_response: results['response'] = np.zeros((nfreq, ninput, ntime), dtype=np.complex64) results['response_err'] = np.zeros((nfreq, ninput, ntime), dtype=np.float32) results['fit_flag'] = fit_flag results['ha_axis'] = ha results['ra'] = ra else: results['gain_eval'] = np.zeros((nfreq, ninput, ngain), dtype=np.complex64) results['weight_eval'] = np.zeros((nfreq, ninput, ngain), dtype=np.float32) results['frac_gain_err'] = np.zeros((nfreq, ninput, ngain, ndir), dtype=np.float32) results['parameter'] = np.zeros((nfreq, ninput, nparam), dtype=np.float32) results['parameter_err'] = np.zeros((nfreq, ninput, nparam), dtype=np.float32) results['index_eval'] = np.full((nfreq, ninput), -1, dtype=np.int8) results['gain'] = np.zeros((nfreq, ninput), dtype=np.complex64) results['weight'] = np.zeros((nfreq, ninput), dtype=np.float32) results['ndof'] = np.zeros((nfreq, ninput, ndir), dtype=np.float32) results['chisq'] = np.zeros((nfreq, ninput, ndir), dtype=np.float32) results['timing'] = np.zeros((nfreq, ninput), dtype=np.complex64) # Initialize metric like variables results['runtime'] = np.zeros((nblock_freq, 2), dtype=np.float64) # Compute distances dist = tools.get_feed_positions(inputmap) for pp, feeds in enumerate(pol): dist[feeds, :] -= dist[phase_ref[pp], np.newaxis, :] # Loop over frequency blocks for gg in range(nblock_freq): mlog.info("Frequency block %d of %d." % (gg, nblock_freq)) fstart = gg * config.nfreq_per_block fstop = min((gg + 1) * config.nfreq_per_block, nfreq) findex = np.arange(fstart, fstop) ngroup = findex.size freq_sel = utils.convert_to_slice(index_freq[findex]) timeit_start_gg = time.time() # if config.return_response: gstart = start gstop = stop tslc = slice(0, ntime) else: good_times = np.flatnonzero(np.any(fit_flag[findex], axis=(0, 1))) if good_times.size == 0: continue gstart = int(np.min(good_times)) gstop = int(np.max(good_times)) + 1 tslc = slice(gstart, gstop) gstart += start gstop += start hag = ha[tslc] itrans = np.argmin(np.abs(hag)) # Load eigenvectors. nudata = andata.CorrData.from_acq_h5( file_list, datasets=['evec', 'vis', 'flags/vis_weight'], apply_gain=False, freq_sel=freq_sel, start=gstart, stop=gstop) # Save time to load data results['runtime'][gg, 0] = time.time() - timeit_start_gg timeit_start_gg = time.time() mlog.info("Time to load (per frequency): %0.3f sec" % (results['runtime'][gg, 0] / ngroup, )) # Loop over polarizations for pp, feeds in enumerate(pol): # Get timing correction if tcorr is not None: tgain = tcorr.get_gain(nudata.freq, nudata.input[feeds], nudata.time) tgain *= tgain[:, phase_ref_by_pol[pp], np.newaxis, :].conj() tgain_transit = tgain[:, :, itrans].copy() tgain *= tgain_transit[:, :, np.newaxis].conj() # Create the polarization masking vector P = np.zeros((1, ninput, 1), dtype=np.float64) P[:, feeds, :] = 1.0 # Loop over frequencies for gff, ff in enumerate(findex): flg = fit_flag[ff, pp, tslc] if (2 * int(np.sum(flg))) < (nparam + 1) and not config.return_response: continue # Normalize by eigenvalue and correct for pi phase flips in process. resp = (nudata['evec'][gff, 0:neigen, :, :] * np.sqrt(data['eval'][ff, 0:neigen, np.newaxis, tslc]) * sign0[ff, :, np.newaxis, tslc]) # Rotate to single-pol response # Move time to first axis for the matrix multiplication invL = tools.invert_no_zero( np.rollaxis(data['eval'][ff, 0:neigen, np.newaxis, tslc], -1, 0)) UT = np.rollaxis(resp, -1, 0) U = np.swapaxes(UT, -1, -2) mu, vp = np.linalg.eigh(np.matmul(UT.conj(), P * U)) rsign0 = (1.0 - 2.0 * (vp[:, 0, np.newaxis, :].real < 0.0)) resp = mu[:, np.newaxis, :] * np.matmul(U, rsign0 * vp * invL) # Extract feeds of this pol # Transpose so that time is back to last axis resp = resp[:, feeds, -1].T # Compute error on response dataflg = ((nudata.weight[gff, feeds, :] > 0.0) & np.isfinite(nudata.weight[gff, feeds, :])).astype( np.float32) resp_err = dataflg * base_err[ff, :, tslc] * np.sqrt( nudata.vis[gff, feeds, :].real) * tools.invert_no_zero( np.sqrt(mu[np.newaxis, :, -1])) # Reference to specific input resp *= np.exp( -1.0J * np.angle(resp[phase_ref_by_pol[pp], np.newaxis, :])) # Apply timing correction if tcorr is not None: resp *= tgain[gff] results['timing'][ff, feeds] = tgain_transit[gff] # Fringestop lmbda = scipy.constants.c * 1e-6 / nudata.freq[gff] resp *= tools.fringestop_phase( hag[np.newaxis, :], lat, src_dec, dist[feeds, 0, np.newaxis] / lmbda, dist[feeds, 1, np.newaxis] / lmbda) # Normalize by source flux resp *= inv_rt_flux_density[ff] resp_err *= inv_rt_flux_density[ff] # If requested, reference phase to the median value if config.med_phase_ref: phi0 = np.angle(resp[:, itrans, np.newaxis]) resp *= np.exp(-1.0J * phi0) resp *= np.exp( -1.0J * np.median(np.angle(resp), axis=0, keepdims=True)) resp *= np.exp(1.0J * phi0) # Check if return_response flag was set by user if not config.return_response: if config.multi_sample: moving_window = config.nsigma2 and config.nsigma2 * np.radians( utils.get_window(nudata.freq[gff], pol=polstr[pp], dec=src_dec, deg=True)) # Loop over inputs for pii, ii in enumerate(feeds): is_good = flg & (np.abs(resp[pii, :]) > 0.0) & (resp_err[pii, :] > 0.0) # Set the intial gains based on raw response at transit if is_good[itrans]: results['gain_eval'][ff, ii, 0] = tools.invert_no_zero( resp[pii, itrans]) results['frac_gain_err'][ff, ii, 0, :] = ( resp_err[pii, itrans] * tools.invert_no_zero( np.abs(resp[pii, itrans]))) results['weight_eval'][ff, ii, 0] = 0.5 * ( np.abs(resp[pii, itrans])**2 * tools.invert_no_zero(resp_err[pii, itrans]))**2 results['index_eval'][ff, ii] = 0 results['gain'][ff, ii] = results['gain_eval'][ff, ii, 0] results['weight'][ff, ii] = results['weight_eval'][ff, ii, 0] # Exit if not performing multi time sample fit if not config.multi_sample: continue if (2 * int(np.sum(is_good))) < (nparam + 1): continue try: param, param_err, gain, gain_err, ndof, chisq, tval = fitter_function( hag[is_good], resp[pii, is_good], resp_err[pii, is_good], ha_eval, window=moving_window, tval=tval, **config.fit) except Exception as rex: if config.verbose: mlog.info( "Frequency %0.2f, Feed %d failed with error: %s" % (nudata.freq[gff], ii, rex)) continue # Check for nan wfit = (np.abs(gain) * tools.invert_no_zero(np.abs(gain_err)))**2 if np.any(~np.isfinite(np.abs(gain))) or np.any( ~np.isfinite(wfit)): continue # Save to results using the convention that you should *multiply* the visibilites by the gains results['gain_eval'][ ff, ii, fitslc] = tools.invert_no_zero(gain) results['frac_gain_err'][ff, ii, fitslc, 0] = gain_err.real results['frac_gain_err'][ff, ii, fitslc, 1] = gain_err.imag results['weight_eval'][ff, ii, fitslc] = wfit results['parameter'][ff, ii, :] = param results['parameter_err'][ff, ii, :] = param_err results['ndof'][ff, ii, :] = ndof results['chisq'][ff, ii, :] = chisq # Check if the fit was succesful and update the gain evaluation index appropriately if np.all((chisq / ndof.astype(np.float32) ) <= config.chisq_per_dof_threshold): results['index_eval'][ff, ii] = ind_eval results['gain'][ff, ii] = results['gain_eval'][ ff, ii, ind_eval] results['weight'][ff, ii] = results['weight_eval'][ ff, ii, ind_eval] else: # Return response only (do not fit model) results['response'][ff, feeds, :] = resp results['response_err'][ff, feeds, :] = resp_err # Save time to fit data results['runtime'][gg, 1] = time.time() - timeit_start_gg mlog.info("Time to fit (per frequency): %0.3f sec" % (results['runtime'][gg, 1] / ngroup, )) # Clean up del nudata gc.collect() # Print total run time mlog.info("TOTAL TIME TO LOAD: %0.3f min" % (np.sum(results['runtime'][:, 0]) / 60.0, )) mlog.info("TOTAL TIME TO FIT: %0.3f min" % (np.sum(results['runtime'][:, 1]) / 60.0, )) # Set the best estimate of the gain if not config.return_response: flag = results['index_eval'] >= 0 gain = results['gain'] # Compute amplitude amp = np.abs(gain) # Hard cutoffs on the amplitude med_amp = np.median(amp[flag]) min_amp = med_amp * config.min_amp_scale_factor max_amp = med_amp * config.max_amp_scale_factor flag &= ((amp >= min_amp) & (amp <= max_amp)) # Flag outliers in amplitude for each frequency for pp, feeds in enumerate(pol): med_amp_by_pol = np.zeros(nfreq, dtype=np.float32) sig_amp_by_pol = np.zeros(nfreq, dtype=np.float32) for ff in range(nfreq): this_flag = flag[ff, feeds] if np.any(this_flag): med, slow, shigh = utils.estimate_directional_scale( amp[ff, feeds[this_flag]]) lower = med - config.nsigma_outlier * slow upper = med + config.nsigma_outlier * shigh flag[ff, feeds] &= ((amp[ff, feeds] >= lower) & (amp[ff, feeds] <= upper)) med_amp_by_pol[ff] = med sig_amp_by_pol[ff] = 0.5 * (shigh - slow) / np.sqrt( np.sum(this_flag, dtype=np.float32)) if config.nsigma_med_outlier: med_flag = med_amp_by_pol > 0.0 not_outlier = flag_outliers(med_amp_by_pol, med_flag, window=config.window_med_outlier, nsigma=config.nsigma_med_outlier) flag[:, feeds] &= not_outlier[:, np.newaxis] mlog.info("Pol %s: %d frequencies are outliers." % (polstr[pp], np.sum(~not_outlier & med_flag, dtype=np.int))) # Determine bad frequencies flag_freq = (np.sum(flag, axis=1, dtype=np.float32) / float(ninput)) > config.threshold_good_freq good_freq = np.flatnonzero(flag_freq) # Determine bad inputs fraction_good = np.sum(flag[good_freq, :], axis=0, dtype=np.float32) / float(good_freq.size) flag_input = fraction_good > config.threshold_good_input # Finalize flag flag &= (flag_freq[:, np.newaxis] & flag_input[np.newaxis, :]) # Interpolate gains interp_gain, interp_weight = interpolate_gain( freq, gain, results['weight'], flag=flag, length_scale=config.interpolation_length_scale, mlog=mlog) # Save gains to object results['flag'] = flag results['gain'] = interp_gain results['weight'] = interp_weight # Return results return results
def solve_ps_transit(filename, corrs, feeds, inp, src, nfreq=1024, transposed=False, nfeed=128): """ Function that fringestops time slice where point source is in the beam, takes all correlations for a given polarization, and then eigendecomposes the correlation matrix freq by freq after removing the fpga phases. It will also plot intermediate steps to verify the phase solution. Parameters ---------- filename : np.str Full-path filename corrs : list List of correlations to use in solver feeds : list List of feeds to use inp : Correlator inputs (output of ch_util.tools.get_correlator_inputs) src : ephem.FixedBody Source to calibrate off of. e.g. ch_util.ephemeris.TauA Returns ------- Gains : np.array Complex gain array (nfreq, nfeed) """ nsplit = 32 # Number of freq chunks to divide nfreq into del_t = 800 f = h5py.File(filename, 'r') # Add half an integration time to each. Hack. times = f['index_map']['time'].value['ctime'] + 10.50 src_trans = eph.transit_times(src, times[0]) # try to account for differential arrival time from # cylinder rotation. del_phi = (src._dec - np.radians(eph.CHIMELATITUDE)) \ * np.sin(np.radians(1.988)) del_phi *= (24 * 3600.0) / (2 * np.pi) # Adjust the transit time accordingly src_trans += del_phi # Select +- del_t of transit, accounting for the mispointing t_range = np.where((times < src_trans + del_t) & (times > src_trans - del_t))[0] print "\n...... This data is from %s starting at RA: %f ...... \n" \ % (eph.unix_to_datetime(times[0]), eph.transit_RA(times[0])) assert (len(t_range) > 0), "Source is not in this acq" # Create gains array to fill in solution Gains = np.zeros([nfreq, nfeed], np.complex128) print "Starting the solver" times = times[t_range[0]:t_range[-1]] k = 0 # Start at a strong freq channel that can be plotted # and from which we can find the noise source on-sample for i in range(12, nsplit) + range(0, 12): k += 1 # Divides the arrays up into nfreq / nsplit freq chunks and solves those frq = range(i * nfreq // nsplit, (i + 1) * nfreq // nsplit) print " %d:%d \n" % (frq[0], frq[-1]) # Read in time and freq slice if data has already been transposed if transposed is True: v = f['vis'][frq[0]:frq[-1] + 1, corrs, :] v = v[..., t_range[0]:t_range[-1]] vis = v['r'] + 1j * v['i'] if k == 1: autos = auto_corrs(nfeed) offp = (abs(vis[:, autos, 0::2]).mean() > \ (abs(vis[:, autos, 1::2]).mean())).astype(int) times = times[offp::2] vis = vis[..., offp::2] gg = f['gain_coeff'][frq[0]:frq[-1] + 1, feeds, t_range[0]:t_range[-1]][..., offp::2] gain_coeff = gg['r'] + 1j * gg['i'] del gg # Read in time and freq slice if data has not yet been transposed if transposed is False: print "TRANSPOSED V OF CODE DOESN'T WORK YET!" v = f['vis'][t_range[0]:t_range[-1]:2, frq[0]:frq[-1] + 1, corrs] vis = v['r'][:] + 1j * v['i'][:] del v gg = f['gain_coeff'][0, frq[0]:frq[-1] + 1, feeds] gain_coeff = gg['r'][:] + 1j * gg['i'][:] vis = vis[..., offp::2] vis = np.transpose(vis, (1, 2, 0)) # Remove fpga gains from data vis = remove_fpga_gains(vis, gain_coeff, nfeed=nfeed, triu=False) # Remove offset from galaxy vis -= 0.5 * (vis[..., 0] + vis[..., -1])[..., np.newaxis] # Get physical freq for fringestopper freq_MHZ = 800.0 - np.array(frq) / 1024.0 * 400. baddies = np.where(np.isnan(tools.get_feed_positions(inp)[:, 0]))[0] a, b, c = select_corrs(baddies, nfeed=128) vis[:, a + b] = 0.0 # Fringestop to location of "src" data_fs = tools.fringestop_pathfinder(vis, eph.transit_RA(times), freq_MHZ, inp, src) del vis dr, sol_arr = solve_gain(data_fs) # Find index of point source transit drlist = np.argmax(dr, axis=-1) # If multiple freq channels are zerod, the trans_pix # will end up being 0. This is bad, so ensure that # you are only looking for non-zero transit pixels. drlist = [x for x in drlist if x != 0] trans_pix = np.argmax(np.bincount(drlist)) assert trans_pix != 0.0 Gains[frq] = sol_arr[..., trans_pix - 3:trans_pix + 4].mean(-1) zz = h5py.File('data' + str(i) + '.hdf5', 'w') zz.create_dataset('data', data=dr) zz.close() print "%f, %d Nans out of %d" % (np.isnan(sol_arr).sum(), np.isnan(Gains[frq]).sum(), np.isnan(Gains[frq]).sum()) print trans_pix, sol_arr[..., trans_pix - 3:trans_pix + 4].mean(-1).sum(), sol_arr.mean(-1).sum() # Plot up post-fs phases to see if everything has been fixed if frq[0] == 12 * nsplit: print "======================" print " Plotting up freq: %d" % frq[0] print "======================" img_nm = './phs_plots/dfs' + np.str(frq[17]) + np.str( np.int(time.time())) + '.png' img_nmcorr = './phs_plots/dfs' + np.str(frq[17]) + np.str( np.int(time.time())) + 'corr.png' plt_gains(data_fs, 0, img_name=img_nm, bad_chans=baddies) dfs_corr = correct_dfs(data_fs, np.angle(Gains[frq])[..., np.newaxis], nfeed=128) plt_gains(dfs_corr, 0, img_name=img_nmcorr, bad_chans=baddies) del dfs_corr del data_fs, a return Gains
def main(config_file=None, logging_params=DEFAULT_LOGGING): # Setup logging log.setup_logging(logging_params) mlog = log.get_logger(__name__) # Set config config = DEFAULTS.deepcopy() if config_file is not None: config.merge(NameSpace(load_yaml_config(config_file))) # Create transit tracker source_list = FluxCatalog.sort( ) if not config.source_list else config.source_list cal_list = [ name for name, obj in FluxCatalog.iteritems() if (obj.dec >= config.min_dec) and ( obj.predict_flux(config.freq_nominal) >= config.min_flux) and ( name in source_list) ] if not cal_list: raise RuntimeError("No calibrators found.") # Sort list by flux at nominal frequency cal_list.sort( key=lambda name: FluxCatalog[name].predict_flux(config.freq_nominal)) # Add to transit tracker transit_tracker = containers.TransitTrackerOffline( nsigma=config.nsigma_source, extend_night=config.extend_night) for name in cal_list: transit_tracker[name] = FluxCatalog[name].skyfield mlog.info("Initializing offline point source processing.") search_time = config.start_time or 0 # Find all calibration files all_files = sorted( glob.glob( os.path.join(config.acq_dir, '*' + config.correlator + config.acq_suffix, '*.h5'))) if not all_files: return # Remove files whose last modified time is before the time of the most recent update all_files = [ ff for ff in all_files if (os.path.getmtime(ff) > search_time) ] if not all_files: return # Remove files that are currently locked all_files = [ ff for ff in all_files if not os.path.isfile(os.path.splitext(ff)[0] + '.lock') ] if not all_files: return # Add files to transit tracker for ff in all_files: transit_tracker.add_file(ff) # Extract point source transits ready for analysis all_transits = transit_tracker.get_transits() # Create dictionary to hold results h5_psrc_fit = {} inputmap = None # Loop over transits for transit in all_transits: src, csd, is_day, files, start, stop = transit # Discard any point sources with unusual csd value if (csd < config.min_csd) or (csd > config.max_csd): continue # Discard any point sources transiting during the day if is_day > config.process_daytime: continue mlog.info( 'Processing %s transit on CSD %d (%d files, %d time samples)' % (src, csd, len(files), stop - start + 1)) # Load inputmap if inputmap is None: if config.inputmap is None: inputmap = tools.get_correlator_inputs( ephemeris.unix_to_datetime(ephemeris.csd_to_unix(csd)), correlator=config.correlator) else: with open(config.inputmap, 'r') as handler: inputmap = pickle.load(handler) # Grab the timing correction for this transit tcorr = None if config.apply_timing: if config.timing_glob is not None: mlog.info( "Loading timing correction from extended timing solutions." ) timing_files = sorted(glob.glob(config.timing_glob)) if timing_files: try: tcorr = search_extended_timing_solutions( timing_files, ephemeris.csd_to_unix(csd)) except Exception as e: mlog.error( 'search_extended_timing_solutions failed with error: %s' % e) else: mlog.info(str(tcorr)) if tcorr is None: mlog.info( "Loading timing correction from chimetiming acquisitions.") try: tcorr = timing.load_timing_correction( files, start=start, stop=stop, window=config.timing_window, instrument=config.correlator) except Exception as e: mlog.error( 'timing.load_timing_correction failed with error: %s' % e) mlog.warning( 'No timing correction applied to %s transit on CSD %d.' % (src, csd)) else: mlog.info(str(tcorr)) # Call the main routine to process data try: outdct = offline_cal.offline_point_source_calibration( files, src, start=start, stop=stop, inputmap=inputmap, tcorr=tcorr, logging_params=logging_params, **config.analysis.as_dict()) except Exception as e: msg = 'offline_cal.offline_point_source_calibration failed with error: %s' % e mlog.error(msg) continue #raise RuntimeError(msg) # Find existing gain files for this particular point source if src not in h5_psrc_fit: output_files = find_files(config, psrc=src) if output_files is not None: output_files = output_files[-1] mlog.info('Writing %s transit on CSD %d to existing file %s.' % (src, csd, output_files)) h5_psrc_fit[src] = containers.PointSourceWriter( src, output_file=output_files, output_dir=config.output_dir, output_suffix=point_source_name_to_file_suffix(src), instrument=config.correlator, max_file_size=config.max_file_size, max_num=config.max_num_time, memory_size=0) # Associate this gain calibration to the transit time this_time = ephemeris.transit_times(FluxCatalog[src].skyfield, ephemeris.csd_to_unix(csd))[0] outdct['csd'] = csd outdct['is_daytime'] = is_day outdct['acquisition'] = os.path.basename(os.path.dirname(files[0])) # Write to output file mlog.info('Writing to disk results from %s transit on CSD %d.' % (src, csd)) h5_psrc_fit[src].write(this_time, **outdct) # Dump an individual file for this point source transit mlog.info('Dumping to disk single file for %s transit on CSD %d.' % (src, csd)) dump_dir = os.path.join(config.output_dir, 'point_source_gains') containers.mkdir(dump_dir) dump_file = os.path.join(dump_dir, '%s_csd_%d.h5' % (src.lower(), csd)) h5_psrc_fit[src].dump(dump_file, datasets=[ 'csd', 'acquisition', 'is_daytime', 'gain', 'weight', 'timing', 'model' ]) mlog.info('Finished analysis of %s transit on CSD %d.' % (src, csd))
def fs_from_file(filename, frq, src, del_t=900, transposed=True, subtract_avg=False): f = h5py.File(filename, 'r') times = f['index_map']['time'].value['ctime'] + 10.6 src_trans = eph.transit_times(src, times[0]) # try to account for differential arrival time from cylinder rotation. del_phi = (src._dec - np.radians(eph.CHIMELATITUDE)) * np.sin( np.radians(1.988)) del_phi *= (24 * 3600.0) / (2 * np.pi) # Adjust the transit time accordingly src_trans += del_phi # Select +- del_t of transit, accounting for the mispointing t_range = np.where((times < src_trans + del_t) & (times > src_trans - del_t))[0] times = times[t_range[0]:t_range[-1]] #[offp::2] test print "Time range:", times[0], times[-1] print "\n...... This data is from %s starting at RA: %f ...... \n" \ % (eph.unix_to_datetime(times[0]), eph.transit_RA(times[0])) if transposed is True: v = f['vis'][frq[0]:frq[-1] + 1, :] v = v[..., t_range[0]:t_range[-1]] vis = v['r'] + 1j * v['i'] del v # Read in time and freq slice if data has not yet been transposed if transposed is False: v = f['vis'][t_range[0]:t_range[-1], frq[0]:frq[-1] + 1, :] vis = v['r'][:] + 1j * v['i'][:] del v vis = np.transpose(vis, (1, 2, 0)) inp = gen_inp()[0] # Remove offset from galaxy if subtract_avg is True: vis -= 0.5 * (vis[..., 0] + vis[..., -1])[..., np.newaxis] freq_MHZ = 800.0 - np.array(frq) / 1024.0 * 400. print len(inp) baddies = np.where(np.isnan(tools.get_feed_positions(inp)[:, 0]))[0] # Fringestop to location of "src" data_fs = tools.fringestop_pathfinder(vis, eph.transit_RA(times), freq_MHZ, inp, src) # data_fs = fringestop_pathfinder(vis, eph.transit_RA(times), freq_MHZ, inp, src) return data_fs