def _finalize_transit(self): """Concatenate grouped time streams for the currrent transit.""" # Find where transit starts and ends if len(self.tstreams) == 0 or self.cur_transit is None: self.log.info("Did not find any transits.") return None self.log.debug( "Finalising transit for {}...".format( ephem.unix_to_datetime(self.cur_transit) ) ) all_t = np.concatenate([ts.time for ts in self.tstreams]) start_ind = int(np.argmin(np.abs(all_t - self.start_t))) stop_ind = int(np.argmin(np.abs(all_t - self.end_t))) # Save list of filenames filenames = [ts.attrs["filename"] for ts in self.tstreams] dt = self.tstreams[0].time[1] - self.tstreams[0].time[0] if dt <= 0: self.log.warning( "Time steps are not positive definite: dt={:.3f}".format(dt) + " Skipping." ) ts = None if stop_ind - start_ind > int(self.min_span / 360.0 * SIDEREAL_DAY_SEC / dt): if len(self.tstreams) > 1: # Concatenate timestreams ts = tod.concatenate(self.tstreams, start=start_ind, stop=stop_ind) else: ts = self.tstreams[0] _, dec = ephem.object_coords( self.src, all_t[0], deg=True, obs=self.observer ) ts.attrs["dec"] = dec ts.attrs["source_name"] = self.source ts.attrs["transit_time"] = self.cur_transit ts.attrs["observation_id"] = self.obs_id ts.attrs["tag"] = "{}_{:0>4d}_{}".format( self.source, self.obs_id, ephem.unix_to_datetime(self.cur_transit).strftime("%Y%m%dT%H%M%S"), ) ts.attrs["archivefiles"] = filenames else: self.log.info("Transit too short. Skipping.") ts = None self.tstreams = [] self.cur_transit = None return ts
def _cut_non_chime(data, visi, chan_array, inputs=None): """ Remove non CHIME channels (noise injection, RFI antenna, 26m, etc...) from visibility. Also remove channels marked as powered-off in layout DB. """ # Map of channels to corr. inputs: input_map = data.input tmstp = data.index_map["time"]["ctime"] # time stamp # Datetime halfway through data: half_time = ch_eph.unix_to_datetime(tmstp[int(len(tmstp) // 2)]) # Get information on correlator inputs, if not already supplied if inputs is None: inputs = tools.get_correlator_inputs(half_time) # Reorder inputs to have sema order as input map (and data) inputs = tools.reorder_correlator_inputs(input_map, inputs) # Get noise source channel index: # Test if inputs are attached to CHIME antenna and powered on: pwds = tools.is_chime_on(inputs) for ii in range(len(inputs)): # if ( (not tools.is_chime(inputs[ii])) if (not pwds[ii]) and (ii in chan_array): # Remove non-CHIME-on channels from visibility matrix... idx = np.where(chan_array == ii)[0][0] # index of channel visi = np.delete(visi, idx, axis=0) # ...and from product array: chan_array = np.delete(chan_array, idx, axis=0) return visi, chan_array
def fs_from_file(filename, frq, src, del_t=900, transposed=True, subtract_avg=False): f = h5py.File(filename, 'r') times = f['index_map']['time'].value['ctime'] + 10.6 src_trans = eph.transit_times(src, times[0]) # try to account for differential arrival time from cylinder rotation. del_phi = (src._dec - np.radians(eph.CHIMELATITUDE)) * np.sin(np.radians(1.988)) del_phi *= (24 * 3600.0) / (2 * np.pi) # Adjust the transit time accordingly src_trans += del_phi # Select +- del_t of transit, accounting for the mispointing t_range = np.where((times < src_trans + del_t) & (times > src_trans - del_t))[0] times = times[t_range[0]:t_range[-1]]#[offp::2] test print "Time range:", times[0], times[-1] print "\n...... This data is from %s starting at RA: %f ...... \n" \ % (eph.unix_to_datetime(times[0]), eph.transit_RA(times[0])) if transposed is True: v = f['vis'][frq[0]:frq[-1]+1, :] v = v[..., t_range[0]:t_range[-1]] vis = v['r'] + 1j * v['i'] del v # Read in time and freq slice if data has not yet been transposed if transposed is False: v = f['vis'][t_range[0]:t_range[-1], frq[0]:frq[-1]+1, :] vis = v['r'][:] + 1j * v['i'][:] del v vis = np.transpose(vis, (1, 2, 0)) inp = gen_inp()[0] # Remove offset from galaxy if subtract_avg is True: vis -= 0.5 * (vis[..., 0] + vis[..., -1])[..., np.newaxis] freq_MHZ = 800.0 - np.array(frq) / 1024.0 * 400. print len(inp) baddies = np.where(np.isnan(tools.get_feed_positions(inp)[:, 0]))[0] # Fringestop to location of "src" data_fs = tools.fringestop_pathfinder(vis, eph.transit_RA(times), freq_MHZ, inp, src) # data_fs = fringestop_pathfinder(vis, eph.transit_RA(times), freq_MHZ, inp, src) return data_fs
def print_timestamp(fn): src_dict = {'CasA': eph.CasA, 'TauA': eph.TauA, 'CygA': eph.CygA, 'VirA': eph.VirA, '1929': 292.0, '0329': 54.0} try: f = h5py.File(fn, 'r') except IOError: print "File couldn't be opened" return times = f['index_map']['time'].value['ctime'][:] print "-------------------------------------" print "start time %s in PT\n" % (eph.unix_to_datetime(times[0])) print "RA range %f : %f" % (np.round(eph.transit_RA(times[0]), 1), np.round(eph.transit_RA(times[-1]),1)) print "-------------------------------------" srcz = [] for src in src_dict: if eph.transit_times(src_dict[src], times[0])[0] < times[-1]: print "%s is in this file" % src srcz.append(src) return srcz
def search_extended_timing_solutions(timing_files, timestamp): # Load the timing correction nfiles = len(timing_files) tstart = np.zeros(nfiles, dtype=np.float32) tstop = np.zeros(nfiles, dtype=np.float32) all_tcorr = [] for ff, filename in enumerate(timing_files): kwargs = {} with h5py.File(filename, 'r') as handler: for key in ['tau', 'avg_phase', 'noise_source', 'time']: kwargs[key] = handler[key][:] tcorr = timing.TimingCorrection(**kwargs) all_tcorr.append(tcorr) tstart[ff] = tcorr.time[0] tstop[ff] = tcorr.time[-1] # Map timestamp to a timing correction object imatch = np.flatnonzero((timestamp >= tstart) & (timestamp <= tstop)) if imatch.size > 1: ValueError("Timing corrections overlap!") elif imatch.size < 1: ValueError("No timing correction for transit on %s (CSD %d)" % (ephemeris.unix_to_datetime(timestamp).strftime("%Y-%m-%d"), ephemeris.unix_to_csd(timestamp))) return all_tcorr[imatch[0]]
def _transit_bounds(self): """Find the start and end times of this transit. Compares the desired HA span to the start and end times of the observation recorded in the database. Also gets the observation ID.""" # subtract half a day from start time to ensure we don't get following day self.start_t = self.cur_transit - self.ha_span / 360.0 / 2.0 * SIDEREAL_DAY_SEC self.end_t = self.cur_transit + self.ha_span / 360.0 / 2.0 * SIDEREAL_DAY_SEC # get bounds of observation from database this_run = [ r for r in self.db_runs if r[1][0] < self.cur_transit and r[1][1] > self.cur_transit ] if len(this_run) == 0: self.log.warning( "Could not find source transit in holography database for {}.".format( ephem.unix_to_datetime(self.cur_transit) ) ) # skip this file self.cur_transit = None else: self.start_t = max(self.start_t, this_run[0][1][0]) self.end_t = min(self.end_t, this_run[0][1][1]) self.obs_id = this_run[0][0]
def print_timestamp(fn): src_dict = { 'CasA': eph.CasA, 'TauA': eph.TauA, 'CygA': eph.CygA, 'VirA': eph.VirA, '1929': 292.0, '0329': 54.0 } try: f = h5py.File(fn, 'r') except IOError: print "File couldn't be opened" return times = f['index_map']['time'].value['ctime'][:] print "-------------------------------------" print "start time %s in PT\n" % (eph.unix_to_datetime(times[0])) print "RA range %f : %f" % (np.round(eph.transit_RA( times[0]), 1), np.round(eph.transit_RA(times[-1]), 1)) print "-------------------------------------" srcz = [] for src in src_dict: if eph.transit_times(src_dict[src], times[0])[0] < times[-1]: print "%s is in this file" % src srcz.append(src) return srcz
def next(self, intervals): """Filter input files to exclude those already processed. Parameters ---------- intervals: list of chimedb.data_index.DataInterval List intervals to filter. Returns ------- files: list of str List of files to be processed. """ self.log.info("Starting next for task %s" % self.__class__.__name__) self.comm.Barrier() files = [] for fi in intervals: start, end = fi[1] # find holography observation that overlaps this set this_obs = [ o for o in self.hol_obs if (o.start_time >= start and o.start_time <= end) or (o.finish_time >= start and o.finish_time <= end) or (o.start_time <= start and o.finish_time >= end) ] if len(this_obs) == 0: self.log.warning( "Could not find source transit in holography database for {}.".format( ephem.unix_to_datetime(start) ) ) elif this_obs[0].id in self.proc_transits: self.log.warning( "Already processed transit for {}. Skipping.".format( ephem.unix_to_datetime(start) ) ) else: files += fi[0] self.log.info("Leaving next for task %s" % self.__class__.__name__) return files
def set_acq_list(self): """This method sets four attributes. The first two attributes are 'night_finder' and 'night_acq_list', which are the finder object and list of acquisitions that contain all night time data between self.t1 and self.t2. The second two attributes are 'finder' and 'acq_list', which are the finder object and list of acquisitions that contain all data beween self.t1 and self.t2 with the sunrise, sun transit, and sunset removed. """ # Create a Finder object and focus on time range f = finder.Finder(node_spoof=_DEFAULT_NODE_SPOOF) f.filter_acqs((data_index.ArchiveInst.name == "pathfinder")) f.only_corr() f.set_time_range(self.t1, self.t2) # Create a list of acquisitions that only contain data collected at night f_night = copy.deepcopy(f) f_night.exclude_daytime() self.night_finder = f_night self.night_acq_list = f_night.get_results() # Create a list of acquisitions that flag out sunrise, sun transit, and sunset mm = ephemeris.unix_to_datetime(self.t1).month dd = ephemeris.unix_to_datetime(self.t1).day mm = mm + float(dd) / 30.0 fct = 3.0 tol1 = (np.arctan( (mm - 3.0) * fct) + np.pi / 2.0) * 10500.0 / np.pi + 1500.0 tol2 = (np.pi / 2.0 - np.arctan( (mm - 11.0) * fct)) * 10500.0 / np.pi + 1500.0 ttol = np.minimum(tol1, tol2) fct = 5.0 tol1 = (np.arctan( (mm - 4.0) * fct) + np.pi / 2.0) * 2100.0 / np.pi + 6000.0 tol2 = (np.pi / 2.0 - np.arctan( (mm - 10.0) * fct)) * 2100.0 / np.pi + 6000.0 rstol = np.minimum(tol1, tol2) f.exclude_sun(time_delta=ttol, time_delta_rise_set=rstol) self.finder = f self.acq_list = f.get_results()
def main(): assert len(sys.argv) == 2, "src" basedir = '/mnt/gong/archive/' if run_ph_solver is True: while True: dir_recent = reorder_dir(basedir)[0] src_search = sys.argv[1] print dir_recent # Search the most recent directory for the source "src_search" trans_file = find_trans(basedir + dir_recent, src_search) if trans_file is None: print "Trans file doesn't exist in this directory, sleeping for 1hr" time.sleep(3600) continue tstring = make_outfile_name(trans_file) outfile = './solutions/' + tstring + src_search + '2.hdf5' # might be useful to just make sure gains are swapped # os.system('python solver_script.py %s %s -solve_gains 0 -gen_pkls 1' \ # % (trans_file, src_search)) if os.path.exists(outfile): print "%s already exists, taking a nap. Will check again in 1 hour." % outfile time.sleep(3600) continue print "feed this", trans_file snm = time.time() os.system('python solver_script.py %s %s -solve_gains 1 -gen_pkls 1' \ % (trans_file, src_search)) print "Going to sleep for a whole damn day" print eph.unix_to_datetime(time.time() - 8 * 3600) time.sleep(12 * 3600)
def main(): assert len(sys.argv) == 2, "src" basedir = '/mnt/gong/archive/' if run_ph_solver is True: while True: dir_recent = reorder_dir(basedir)[0] src_search = sys.argv[1] print dir_recent # Search the most recent directory for the source "src_search" trans_file = find_trans(basedir + dir_recent, src_search) if trans_file is None: print "Trans file doesn't exist in this directory, sleeping for 1hr" time.sleep(3600) continue tstring = make_outfile_name(trans_file) outfile = './solutions/' + tstring + src_search + '2.hdf5' # might be useful to just make sure gains are swapped # os.system('python solver_script.py %s %s -solve_gains 0 -gen_pkls 1' \ # % (trans_file, src_search)) if os.path.exists(outfile): print "%s already exists, taking a nap. Will check again in 1 hour." % outfile time.sleep(3600) continue print "feed this", trans_file snm = time.time() os.system('python solver_script.py %s %s -solve_gains 1 -gen_pkls 1' \ % (trans_file, src_search)) print "Going to sleep for a whole damn day" print eph.unix_to_datetime(time.time() - 8 * 3600) time.sleep(12 * 3600)
def process_finish(self): """Normalise the stack and return the result. Includes the sample variance over transits within the stack. Returns ------- stack: draco.core.containers.TrackBeam Stacked transits. """ # Divide by norm to get average transit inv_norm = invert_no_zero(self.norm) self.stack.beam[:] *= inv_norm self.stack.weight[:] = invert_no_zero(self.stack.weight[:]) * self.norm**2 self.variance = self.variance * inv_norm - np.abs(self.stack.beam[:]) ** 2 self.pseudo_variance = self.pseudo_variance * inv_norm - self.stack.beam[:] ** 2 # Calculate the covariance between the real and imaginary component # from the accumulated variance and psuedo-variance self.stack.sample_variance[0] = 0.5 * ( self.variance + self.pseudo_variance.real ) self.stack.sample_variance[1] = 0.5 * self.pseudo_variance.imag self.stack.sample_variance[2] = 0.5 * ( self.variance - self.pseudo_variance.real ) # Create tag time_range = np.percentile(self.stack.attrs["transit_time"], [0, 100]) self.stack.attrs["tag"] = "{}_{}_to_{}".format( self.stack.attrs["source_name"], ephem.unix_to_datetime(time_range[0]).strftime("%Y%m%dT%H%M%S"), ephem.unix_to_datetime(time_range[1]).strftime("%Y%m%dT%H%M%S"), ) return self.stack
def set_metadata(self, tms, input_map): """Sets self.corr_inputs, self.pwds, self.pstns, self.p1_idx, self.p2_idx""" from ch_util import tools # Get CHIME ON channels: half_time = ephemeris.unix_to_datetime(tms[int(len(tms) // 2)]) corr_inputs = tools.get_correlator_inputs(half_time) self.corr_inputs = tools.reorder_correlator_inputs( input_map, corr_inputs) pwds = tools.is_chime_on( self.corr_inputs) # Which inputs are CHIME ON antennas self.pwds = np.array(pwds, dtype=bool) # Get cylinders and polarizations self.pstns, self.p1_idx, self.p2_idx = self.get_pos_pol( self.corr_inputs, self.pwds)
def get_prod_sel(self, data): """ """ from ch_util import tools input_map = data.input tms = data.time half_time = ephemeris.unix_to_datetime(tms[int(len(tms) // 2)]) corr_inputs = tools.get_correlator_inputs(half_time) corr_inputs = tools.reorder_correlator_inputs(input_map, corr_inputs) pwds = tools.is_chime_on( corr_inputs) # Which inputs are CHIME ON antennas wchp1, wchp2, echp1, echp2 = self.get_cyl_pol(corr_inputs, pwds) # Ensure base channels are CHIME and ON while not pwds[np.where(input_map["chan_id"] == self.bswp1)[0][0]]: self.bswp1 += 1 while not pwds[np.where(input_map["chan_id"] == self.bswp2)[0][0]]: self.bswp2 += 1 while not pwds[np.where(input_map["chan_id"] == self.bsep1)[0][0]]: self.bsep1 += 1 while not pwds[np.where(input_map["chan_id"] == self.bsep2)[0][0]]: self.bsep2 += 1 prod_sel = [] for (ii, prod) in enumerate(data.prod): add_prod = False add_prod = add_prod or ( (prod[0] == self.bswp1 and prod[1] in echp1) or (prod[1] == self.bswp1 and prod[0] in echp1)) add_prod = add_prod or ( (prod[0] == self.bswp2 and prod[1] in echp2) or (prod[1] == self.bswp2 and prod[0] in echp2)) add_prod = add_prod or ( (prod[0] == self.bsep1 and prod[1] in wchp1) or (prod[1] == self.bsep1 and prod[0] in wchp1)) add_prod = add_prod or ( (prod[0] == self.bsep2 and prod[1] in wchp2) or (prod[1] == self.bsep2 and prod[0] in wchp2)) if add_prod: prod_sel.append(ii) prod_sel.sort() return prod_sel, pwds
def next(self, ts): """Generate an input description from the timestream passed in. Parameters ---------- ts : andata.CorrData Timestream container. Returns ------- inputs : list of :class:`CorrInput` A list of describing the inputs as they are in the file. """ # Fetch from the cache if we can if self.cache and self._cached_inputs: self.log.debug("Using cached inputs.") return self._cached_inputs inputs = None if mpiutil.rank0: # Get the datetime of the middle of the file time = ephemeris.unix_to_datetime(0.5 * (ts.time[0] + ts.time[-1])) inputs = tools.get_correlator_inputs(time) inputs = tools.reorder_correlator_inputs(ts.index_map["input"], inputs) # Broadcast input description to all ranks inputs = mpiutil.world.bcast(inputs, root=0) # Save into the cache for the next iteration if self.cache: self._cached_inputs = inputs # Make sure all nodes have container before return mpiutil.world.Barrier() return inputs
def find_transit(time0='Now', src=eph.CasA): import time if time0 == 'Now': time0 = time.time() # Get reference datetime from unixtime dt_now = eph.unix_to_datetime(time0) dt_now = dt_now.isoformat() # Only use relevant characters in datetime string dt_str = dt_now.replace("-", "")[:7] dirnm = '/mnt/gong/archive/' + dt_str filelist = glob.glob(dirnm + '*/*h5') # Step through each file and find that day's transit for ff in filelist: try: andataReader = andata.Reader(ff) acqtimes = andataReader.time trans_time = eph.transit_times(src, time0) #print ff # print eph.transit_RA(trans_time), eph.transit_RA(acqtimes[0]) del andataReader if np.abs(acqtimes - trans_time[0]).min() < 1000.0: # print "On ", eph.unix_to_datetime(trans_time[0]) # print "foundit in %s \n" % ff return ff break except (KeyError, ValueError, IOError): pass return None
def find_transit(time0='Now', src=eph.CasA): import time if time0 == 'Now': time0 = time.time() # Get reference datetime from unixtime dt_now = eph.unix_to_datetime(time0) dt_now = dt_now.isoformat() # Only use relevant characters in datetime string dt_str = dt_now.replace("-", "")[:7] dirnm = '/mnt/gong/archive/' + dt_str filelist = glob.glob(dirnm + '*/*h5') # Step through each file and find that day's transit for ff in filelist: try: andataReader = andata.Reader(ff) acqtimes = andataReader.time trans_time = eph.transit_times(src, time0) #print ff # print eph.transit_RA(trans_time), eph.transit_RA(acqtimes[0]) del andataReader if np.abs(acqtimes - trans_time[0]).min() < 1000.0: # print "On ", eph.unix_to_datetime(trans_time[0]) # print "foundit in %s \n" % ff return ff break except (KeyError, ValueError, IOError): pass return None
def process(self, filelist): """Generate timing correction from an input list of files. Parameters ---------- filelist : list of files Returns ------- tcorr : ch_util.timing.TimingCorrection Timing correction derived from noise source data. """ # Determine the acquisition new_acq = np.unique( [os.path.basename(os.path.dirname(ff)) for ff in filelist]) if new_acq.size > 1: raise RuntimeError( "Cannot process multiple acquisitions. Received %d." % new_acq.size) else: new_acq = new_acq[0] # If this is a new acquisition, then ensure the # static phase and amplitude are recalculated if new_acq != self.current_acq: self.current_acq = new_acq for key in self._datasets_fixed_for_acq: self.kwargs[key] = None # Process the chimetiming data self.log.info("Processing %d files from %s." % (len(filelist), self.current_acq)) tcorr = timing.TimingData.from_acq_h5( filelist, only_correction=True, distributed=self.comm.size > 1, comm=self.comm, **self.kwargs, ) self.log.info("Finished processing %d files from %s." % (len(filelist), self.current_acq)) # Save the static phase and amplitude to be used on subsequent iterations # within this acquisition for key in self._datasets_fixed_for_acq: if key in tcorr.datasets: self.kwargs[key] = tcorr.datasets[key][:] elif key in tcorr.flags: self.kwargs[key] = tcorr.flags[key][:] else: msg = "Dataset %s could not be found in timing correction object." % key raise RuntimeError(msg) # Save the names of the files used to construct the correction tcorr.attrs["archive_files"] = np.array(filelist) # Create a tag indicating the range of time processed tfmt = "%Y%m%dT%H%M%SZ" start_time = ephemeris.unix_to_datetime(tcorr.time[0]).strftime(tfmt) end_time = ephemeris.unix_to_datetime(tcorr.time[-1]).strftime(tfmt) tag = [start_time, "to", end_time] if self.output_suffix: tag.append(self.output_suffix) tcorr.attrs["tag"] = "_".join(tag) # Return timing correction return tcorr
def from_lsd(cls, lsd: int): unix = csd_to_unix(lsd) lsd = int(lsd) date = unix_to_datetime(unix).date() day = cls(lsd, date) return day
def process(self, tstream): """Apply the timing correction to the input timestream. Parameters ---------- tstream : andata.CorrData, containers.TimeStream, or containers.SiderealStream Apply the timing correction to the visibilities stored in this container. Returns ------- tstream_corr : same as `tstream` Timestream with corrected visibilities. """ # Determine times if "time" in tstream.index_map: timestamp = tstream.time else: csd = (tstream.attrs["lsd"] if "lsd" in tstream.attrs else tstream.attrs["csd"]) timestamp = ephemeris.csd_to_unix(csd + tstream.ra / 360.0) # Extract local frequencies tstream.redistribute("freq") nfreq = tstream.vis.local_shape[0] sfreq = tstream.vis.local_offset[0] efreq = sfreq + nfreq freq = tstream.freq[sfreq:efreq] # If requested, extract the input flags input_flags = tstream.input_flags[:] if self.use_input_flags else None # Check needs_timing_correction flags time ranges to see if input timestream # falls within the interval needs_timing_correction = False for flag in self.flags: if (timestamp[0] >= flag["start_time"] and timestamp[0] <= flag["finish_time"]): if timestamp[-1] >= flag["finish_time"]: raise PipelineRuntimeError( f"Data covering {timestamp[0]} to {timestamp[-1]} partially overlaps " f"needs_timing_correction DataFlag covering {ephemeris.unix_to_datetime(flag['start_time']).strftime('%Y%m%dT%H%M%SZ')} " f"to {ephemeris.unix_to_datetime(flag['finish_time']).strftime('%Y%m%dT%H%M%SZ')}." ) else: self.log.info( f"Data covering {timestamp[0]} to {timestamp[-1]} flagged by " f"needs_timing_correction DataFlag covering {ephemeris.unix_to_datetime(flag['start_time']).strftime('%Y%m%dT%H%M%SZ')} " f"to {ephemeris.unix_to_datetime(flag['finish_time']).strftime('%Y%m%dT%H%M%SZ')}. Timing correction will be applied." ) needs_timing_correction = True break if not needs_timing_correction: self.log.info( f"Data in span {ephemeris.unix_to_datetime(timestamp[0]).strftime('%Y%m%dT%H%M%SZ')} to {ephemeris.unix_to_datetime(timestamp[-1]).strftime('%Y%m%dT%H%M%SZ')} does not need timing correction" ) return tstream # Find the right timing correction for tcorr in self.tcorr: if timestamp[0] >= tcorr.time[0] and timestamp[-1] <= tcorr.time[ -1]: break else: msg = ( "Could not find timing correction file covering " "range of timestream data (%s to %s)." % tuple( ephemeris.unix_to_datetime([timestamp[0], timestamp[-1]]))) if self.pass_if_missing: self.log.warning(msg + " Doing nothing.") return tstream raise RuntimeError(msg) self.log.info("Using correction file %s" % tcorr.attrs["tag"]) # If requested, reference the timing correct with respect to source transit time if self.refer_to_transit: # First check for transit_time attribute in file ttrans = tstream.attrs.get("transit_time", None) if ttrans is None: source = tstream.attrs["source_name"] ttrans = ephemeris.transit_times( ephemeris.source_dictionary[source], tstream.time[0], tstream.time[-1], ) if ttrans.size != 1: raise RuntimeError( "Found %d transits of %s in timestream. " "Require single transit." % (ttrans.size, source)) else: ttrans = ttrans[0] self.log.info( "Referencing timing correction to %s (RA=%0.1f deg)." % ( ephemeris.unix_to_datetime(ttrans).strftime( "%Y%m%dT%H%M%SZ"), ephemeris.lsa(ttrans), )) tcorr.set_global_reference_time(ttrans, window=self.transit_window, interpolate=True, interp="linear") # Apply the timing correction tcorr.apply_timing_correction(tstream, time=timestamp, freq=freq, input_flags=input_flags, copy=False) return tstream
def svd_panels(data, results, rank, cmap=COLORMAP, vrng=None, nticks=20, cyl_size=256, input_range=None, input_map=None, corr_order=False, zoom_date=None, timezone='Canada/Pacific', phys_unit=True): inputs = data.index_map['input'][:] ninput = inputs.size timestamp = data.time[:] ntime = timestamp.size good_input = results['good_input'] good_transit = results['good_transit'] if (input_map is not None) and corr_order: cord = np.array([inp.corr_order for inp in input_map]) cord = cord[good_input] isort = np.argsort(cord) cttl = "Correlator Order" else: cord = good_input isort = np.argsort(cord) cttl = "Feed Number" if input_range is None: input_range = [0, ninput] tfmt = "%b-%d" tz = pytz.timezone(timezone) skip = ntime // nticks xtck = np.arange(0, ntime, skip, dtype=np.int) xtcklbl = [ tt.astimezone(tz).strftime(tfmt) for tt in ephemeris.unix_to_datetime(timestamp[::skip]) ] ytck = np.arange(ninput // cyl_size + 1) * cyl_size if zoom_date is None: zoom_date = [] else: zoom_date = [list(ephemeris.datetime_to_unix(zd)) for zd in zoom_date] zoom_date = [[timestamp[0], timestamp[-1]]] + zoom_date nzoom = len(zoom_date) gs = gridspec.GridSpec(2 + nzoom // 2, 2, height_ratios=[3, 2] + [2] * (nzoom // 2), hspace=0.30) this_rank = np.outer(results['S'][rank] * results['U'][:, rank], results['VH'][rank, :]) var = results['S']**2 exp_var = 100.0 * var / np.sum(var) exp_var = exp_var[rank] if vrng is None: vrng = np.nanpercentile(this_rank, [2, 98]) y = np.full((ninput, ntime), np.nan, dtype=np.float32) for ii, gi in enumerate(cord): y[gi, good_transit] = this_rank[:, ii] plt.subplot(gs[0, :]) img = plt.imshow(y, aspect='auto', origin='lower', interpolation='nearest', extent=(0, ntime, 0, ninput), vmin=vrng[0], vmax=vrng[1], cmap=cmap) cbar = plt.colorbar(img) cbar.ax.set_ylabel(r'$\tau$ [picosec]') plt.ylabel(cttl) plt.title("k = %d | Explains %0.1f%% of Variance" % (rank, exp_var)) plt.xticks(xtck, xtcklbl, rotation=70.0) plt.yticks(ytck) plt.ylim(input_range) if phys_unit: scale = 1.48625 * np.median( np.abs(results['S'][rank] * results['U'][:, rank])) lbl = r"v$_{%d}$ x MED$(|\sigma_{%d} u_{%d}|)$ [picosec]" % ( rank, rank, rank) else: scale = 1.0 lbl = r"v$_{%d}$" % rank plt.subplot(gs[1, 0]) plt.plot(cord, scale * results['VH'][rank, :], color='k', marker='.', linestyle='None') plt.grid() plt.xticks(ytck, rotation=70) plt.xlim(input_range) plt.xlabel(cttl) plt.ylabel(lbl) scolors = ['b', 'r', 'forestgreen', 'magenta', 'orange'] list_good = list(results['good_transit']) if phys_unit: scale = 1.48625 * np.median(np.abs(results['VH'][rank, :])) lbl = r"$\sigma_{%d}$ u$_{%d}$ x MED$(|v_{%d}|)$ [picosec]" % ( rank, rank, rank) else: scale = 1.0 lbl = r"$\sigma_{%d}$ u$_{%d}$" % (rank, rank) classification = np.char.add(np.char.add(data['calibrator'][:], '/'), data['source'][:]) usource, ind_source = np.unique(classification, return_inverse=True) nsource = usource.size for zz, (start_time, end_time) in enumerate(zoom_date): row = (zz + 1) // 2 + 1 col = (zz + 1) % 2 plt.subplot(gs[row, col]) this_zoom = np.flatnonzero((data.time >= start_time) & (data.time <= end_time)) nsamples = this_zoom.size skip = nsamples // (nticks // 2) xtck = np.arange(0, nsamples, skip, dtype=np.int) ztfmt = "%b-%d %H:%M" if zz > 0 else tfmt xtcklbl = np.array([ tt.astimezone(tz).strftime(ztfmt) for tt in ephemeris.unix_to_datetime(data.time[this_zoom[xtck]]) ]) for ss, src in enumerate(usource): this_source = np.flatnonzero(ind_source[this_zoom] == ss) if this_source.size == 0: continue this_source = np.array( [ts for ts in this_source if this_zoom[ts] in list_good]) this_source_good = np.array( [list_good.index(ind) for ind in this_zoom[this_source]]) plt.plot(this_source, scale * results['S'][rank] * results['U'][this_source_good, rank], color=scolors[ss], marker='.', markersize=4, linestyle='None', label=src) plt.xticks(xtck, xtcklbl, rotation=70.0) plt.xlim(0, nsamples) plt.grid() plt.ylabel(lbl) plt.legend(prop={'size': 10})
def main(config_file=None, logging_params=DEFAULT_LOGGING): # Load config config = DEFAULTS.deepcopy() if config_file is not None: print(config_file) config.merge(NameSpace(load_yaml_config(config_file))) # Setup logging log.setup_logging(logging_params) logger = log.get_logger(__name__) timer = Timer(logger) # Load data sfile = config.data.filename if os.path.isabs( config.data.filename) else os.path.join(config.directory, config.data.filename) sdata = StabilityData.from_file(sfile) ninput, ntime = sdata['tau'].shape # Load temperature data tfile = (config.temperature.filename if os.path.isabs(config.temperature.filename) else os.path.join( config.directory, config.temperature.filename)) tkeys = ['flag', 'data_flag', 'outlier'] if config.temperature.load: tkeys += config.temperature.load tdata = TempData.from_acq_h5(tfile, datasets=tkeys) # Query layout database inputmap = tools.get_correlator_inputs(ephemeris.unix_to_datetime( np.median(sdata.time[:])), correlator='chime') good_input = np.flatnonzero(np.any(sdata['flags']['tau'][:], axis=-1)) pol = sutil.get_pol(sdata, inputmap) npol = len(pol) mezz_index, crate_index = sutil.get_mezz_and_crate(sdata, inputmap) if config.mezz_ref.enable: phase_ref = [ ipol[mezz_index[ipol] == iref] for ipol, iref in zip(pol, config.mezz_ref.mezz) ] else: phase_ref = config.data.phase_ref # Load timing if config.timing.enable: # Extract filenames from config timing_files = [ tf if os.path.isabs(tf) else os.path.join(config.directory, tf) for tf in config.timing.files ] timing_files_hpf = [ os.path.join(os.path.dirname(tf), 'hpf', os.path.basename(tf)) for tf in timing_files ] timing_files_lpf = [ os.path.join(os.path.dirname(tf), 'lpf', os.path.basename(tf)) for tf in timing_files ] # If requested, add the timing data back into the delay data if config.timing.add.enable: timer.start("Adding timing data to delay measurements.") ns_tau, _, ns_flag, ns_inputs = sutil.get_timing_correction( sdata, timing_files, **config.timing.add.kwargs) index = timing.map_input_to_noise_source(sdata.index_map['input'], ns_inputs) timing_tau = ns_tau[index, :] timing_flag = ns_flag[index, :] for ipol, iref in zip(pol, config.data.phase_ref): timing_tau[ipol, :] = timing_tau[ipol, :] - timing_tau[ iref, np.newaxis, :] timing_flag[ipol, :] = timing_flag[ipol, :] & timing_flag[ iref, np.newaxis, :] sdata['tau'][:] = sdata['tau'][:] + timing_tau sdata['flags']['tau'][:] = sdata['flags']['tau'][:] & timing_flag timer.stop() # Extract the dependent variables from the timing dataset timer.start("Calculating timing dependence.") if config.timing.sep_delay: logger.info("Fitting HPF and LPF timing correction separately.") files = timing_files_hpf files2 = timing_files_lpf else: files2 = None if config.timing.hpf_delay: logger.info("Using HPF timing correction for delay.") files = timing_files_hpf elif config.timing.lpf_delay: logger.info("Using LPF timing correction for delay.") files = timing_files_lpf else: logger.info("Using full timing correction for delay.") files = timing_files kwargs = {} if config.timing.lpf_amp: logger.info("Using LPF timing correction for amplitude.") kwargs['afiles'] = timing_files_lpf elif config.timing.hpf_amp: logger.info("Using HPF timing correction for amplitude.") kwargs['afiles'] = timing_files_hpf else: logger.info("Using full timing correction for amplitude.") kwargs['afiles'] = timing_files for key in ['ns_ref', 'inter_cmn', 'fit_amp', 'ref_amp', 'cmn_amp']: if key in config.timing: kwargs[key] = config.timing[key] xtiming, xtiming_flag, xtiming_group = sutil.timing_dependence( sdata, files, inputmap, **kwargs) if files2 is not None: logger.info("Calculating second timing dependence.") kwargs['fit_amp'] = False xtiming2, xtiming2_flag, xtiming2_group = sutil.timing_dependence( sdata, files2, inputmap, **kwargs) xtiming = np.concatenate((xtiming, xtiming2), axis=-1) xtiming_flag = np.concatenate((xtiming_flag, xtiming2_flag), axis=-1) xtiming_group = np.concatenate((xtiming_group, xtiming2_group), axis=-1) timer.stop() else: xtiming = None xtiming_flag = None xtiming_group = None # Reference delay data to mezzanine if config.mezz_ref.enable: timer.start("Referencing delay measurements to mezzanine.") for ipol, iref in zip(pol, config.mezz_ref.mezz): this_mezz = ipol[mezz_index[ipol] == iref] wmezz = sdata['flags']['tau'][this_mezz, :].astype(np.float32) norm = np.sum(wmezz, axis=0) taut_mezz = np.sum(wmezz * sdata['tau'][this_mezz, :], axis=0) * tools.invert_no_zero(norm) flagt_mezz = norm > 0.0 sdata['tau'][ ipol, :] = sdata['tau'][ipol, :] - taut_mezz[np.newaxis, :] sdata['flags']['tau'][ipol, :] = sdata['flags']['tau'][ ipol, :] & flagt_mezz[np.newaxis, :] timer.stop() # Load cable monitor if config.cable_monitor.enable: timer.start("Calculating cable monitor dependence.") cbl = timing.TimingCorrection.from_acq_h5( config.cable_monitor.filename) kwargs = {'include_diff': config.cable_monitor.include_diff} xcable, xcable_flag, xcable_group = sutil.cable_monitor_dependence( sdata, cbl, **kwargs) timer.stop() else: xcable = None xcable_flag = None xcable_group = None # Load NS distance if config.ns_distance.enable: timer.start("Calculating NS distance dependence.") kwargs = {} kwargs['phase_ref'] = phase_ref for key in [ 'sensor', 'temp_field', 'sep_cyl', 'sep_feed', 'include_offset', 'include_ha' ]: if key in config.ns_distance: kwargs[key] = config.ns_distance[key] if config.ns_distance.use_cable_monitor: kwargs['is_cable_monitor'] = True kwargs['use_alpha'] = config.ns_distance.use_alpha nsx = timing.TimingCorrection.from_acq_h5( config.cable_monitor.filename) else: kwargs['is_cable_monitor'] = False nsx = tdata xdist, xdist_flag, xdist_group = sutil.ns_distance_dependence( sdata, nsx, inputmap, **kwargs) if (config.ns_distance.deriv is not None) and (config.ns_distance.deriv > 0): for dd in range(1, config.ns_distance.deriv + 1): d_xdist, d_xdist_flag, d_xdist_group = sutil.ns_distance_dependence( sdata, tdata, inputmap, deriv=dd, **kwargs) tind = np.atleast_1d(1) xdist = np.concatenate((xdist, d_xdist[:, :, tind]), axis=-1) xdist_flag = xnp.concatenate( (xdist_flag, d_xdist_flag[:, :, tind]), axis=-1) xdist_group = np.concatenate( (xdist_group, d_xdist_group[:, tind]), axis=-1) timer.stop() else: xdist = None xdist_flag = None xdist_group = None # Load temperatures if config.temperature.enable: timer.start("Calculating temperature dependence.") xtemp, xtemp_flag, xtemp_group, xtemp_name = sutil.temperature_dependence( sdata, tdata, config.temperature.sensor, field=config.temperature.temp_field, inputmap=inputmap, phase_ref=phase_ref, check_hut=config.temperature.check_hut) if (config.temperature.deriv is not None) and (config.temperature.deriv > 0): for dd in range(1, config.temperature.deriv + 1): d_xtemp, d_xtemp_flag, d_xtemp_group, d_xtemp_name = sutil.temperature_dependence( sdata, tdata, config.temperature.sensor, field=config.temperature.temp_field, deriv=dd, inputmap=inputmap, phase_ref=phase_ref, check_hut=config.temperature.check_hut) xtemp = np.concatenate((xtemp, d_xtemp), axis=-1) xtemp_flag = xnp.concatenate((xtemp_flag, d_xtemp_flag), axis=-1) xtemp_group = np.concatenate((xtemp_group, d_xtemp_group), axis=-1) xtemp_name += d_xtemp_name timer.stop() else: xtemp = None xtemp_flag = None xtemp_group = None xtemp_name = None # Combine into single feature matrix x, coeff_name = _concatenate(xdist, xtemp, xcable, xtiming, name_xtemp=xtemp_name) x_group, _ = _concatenate(xdist_group, xtemp_group, xcable_group, xtiming_group) x_flag, _ = _concatenate(xdist_flag, xtemp_flag, xcable_flag, xtiming_flag) x_flag = np.all(x_flag, axis=-1) & sdata.flags['tau'][:] nfeature = x.shape[-1] logger.info("Fitting %d features." % nfeature) # Save data if config.preliminary_save.enable: if config.preliminary_save.filename is not None: ofile = (config.preliminary_save.filename if os.path.isabs( config.preliminary_save.filename) else os.path.join( config.directory, config.preliminary_save.filename)) else: ofile = os.path.splitext( sfile)[0] + '_%s.h5' % config.preliminary_save.suffix sdata.save(ofile, mode='w') # Subtract mean if config.mean_subtract: timer.start("Subtracting mean value.") tau, mu_tau, mu_tau_flag = sutil.mean_subtract(sdata, sdata['tau'][:], x_flag, use_calibrator=True) mu_x = np.zeros(mu_tau.shape + (nfeature, ), dtype=x.dtype) mu_x_flag = np.zeros(mu_tau.shape + (nfeature, ), dtype=np.bool) x_no_mu = x.copy() for ff in range(nfeature): x_no_mu[..., ff], mu_x[..., ff], mu_x_flag[..., ff] = sutil.mean_subtract( sdata, x[:, :, ff], x_flag, use_calibrator=True) timer.stop() else: x_no_mu = x.copy() tau = sdata['tau'][:].copy() # Calculate unique days csd_uniq, bmap = np.unique(sdata['csd'][:], return_inverse=True) ncsd = csd_uniq.size # Prepare unique sources classification = np.char.add(np.char.add(sdata['calibrator'][:], '/'), sdata['source'][:]) # If requested, load existing coefficients if config.coeff is not None: coeff = andata.BaseData.from_acq_h5(config.coeff) evaluate_only = True else: evaluate_only = False # If requested, set up boot strapping if config.bootstrap.enable: nboot = config.bootstrap.number nchoices = ncsd if config.bootstrap.by_transit else ntime nsample = int(config.bootstrap.fraction * nchoices) bindex = np.zeros((nboot, nsample), dtype=np.int) for roll in range(nboot): bindex[roll, :] = np.sort( np.random.choice(nchoices, size=nsample, replace=config.bootstrap.replace)) else: nboot = 1 bindex = np.arange(ntime, dtype=np.int)[np.newaxis, :] # Prepare output if config.output.directory is not None: output_dir = config.output.directory else: output_dir = config.data.directory if config.output.suffix is not None: output_suffix = config.output.suffix else: output_suffix = os.path.splitext(os.path.basename( config.data.filename))[0] # Perform joint fit for bb, bind in enumerate(bindex): if config.bootstrap.enable and config.bootstrap.by_transit: tind = np.concatenate( tuple([np.flatnonzero(bmap == ii) for ii in bind])) else: tind = bind ntime = tind.size if config.jackknife.enable: start = int( config.jackknife.start * ncsd ) if config.jackknife.start <= 1.0 else config.jackknife.start end = int( config.jackknife.end * ncsd) if config.jackknife.end <= 1.0 else config.jackknife.end time_flag_fit = (bmap >= start) & (bmap < end) if config.jackknife.restrict_stat: time_flag_stat = np.logical_not(time_flag_fit) else: time_flag_stat = np.ones(ntime, dtype=np.bool) else: time_flag_fit = np.ones(ntime, dtype=np.bool) time_flag_stat = np.ones(ntime, dtype=np.bool) logger.info( "Fitting data between %s (CSD %d) and %s (CSD %d)" % (ephemeris.unix_to_datetime(np.min( sdata.time[tind[time_flag_fit]])).strftime("%Y-%m-%d"), np.min(sdata['csd'][:][tind[time_flag_fit]]), ephemeris.unix_to_datetime(np.max( sdata.time[tind[time_flag_fit]])).strftime("%Y-%m-%d"), np.max(sdata['csd'][:][tind[time_flag_fit]]))) logger.info( "Calculating statistics from data between %s (CSD %d) and %s (CSD %d)" % (ephemeris.unix_to_datetime( np.min(sdata.time[tind[time_flag_stat]])).strftime("%Y-%m-%d"), np.min(sdata['csd'][:][tind[time_flag_stat]]), ephemeris.unix_to_datetime( np.max( sdata.time[tind[time_flag_stat]])).strftime("%Y-%m-%d"), np.max(sdata['csd'][:][tind[time_flag_stat]]))) if evaluate_only: timer.start("Evaluating coefficients provided.") fitter = sutil.JointTempEvaluation( x_no_mu[:, tind, :], tau[:, tind], coeff['coeff'][:], flag=x_flag[:, tind], coeff_name=coeff.index_map['feature'][:], feature_name=coeff_name, intercept=coeff['intercept'][:], intercept_name=coeff.index_map['classification'][:], classification=classification[tind]) timer.stop() else: timer.start("Setting up fit. Bootstrap %d of %d." % (bb + 1, nboot)) fitter = sutil.JointTempRegression( x_no_mu[:, tind, :], tau[:, tind], x_group, flag=x_flag[:, tind], classification=classification[tind], coeff_name=coeff_name) timer.stop() timer.start("Performing fit. Bootstrap %d of %d." % (bb + 1, nboot)) fitter.fit_temp(time_flag=time_flag_fit, **config.fit_options) timer.stop() # If bootstrapping, append counter to filename if config.bootstrap.enable: output_suffix_bb = output_suffix + "_bootstrap_%04d" % ( config.bootstrap.index_start + bb, ) with open( os.path.join(output_dir, "bootstrap_index_%s.json" % output_suffix_bb), 'w') as jhandler: json.dump({ "bind": bind.tolist(), "tind": tind.tolist() }, jhandler) else: output_suffix_bb = output_suffix # Save statistics to file if config.output.stat: # If requested, break the model up into its various components for calculating statistics stat_key = ['data', 'model', 'resid'] if config.refine_model.enable: stat_add = fitter.refine_model(config.refine_model.include) stat_key += stat_add # Redefine axes bdata = StabilityData() for dset in ["source", "csd", "calibrator", "calibrator_time"]: bdata.create_dataset(dset, data=sdata[dset][tind]) bdata.create_index_map("time", sdata.index_map["time"][tind]) bdata.create_index_map("input", sdata.index_map["input"][:]) bdata.attrs["calibrator"] = sdata.attrs.get("calibrator", "CYG_A") # Calculate statistics stat = {} for statistic in ['std', 'mad']: for attr in stat_key: for ref, ref_common in zip(['mezz', 'cmn'], [False, True]): stat[(statistic, attr, ref)] = sutil.short_long_stat( bdata, getattr(fitter, attr), fitter._flag & time_flag_stat[np.newaxis, :], stat=statistic, ref_common=ref_common, pol=pol) output_filename = os.path.join(output_dir, "stat_%s.h5" % output_suffix_bb) write_stat(bdata, stat, fitter, output_filename) # Save coefficients to file if config.output.coeff: output_filename = os.path.join(output_dir, "coeff_%s.h5" % output_suffix_bb) write_coeff(sdata, fitter, output_filename) # Save residuals to file if config.output.resid: output_filename = os.path.join(output_dir, "resid_%s.h5" % output_suffix_bb) write_resid(sdata, fitter, output_filename) del fitter gc.collect()
def check_for_duplicates(t, src, start_tol, ignore_src_mismatch=False): """ Check for duplicate holography observations, comparing the given observation to the existing database Inputs ------ t: Skyfield Time object beginning time of observation src: HolographySource target source start_tol: float Tolerance in seconds within which to search for duplicates ignore_src_mismatch: bool (default: False) If True, consider observations a match if the time matches but the source does not Outputs ------- If a duplicate is found: :py:class:`HolographyObservation` object for the existing entry in the database If no duplicate is found: None """ ts = ephemeris.skyfield_wrapper.timescale unixt = ephemeris.ensure_unix(t) dup_found = False existing_db_entry = cls.select().where( cls.start_time.between(unixt - start_tol, unixt + start_tol)) if len(existing_db_entry) > 0: if len(existing_db_entry) > 1: print("Multiple entries found.") for entry in existing_db_entry: tt = ts.utc(ephemeris.unix_to_datetime(entry.start_time)) # LST = GST + east longitude ttlst = np.mod(tt.gmst + DRAO_lon, 24.0) # Check if source name matches. If not, print a warning # but proceed anyway. if src.name.upper() == entry.source.name.upper(): dup_found = True if verbose: print("Observation is already in database.") else: if ignore_src_mismatch: dup_found = True print( "** Observation at same time but with different " + "sources in database: ", src.name, entry.source.name, tt.utc_datetime().isoformat(), ) # if the observations match in start time and source, # call them the same observation. Not the most strict # check possible. if dup_found: tf = ts.utc( ephemeris.unix_to_datetime(entry.finish_time)) print("Tried to add : {} {}; LST={:.3f}".format( src.name, t.utc_datetime().strftime(DATE_FMT_STR), ttlst)) print("Existing entry: {} {}; LST={:.3f}".format( entry.source.name, tt.utc_datetime().strftime(DATE_FMT_STR), ttlst, )) if dup_found: return existing_db_entry else: return None
def main(config_file=None, logging_params=DEFAULT_LOGGING): # Setup logging log.setup_logging(logging_params) mlog = log.get_logger(__name__) # Set config config = DEFAULTS.deepcopy() if config_file is not None: config.merge(NameSpace(load_yaml_config(config_file))) # Set niceness current_niceness = os.nice(0) os.nice(config.niceness - current_niceness) mlog.info('Changing process niceness from %d to %d. Confirm: %d' % (current_niceness, config.niceness, os.nice(0))) # Find acquisition files acq_files = sorted(glob(os.path.join(config.data_dir, config.acq, "*.h5"))) nfiles = len(acq_files) # Determine time range of each file findex = [] tindex = [] for ii, filename in enumerate(acq_files): subdata = andata.CorrData.from_acq_h5(filename, datasets=()) findex += [ii] * subdata.ntime tindex += range(subdata.ntime) findex = np.array(findex) tindex = np.array(tindex) # Determine transits within these files transits = [] data = andata.CorrData.from_acq_h5(acq_files, datasets=()) solar_rise = ephemeris.solar_rising(data.time[0] - 24.0 * 3600.0, end_time=data.time[-1]) for rr in solar_rise: ss = ephemeris.solar_setting(rr)[0] solar_flag = np.flatnonzero((data.time >= rr) & (data.time <= ss)) if solar_flag.size > 0: solar_flag = solar_flag[::config.downsample] tval = data.time[solar_flag] this_findex = findex[solar_flag] this_tindex = tindex[solar_flag] file_list, tindices = [], [] for ii in range(nfiles): this_file = np.flatnonzero(this_findex == ii) if this_file.size > 0: file_list.append(acq_files[ii]) tindices.append(this_tindex[this_file]) date = ephemeris.unix_to_datetime(rr).strftime('%Y%m%dT%H%M%SZ') transits.append((date, tval, file_list, tindices)) # Create file prefix and suffix prefix = [] prefix.append("redundant_calibration") if config.output_prefix is not None: prefix.append(config.output_prefix) prefix = '_'.join(prefix) suffix = [] if config.include_auto: suffix.append("wauto") else: suffix.append("noauto") if config.include_intracyl: suffix.append("wintra") else: suffix.append("nointra") if config.fix_degen: suffix.append("fixed_degen") else: suffix.append("degen") suffix = '_'.join(suffix) # Loop over solar transits for date, timestamps, files, time_indices in transits: nfiles = len(files) mlog.info("%s (%d files) " % (date, nfiles)) output_file = os.path.join(config.output_dir, "%s_SUN_%s_%s.h5" % (prefix, date, suffix)) mlog.info("Saving to: %s" % output_file) # Get info about this set of files data = andata.CorrData.from_acq_h5(files, datasets=['flags/inputs'], apply_gain=False, renormalize=False) coord = sun_coord(timestamps, deg=True) fstart = config.freq_start if config.freq_start is not None else 0 fstop = config.freq_stop if config.freq_stop is not None else data.freq.size freq_index = range(fstart, fstop) freq = data.freq[freq_index] ntime = timestamps.size nfreq = freq.size # Determind bad inputs if config.bad_input_file is None or not os.path.isfile( config.bad_input_file): bad_input = np.flatnonzero( ~np.all(data.flags['inputs'][:], axis=-1)) else: with open(config.bad_input_file, 'r') as handler: bad_input = pickle.load(handler) mlog.info("%d inputs flagged as bad." % bad_input.size) nant = data.ninput # Determine polarization product maps dbinputs = tools.get_correlator_inputs(ephemeris.unix_to_datetime( timestamps[0]), correlator='chime') dbinputs = tools.reorder_correlator_inputs(data.input, dbinputs) feedpos = tools.get_feed_positions(dbinputs) prod = defaultdict(list) dist = defaultdict(list) for pp, this_prod in enumerate(data.prod): aa, bb = this_prod inp_aa = dbinputs[aa] inp_bb = dbinputs[bb] if (aa in bad_input) or (bb in bad_input): continue if not tools.is_chime(inp_aa) or not tools.is_chime(inp_bb): continue if not config.include_intracyl and (inp_aa.cyl == inp_bb.cyl): continue if not config.include_auto and (aa == bb): continue this_dist = list(feedpos[aa, :] - feedpos[bb, :]) if tools.is_array_x(inp_aa) and tools.is_array_x(inp_bb): key = 'XX' elif tools.is_array_y(inp_aa) and tools.is_array_y(inp_bb): key = 'YY' elif not config.include_crosspol: continue elif tools.is_array_x(inp_aa) and tools.is_array_y(inp_bb): key = 'XY' elif tools.is_array_y(inp_aa) and tools.is_array_x(inp_bb): key = 'YX' else: raise RuntimeError("CHIME feeds not polarized.") prod[key].append(pp) dist[key].append(this_dist) polstr = sorted(prod.keys()) polcnt = 0 pol_sky_id = [] bmap = {} for key in polstr: prod[key] = np.array(prod[key]) dist[key] = np.array(dist[key]) p_bmap, p_ubaseline = generate_mapping(dist[key]) nubase = p_ubaseline.shape[0] bmap[key] = p_bmap + polcnt if polcnt > 0: ubaseline = np.concatenate((ubaseline, p_ubaseline), axis=0) pol_sky_id += [key] * nubase else: ubaseline = p_ubaseline.copy() pol_sky_id = [key] * nubase polcnt += nubase mlog.info("%d unique baselines" % polcnt) nsky = ubaseline.shape[0] # Create arrays to hold the results ores = {} ores['freq'] = freq ores['input'] = data.input ores['time'] = timestamps ores['coord'] = coord ores['pol'] = np.array(pol_sky_id) ores['baseline'] = ubaseline # Create array to hold gain results ores['gain'] = np.zeros((nfreq, nant, ntime), dtype=np.complex) ores['sky'] = np.zeros((nfreq, nsky, ntime), dtype=np.complex) ores['err'] = np.zeros((nfreq, nant + nsky, ntime, 2), dtype=np.float) # Loop over polarisations for key in polstr: reverse_map = bmap[key] p_prod = prod[key] isort = np.argsort(reverse_map) p_prod = p_prod[isort] p_ant1 = data.prod['input_a'][p_prod] p_ant2 = data.prod['input_b'][p_prod] p_vismap = reverse_map[isort] # Find the redundant groups tmp = np.where(np.diff(p_vismap) != 0)[0] edges = np.zeros(2 + tmp.size, dtype='int') edges[0] = 0 edges[1:-1] = tmp + 1 edges[-1] = p_vismap.size kept_base = np.unique(p_vismap) # Determine the unique antennas kept_ants = np.unique(np.concatenate([p_ant1, p_ant2])) antmap = np.zeros(kept_ants.max() + 1, dtype='int') - 1 p_nant = kept_ants.size for i in range(p_nant): antmap[kept_ants[i]] = i p_ant1_use = antmap[p_ant1].copy() p_ant2_use = antmap[p_ant2].copy() # Create matrix p_nvis = p_prod.size nred = edges.size - 1 npar = p_nant + nred A = np.zeros((p_nvis, npar), dtype=np.float32) B = np.zeros((p_nvis, npar), dtype=np.float32) for kk in range(p_nant): flag_ant1 = p_ant1_use == kk if np.any(flag_ant1): A[flag_ant1, kk] = 1.0 B[flag_ant1, kk] = 1.0 flag_ant2 = p_ant2_use == kk if np.any(flag_ant2): A[flag_ant2, kk] = 1.0 B[flag_ant2, kk] = -1.0 for ee in range(nred): A[edges[ee]:edges[ee + 1], p_nant + ee] = 1.0 B[edges[ee]:edges[ee + 1], p_nant + ee] = 1.0 # Add equations to break degeneracy if config.fix_degen: A = np.concatenate((A, np.zeros((1, npar), dtype=np.float32))) A[-1, 0:p_nant] = 1.0 B = np.concatenate((B, np.zeros((3, npar), dtype=np.float32))) B[-3, 0:p_nant] = 1.0 B[-2, 0:p_nant] = feedpos[kept_ants, 0] B[-1, 0:p_nant] = feedpos[kept_ants, 1] # Loop over frequencies for ff, find in enumerate(freq_index): mlog.info("Freq %d of %d. %0.2f MHz." % (ff + 1, nfreq, freq[ff])) cnt = 0 # Loop over files for ii, (filename, tind) in enumerate(zip(files, time_indices)): ntind = len(tind) mlog.info("Processing file %s (%d time samples)" % (filename, ntind)) # Compute noise weight with h5py.File(filename, 'r') as hf: wnoise = np.median(hf['flags/vis_weight'][find, :, :], axis=-1) # Loop over times for tt in tind: t0 = time.time() mlog.info("Time %d of %d. %d index of current file." % (cnt + 1, ntime, tt)) # Load visibilities with h5py.File(filename, 'r') as hf: snap = hf['vis'][find, :, tt] wsnap = wnoise * ( (hf['flags/vis_weight'][find, :, tt] > 0.0) & (np.abs(snap) > 0.0)).astype(np.float32) # Extract relevant products for this polarization snap = snap[p_prod] wsnap = wsnap[p_prod] # Turn into amplitude and phase, avoiding NaN mask = (wsnap > 0.0) amp = np.where(mask, np.log(np.abs(snap)), 0.0) phi = np.where(mask, np.angle(snap), 0.0) # Deal with phase wrapping for aa, bb in zip(edges[:-1], edges[1:]): dphi = phi[aa:bb] - np.sort(phi[aa:bb])[int( (bb - aa) / 2)] phi[aa:bb] += (2.0 * np.pi * (dphi < -np.pi) - 2.0 * np.pi * (dphi > np.pi)) # Add elements to fix degeneracy if config.fix_degen: amp = np.concatenate((amp, np.zeros(1))) phi = np.concatenate((phi, np.zeros(3))) # Determine noise matrix inv_diagC = wsnap * np.abs(snap)**2 * 2.0 if config.fix_degen: inv_diagC = np.concatenate((inv_diagC, np.ones(1))) # Amplitude estimate and covariance amp_param_cov = np.linalg.inv( np.dot(A.T, inv_diagC[:, np.newaxis] * A)) amp_param = np.dot(amp_param_cov, np.dot(A.T, inv_diagC * amp)) # Phase estimate and covariance if config.fix_degen: inv_diagC = np.concatenate((inv_diagC, np.ones(2))) phi_param_cov = np.linalg.inv( np.dot(B.T, inv_diagC[:, np.newaxis] * B)) phi_param = np.dot(phi_param_cov, np.dot(B.T, inv_diagC * phi)) # Save to large array ores['gain'][ff, kept_ants, cnt] = np.exp(amp_param[0:p_nant] + 1.0J * phi_param[0:p_nant]) ores['sky'][ff, kept_base, cnt] = np.exp(amp_param[p_nant:] + 1.0J * phi_param[p_nant:]) ores['err'][ff, kept_ants, cnt, 0] = np.diag(amp_param_cov[0:p_nant, 0:p_nant]) ores['err'][ff, nant + kept_base, cnt, 0] = np.diag(amp_param_cov[p_nant:, p_nant:]) ores['err'][ff, kept_ants, cnt, 1] = np.diag(phi_param_cov[0:p_nant, 0:p_nant]) ores['err'][ff, nant + kept_base, cnt, 1] = np.diag(phi_param_cov[p_nant:, p_nant:]) # Increment time counter cnt += 1 # Print time elapsed mlog.info("Took %0.1f seconds." % (time.time() - t0, )) # Save to pickle file with h5py.File(output_file, 'w') as handler: handler.attrs['date'] = date for key, val in ores.iteritems(): handler.create_dataset(key, data=val)
def main(config_file=None, logging_params=DEFAULT_LOGGING): # Setup logging log.setup_logging(logging_params) mlog = log.get_logger(__name__) # Set config config = DEFAULTS.deepcopy() if config_file is not None: config.merge(NameSpace(load_yaml_config(config_file))) # Create transit tracker source_list = FluxCatalog.sort( ) if not config.source_list else config.source_list cal_list = [ name for name, obj in FluxCatalog.iteritems() if (obj.dec >= config.min_dec) and ( obj.predict_flux(config.freq_nominal) >= config.min_flux) and ( name in source_list) ] if not cal_list: raise RuntimeError("No calibrators found.") # Sort list by flux at nominal frequency cal_list.sort( key=lambda name: FluxCatalog[name].predict_flux(config.freq_nominal)) # Add to transit tracker transit_tracker = containers.TransitTrackerOffline( nsigma=config.nsigma_source, extend_night=config.extend_night) for name in cal_list: transit_tracker[name] = FluxCatalog[name].skyfield mlog.info("Initializing offline point source processing.") search_time = config.start_time or 0 # Find all calibration files all_files = sorted( glob.glob( os.path.join(config.acq_dir, '*' + config.correlator + config.acq_suffix, '*.h5'))) if not all_files: return # Remove files whose last modified time is before the time of the most recent update all_files = [ ff for ff in all_files if (os.path.getmtime(ff) > search_time) ] if not all_files: return # Remove files that are currently locked all_files = [ ff for ff in all_files if not os.path.isfile(os.path.splitext(ff)[0] + '.lock') ] if not all_files: return # Add files to transit tracker for ff in all_files: transit_tracker.add_file(ff) # Extract point source transits ready for analysis all_transits = transit_tracker.get_transits() # Create dictionary to hold results h5_psrc_fit = {} inputmap = None # Loop over transits for transit in all_transits: src, csd, is_day, files, start, stop = transit # Discard any point sources with unusual csd value if (csd < config.min_csd) or (csd > config.max_csd): continue # Discard any point sources transiting during the day if is_day > config.process_daytime: continue mlog.info( 'Processing %s transit on CSD %d (%d files, %d time samples)' % (src, csd, len(files), stop - start + 1)) # Load inputmap if inputmap is None: if config.inputmap is None: inputmap = tools.get_correlator_inputs( ephemeris.unix_to_datetime(ephemeris.csd_to_unix(csd)), correlator=config.correlator) else: with open(config.inputmap, 'r') as handler: inputmap = pickle.load(handler) # Grab the timing correction for this transit tcorr = None if config.apply_timing: if config.timing_glob is not None: mlog.info( "Loading timing correction from extended timing solutions." ) timing_files = sorted(glob.glob(config.timing_glob)) if timing_files: try: tcorr = search_extended_timing_solutions( timing_files, ephemeris.csd_to_unix(csd)) except Exception as e: mlog.error( 'search_extended_timing_solutions failed with error: %s' % e) else: mlog.info(str(tcorr)) if tcorr is None: mlog.info( "Loading timing correction from chimetiming acquisitions.") try: tcorr = timing.load_timing_correction( files, start=start, stop=stop, window=config.timing_window, instrument=config.correlator) except Exception as e: mlog.error( 'timing.load_timing_correction failed with error: %s' % e) mlog.warning( 'No timing correction applied to %s transit on CSD %d.' % (src, csd)) else: mlog.info(str(tcorr)) # Call the main routine to process data try: outdct = offline_cal.offline_point_source_calibration( files, src, start=start, stop=stop, inputmap=inputmap, tcorr=tcorr, logging_params=logging_params, **config.analysis.as_dict()) except Exception as e: msg = 'offline_cal.offline_point_source_calibration failed with error: %s' % e mlog.error(msg) continue #raise RuntimeError(msg) # Find existing gain files for this particular point source if src not in h5_psrc_fit: output_files = find_files(config, psrc=src) if output_files is not None: output_files = output_files[-1] mlog.info('Writing %s transit on CSD %d to existing file %s.' % (src, csd, output_files)) h5_psrc_fit[src] = containers.PointSourceWriter( src, output_file=output_files, output_dir=config.output_dir, output_suffix=point_source_name_to_file_suffix(src), instrument=config.correlator, max_file_size=config.max_file_size, max_num=config.max_num_time, memory_size=0) # Associate this gain calibration to the transit time this_time = ephemeris.transit_times(FluxCatalog[src].skyfield, ephemeris.csd_to_unix(csd))[0] outdct['csd'] = csd outdct['is_daytime'] = is_day outdct['acquisition'] = os.path.basename(os.path.dirname(files[0])) # Write to output file mlog.info('Writing to disk results from %s transit on CSD %d.' % (src, csd)) h5_psrc_fit[src].write(this_time, **outdct) # Dump an individual file for this point source transit mlog.info('Dumping to disk single file for %s transit on CSD %d.' % (src, csd)) dump_dir = os.path.join(config.output_dir, 'point_source_gains') containers.mkdir(dump_dir) dump_file = os.path.join(dump_dir, '%s_csd_%d.h5' % (src.lower(), csd)) h5_psrc_fit[src].dump(dump_file, datasets=[ 'csd', 'acquisition', 'is_daytime', 'gain', 'weight', 'timing', 'model' ]) mlog.info('Finished analysis of %s transit on CSD %d.' % (src, csd))
def main(config_file=None, logging_params=DEFAULT_LOGGING): # Load config config = DEFAULTS.deepcopy() if config_file is not None: config.merge(NameSpace(load_yaml_config(config_file))) # Setup logging log.setup_logging(logging_params) logger = log.get_logger(__name__) ## Load data for flagging # Load fpga restarts time_fpga_restart = [] if config.fpga_restart_file is not None: with open(config.fpga_restart_file, 'r') as handler: for line in handler: time_fpga_restart.append( ephemeris.datetime_to_unix( ephemeris.timestr_to_datetime(line.split('_')[0]))) time_fpga_restart = np.array(time_fpga_restart) # Load housekeeping flag if config.housekeeping_file is not None: ftemp = TempData.from_acq_h5(config.housekeeping_file, datasets=["time_flag"]) else: ftemp = None # Load jump data if config.jump_file is not None: with h5py.File(config.jump_file, 'r') as handler: jump_time = handler["time"][:] jump_size = handler["jump_size"][:] else: jump_time = None jump_size = None # Load rain data if config.rain_file is not None: with h5py.File(config.rain_file, 'r') as handler: rain_ranges = handler["time_range_conservative"][:] else: rain_ranges = [] # Load data flags data_flags = {} if config.data_flags: finder.connect_database() flag_types = finder.DataFlagType.select() possible_data_flags = [] for ft in flag_types: possible_data_flags.append(ft.name) if ft.name in config.data_flags: new_data_flags = finder.DataFlag.select().where( finder.DataFlag.type == ft) data_flags[ft.name] = list(new_data_flags) # Set desired range of time start_time = (ephemeris.datetime_to_unix( datetime.datetime( *config.start_date)) if config.start_date is not None else None) end_time = (ephemeris.datetime_to_unix(datetime.datetime( *config.end_date)) if config.end_date is not None else None) ## Find gain files files = {} for src in config.sources: files[src] = sorted( glob.glob( os.path.join(config.directory, src.lower(), "%s_%s_lsd_*.h5" % ( config.prefix, src.lower(), )))) csd = {} for src in config.sources: csd[src] = np.array( [int(os.path.splitext(ff)[0][-4:]) for ff in files[src]]) for src in config.sources: logger.info("%s: %d files" % (src, len(csd[src]))) ## Remove files that occur during flag csd_flag = {} for src in config.sources: body = ephemeris.source_dictionary[src] csd_flag[src] = np.ones(csd[src].size, dtype=np.bool) for ii, cc in enumerate(csd[src][:]): ttrans = ephemeris.transit_times(body, ephemeris.csd_to_unix(cc))[0] if (start_time is not None) and (ttrans < start_time): csd_flag[src][ii] = False continue if (end_time is not None) and (ttrans > end_time): csd_flag[src][ii] = False continue # If requested, remove daytime transits if not config.include_daytime.get( src, config.include_daytime.default) and daytime_flag( ttrans)[0]: logger.info("%s CSD %d: daytime transit" % (src, cc)) csd_flag[src][ii] = False continue # Remove transits during HKP drop out if ftemp is not None: itemp = np.flatnonzero( (ftemp.time[:] >= (ttrans - config.transit_window)) & (ftemp.time[:] <= (ttrans + config.transit_window))) tempflg = ftemp['time_flag'][itemp] if (tempflg.size == 0) or ((np.sum(tempflg, dtype=np.float32) / float(tempflg.size)) < 0.50): logger.info("%s CSD %d: no housekeeping" % (src, cc)) csd_flag[src][ii] = False continue # Remove transits near jumps if jump_time is not None: njump = np.sum((jump_size > config.min_jump_size) & (jump_time > (ttrans - config.jump_window)) & (jump_time < ttrans)) if njump > config.max_njump: logger.info("%s CSD %d: %d jumps before" % (src, cc, njump)) csd_flag[src][ii] = False continue # Remove transits near rain for rng in rain_ranges: if (((ttrans - config.transit_window) <= rng[1]) and ((ttrans + config.transit_window) >= rng[0])): logger.info("%s CSD %d: during rain" % (src, cc)) csd_flag[src][ii] = False break # Remove transits during data flag for name, flag_list in data_flags.items(): if csd_flag[src][ii]: for flg in flag_list: if (((ttrans - config.transit_window) <= flg.finish_time) and ((ttrans + config.transit_window) >= flg.start_time)): logger.info("%s CSD %d: %s flag" % (src, cc, name)) csd_flag[src][ii] = False break # Print number of files left after flagging for src in config.sources: logger.info("%s: %d files (after flagging)" % (src, np.sum(csd_flag[src]))) ## Construct pair wise differences npair = len(config.diff_pair) shift = [nd * 24.0 * 3600.0 for nd in config.nday_shift] calmap = [] calpair = [] for (tsrc, csrc), sh in zip(config.diff_pair, shift): body_test = ephemeris.source_dictionary[tsrc] body_cal = ephemeris.source_dictionary[csrc] for ii, cc in enumerate(csd[tsrc]): if csd_flag[tsrc][ii]: test_transit = ephemeris.transit_times( body_test, ephemeris.csd_to_unix(cc))[0] cal_transit = ephemeris.transit_times(body_cal, test_transit + sh)[0] cal_csd = int(np.fix(ephemeris.unix_to_csd(cal_transit))) ttrans = np.sort([test_transit, cal_transit]) if cal_csd in csd[csrc]: jj = list(csd[csrc]).index(cal_csd) if csd_flag[csrc][jj] and not np.any( (time_fpga_restart >= ttrans[0]) & (time_fpga_restart <= ttrans[1])): calmap.append([ii, jj]) calpair.append([tsrc, csrc]) calmap = np.array(calmap) calpair = np.array(calpair) ntransit = calmap.shape[0] logger.info("%d total transit pairs" % ntransit) for ii in range(ntransit): t1 = ephemeris.transit_times( ephemeris.source_dictionary[calpair[ii, 0]], ephemeris.csd_to_unix(csd[calpair[ii, 0]][calmap[ii, 0]]))[0] t2 = ephemeris.transit_times( ephemeris.source_dictionary[calpair[ii, 1]], ephemeris.csd_to_unix(csd[calpair[ii, 1]][calmap[ii, 1]]))[0] logger.info("%s (%d) - %s (%d): %0.1f hr" % (calpair[ii, 0], csd_flag[calpair[ii, 0]][calmap[ii, 0]], calpair[ii, 1], csd_flag[calpair[ii, 1]][calmap[ii, 1]], (t1 - t2) / 3600.0)) # Determine unique diff pairs diff_name = np.array(['%s/%s' % tuple(cp) for cp in calpair]) uniq_diff, lbl_diff, cnt_diff = np.unique(diff_name, return_inverse=True, return_counts=True) ndiff = uniq_diff.size for ud, udcnt in zip(uniq_diff, cnt_diff): logger.info("%s: %d transit pairs" % (ud, udcnt)) ## Load gains inputmap = tools.get_correlator_inputs(datetime.datetime.utcnow(), correlator='chime') ninput = len(inputmap) nfreq = 1024 # Set up gain arrays gain = np.zeros((2, nfreq, ninput, ntransit), dtype=np.complex64) weight = np.zeros((2, nfreq, ninput, ntransit), dtype=np.float32) input_sort = np.zeros((2, ninput, ntransit), dtype=np.int) kcsd = np.zeros((2, ntransit), dtype=np.float32) timestamp = np.zeros((2, ntransit), dtype=np.float64) is_daytime = np.zeros((2, ntransit), dtype=np.bool) for tt in range(ntransit): for kk, (src, ind) in enumerate(zip(calpair[tt], calmap[tt])): body = ephemeris.source_dictionary[src] filename = files[src][ind] logger.info("%s: %s" % (src, filename)) temp = containers.StaticGainData.from_file(filename) freq = temp.freq[:] inputs = temp.input[:] isort = reorder_inputs(inputmap, inputs) inputs = inputs[isort] gain[kk, :, :, tt] = temp.gain[:, isort] weight[kk, :, :, tt] = temp.weight[:, isort] input_sort[kk, :, tt] = isort kcsd[kk, tt] = temp.attrs['lsd'] timestamp[kk, tt] = ephemeris.transit_times( body, ephemeris.csd_to_unix(kcsd[kk, tt]))[0] is_daytime[kk, tt] = daytime_flag(timestamp[kk, tt])[0] if np.any(isort != np.arange(isort.size)): logger.info("Input ordering has changed: %s" % ephemeris.unix_to_datetime( timestamp[kk, tt]).strftime("%Y-%m-%d")) logger.info("") inputs = np.array([(inp.id, inp.input_sn) for inp in inputmap], dtype=[('chan_id', 'u2'), ('correlator_input', 'S32')]) ## Load input flags inpflg = np.ones((2, ninput, ntransit), dtype=np.bool) min_flag_time = np.min(timestamp) - 7.0 * 24.0 * 60.0 * 60.0 max_flag_time = np.max(timestamp) + 7.0 * 24.0 * 60.0 * 60.0 flaginput_files = sorted( glob.glob( os.path.join(config.flaginput_dir, "*" + config.flaginput_suffix, "*.h5"))) if flaginput_files: logger.info("Found %d flaginput files." % len(flaginput_files)) tmp = andata.FlagInputData.from_acq_h5(flaginput_files, datasets=()) start, stop = [ int(yy) for yy in np.percentile( np.flatnonzero((tmp.time[:] >= min_flag_time) & (tmp.time[:] <= max_flag_time)), [0, 100]) ] cont = andata.FlagInputData.from_acq_h5(flaginput_files, start=start, stop=stop, datasets=['flag']) for kk in range(2): inpflg[kk, :, :] = cont.resample('flag', timestamp[kk], transpose=True) logger.info("Flaginput time offsets in minutes (pair %d):" % kk) logger.info( str( np.fix((cont.time[cont.search_update_time(timestamp[kk])] - timestamp[kk]) / 60.0).astype(np.int))) # Sort flags so they are in same order for tt in range(ntransit): for kk in range(2): inpflg[kk, :, tt] = inpflg[kk, input_sort[kk, :, tt], tt] # Do not apply input flag to phase reference for ii in config.index_phase_ref: inpflg[:, ii, :] = True ## Flag out gains with high uncertainty and frequencies with large fraction of data flagged frac_err = tools.invert_no_zero(np.sqrt(weight) * np.abs(gain)) flag = np.all((weight > 0.0) & (np.abs(gain) > 0.0) & (frac_err < config.max_uncertainty), axis=0) freq_flag = ((np.sum(flag, axis=(1, 2), dtype=np.float32) / float(np.prod(flag.shape[1:]))) > config.freq_threshold) if config.apply_rfi_mask: freq_flag &= np.logical_not(rfi.frequency_mask(freq)) flag = flag & freq_flag[:, np.newaxis, np.newaxis] good_freq = np.flatnonzero(freq_flag) logger.info("Number good frequencies %d" % good_freq.size) ## Generate flags with more conservative cuts on frequency c_flag = flag & np.all(frac_err < config.conservative.max_uncertainty, axis=0) c_freq_flag = ((np.sum(c_flag, axis=(1, 2), dtype=np.float32) / float(np.prod(c_flag.shape[1:]))) > config.conservative.freq_threshold) if config.conservative.apply_rfi_mask: c_freq_flag &= np.logical_not(rfi.frequency_mask(freq)) c_flag = c_flag & c_freq_flag[:, np.newaxis, np.newaxis] c_good_freq = np.flatnonzero(c_freq_flag) logger.info("Number good frequencies (conservative thresholds) %d" % c_good_freq.size) ## Apply input flags flag &= np.all(inpflg[:, np.newaxis, :, :], axis=0) ## Update flags based on beam flag if config.beam_flag_file is not None: dbeam = andata.BaseData.from_acq_h5(config.beam_flag_file) db_csd = np.floor(ephemeris.unix_to_csd(dbeam.index_map['time'][:])) for ii, name in enumerate(config.beam_flag_datasets): logger.info("Applying %s beam flag." % name) if not ii: db_flag = dbeam.flags[name][:] else: db_flag &= dbeam.flags[name][:] cnt = 0 for ii, dbc in enumerate(db_csd): this_csd = np.flatnonzero(np.any(kcsd == dbc, axis=0)) if this_csd.size > 0: logger.info("Beam flag for %d matches %s." % (dbc, str(kcsd[:, this_csd]))) flag[:, :, this_csd] &= db_flag[np.newaxis, :, ii, np.newaxis] cnt += 1 logger.info("Applied %0.1f percent of the beam flags" % (100.0 * cnt / float(db_csd.size), )) ## Flag inputs with large amount of missing data input_frac_flagged = ( np.sum(flag[good_freq, :, :], axis=(0, 2), dtype=np.float32) / float(good_freq.size * ntransit)) input_flag = input_frac_flagged > config.input_threshold for ii in config.index_phase_ref: logger.info("Phase reference %d has %0.3f fraction of data flagged." % (ii, input_frac_flagged[ii])) input_flag[ii] = True good_input = np.flatnonzero(input_flag) flag = flag & input_flag[np.newaxis, :, np.newaxis] logger.info("Number good inputs %d" % good_input.size) ## Calibrate gaincal = gain[0] * tools.invert_no_zero(gain[1]) frac_err_cal = np.sqrt(frac_err[0]**2 + frac_err[1]**2) count = np.sum(flag, axis=-1, dtype=np.int) stat_flag = count > config.min_num_transit ## Calculate phase amp = np.abs(gaincal) phi = np.angle(gaincal) ## Calculate polarisation groups pol_dict = {'E': 'X', 'S': 'Y'} cyl_dict = {2: 'A', 3: 'B', 4: 'C', 5: 'D'} if config.group_by_cyl: group_id = [ (inp.pol, inp.cyl) if tools.is_chime(inp) and (ii in good_input) else None for ii, inp in enumerate(inputmap) ] else: group_id = [ inp.pol if tools.is_chime(inp) and (ii in good_input) else None for ii, inp in enumerate(inputmap) ] ugroup_id = sorted([uidd for uidd in set(group_id) if uidd is not None]) ngroup = len(ugroup_id) group_list_noref = [ np.array([ gg for gg, gid in enumerate(group_id) if (gid == ugid) and gg not in config.index_phase_ref ]) for ugid in ugroup_id ] group_list = [ np.array([gg for gg, gid in enumerate(group_id) if gid == ugid]) for ugid in ugroup_id ] if config.group_by_cyl: group_str = [ "%s-%s" % (pol_dict[pol], cyl_dict[cyl]) for pol, cyl in ugroup_id ] else: group_str = [pol_dict[pol] for pol in ugroup_id] index_phase_ref = [] for gstr, igroup in zip(group_str, group_list): candidate = [ii for ii in config.index_phase_ref if ii in igroup] if len(candidate) != 1: index_phase_ref.append(None) else: index_phase_ref.append(candidate[0]) logger.info( "Phase reference: %s" % ', '.join(['%s = %s' % tpl for tpl in zip(group_str, index_phase_ref)])) ## Apply thermal correction to amplitude if config.amp_thermal.enabled: logger.info("Applying thermal correction.") # Load the temperatures tdata = TempData.from_acq_h5(config.amp_thermal.filename) index = tdata.search_sensors(config.amp_thermal.sensor)[0] temp = tdata.datasets[config.amp_thermal.field][index] temp_func = scipy.interpolate.interp1d(tdata.time, temp, **config.amp_thermal.interp) itemp = temp_func(timestamp) dtemp = itemp[0] - itemp[1] flag_func = scipy.interpolate.interp1d( tdata.time, tdata.datasets['flag'][index].astype(np.float32), **config.amp_thermal.interp) dtemp_flag = np.all(flag_func(timestamp) == 1.0, axis=0) flag &= dtemp_flag[np.newaxis, np.newaxis, :] for gstr, igroup in zip(group_str, group_list): pstr = gstr[0] thermal_coeff = np.polyval(config.amp_thermal.coeff[pstr], freq) gthermal = 1.0 + thermal_coeff[:, np.newaxis, np.newaxis] * dtemp[ np.newaxis, np.newaxis, :] amp[:, igroup, :] *= tools.invert_no_zero(gthermal) ## Compute common mode if config.subtract_common_mode_before: logger.info("Calculating common mode amplitude and phase.") cmn_amp, flag_cmn_amp = compute_common_mode(amp, flag, group_list_noref, median=False) cmn_phi, flag_cmn_phi = compute_common_mode(phi, flag, group_list_noref, median=False) # Subtract common mode (from phase only) logger.info("Subtracting common mode phase.") group_flag = np.zeros((ngroup, ninput), dtype=np.bool) for gg, igroup in enumerate(group_list): group_flag[gg, igroup] = True phi[:, igroup, :] = phi[:, igroup, :] - cmn_phi[:, gg, np.newaxis, :] for iref in index_phase_ref: if (iref is not None) and (iref in igroup): flag[:, iref, :] = flag_cmn_phi[:, gg, :] ## If requested, determine and subtract a delay template if config.fit_delay_before: logger.info("Fitting delay template.") omega = timing.FREQ_TO_OMEGA * freq tau, tau_flag, _ = construct_delay_template( omega, phi, c_flag & flag, min_num_freq_for_delay_fit=config.min_num_freq_for_delay_fit) # Compute residuals logger.info("Subtracting delay template.") phi = phi - tau[np.newaxis, :, :] * omega[:, np.newaxis, np.newaxis] ## Normalize by median over time logger.info("Calculating median amplitude and phase.") med_amp = np.zeros((nfreq, ninput, ndiff), dtype=amp.dtype) med_phi = np.zeros((nfreq, ninput, ndiff), dtype=phi.dtype) count_by_diff = np.zeros((nfreq, ninput, ndiff), dtype=np.int) stat_flag_by_diff = np.zeros((nfreq, ninput, ndiff), dtype=np.bool) def weighted_mean(yy, ww, axis=-1): return np.sum(ww * yy, axis=axis) * tools.invert_no_zero( np.sum(ww, axis=axis)) for dd in range(ndiff): this_diff = np.flatnonzero(lbl_diff == dd) this_flag = flag[:, :, this_diff] this_amp = amp[:, :, this_diff] this_amp_err = this_amp * frac_err_cal[:, :, this_diff] * this_flag.astype( np.float32) this_phi = phi[:, :, this_diff] this_phi_err = frac_err_cal[:, :, this_diff] * this_flag.astype( np.float32) count_by_diff[:, :, dd] = np.sum(this_flag, axis=-1, dtype=np.int) stat_flag_by_diff[:, :, dd] = count_by_diff[:, :, dd] > config.min_num_transit if config.weighted_mean == 2: logger.info("Calculating inverse variance weighted mean.") med_amp[:, :, dd] = weighted_mean(this_amp, tools.invert_no_zero(this_amp_err**2), axis=-1) med_phi[:, :, dd] = weighted_mean(this_phi, tools.invert_no_zero(this_phi_err**2), axis=-1) elif config.weighted_mean == 1: logger.info("Calculating uniform weighted mean.") med_amp[:, :, dd] = weighted_mean(this_amp, this_flag.astype(np.float32), axis=-1) med_phi[:, :, dd] = weighted_mean(this_phi, this_flag.astype(np.float32), axis=-1) else: logger.info("Calculating median value.") for ff in range(nfreq): for ii in range(ninput): if np.any(this_flag[ff, ii, :]): med_amp[ff, ii, dd] = wq.median( this_amp[ff, ii, :], this_flag[ff, ii, :].astype(np.float32)) med_phi[ff, ii, dd] = wq.median( this_phi[ff, ii, :], this_flag[ff, ii, :].astype(np.float32)) damp = np.zeros_like(amp) dphi = np.zeros_like(phi) for dd in range(ndiff): this_diff = np.flatnonzero(lbl_diff == dd) damp[:, :, this_diff] = amp[:, :, this_diff] * tools.invert_no_zero( med_amp[:, :, dd, np.newaxis]) - 1.0 dphi[:, :, this_diff] = phi[:, :, this_diff] - med_phi[:, :, dd, np.newaxis] # Compute common mode if not config.subtract_common_mode_before: logger.info("Calculating common mode amplitude and phase.") cmn_amp, flag_cmn_amp = compute_common_mode(damp, flag, group_list_noref, median=True) cmn_phi, flag_cmn_phi = compute_common_mode(dphi, flag, group_list_noref, median=True) # Subtract common mode (from phase only) logger.info("Subtracting common mode phase.") group_flag = np.zeros((ngroup, ninput), dtype=np.bool) for gg, igroup in enumerate(group_list): group_flag[gg, igroup] = True dphi[:, igroup, :] = dphi[:, igroup, :] - cmn_phi[:, gg, np.newaxis, :] for iref in index_phase_ref: if (iref is not None) and (iref in igroup): flag[:, iref, :] = flag_cmn_phi[:, gg, :] ## Compute RMS logger.info("Calculating RMS of amplitude and phase.") mad_amp = np.zeros((nfreq, ninput), dtype=amp.dtype) std_amp = np.zeros((nfreq, ninput), dtype=amp.dtype) mad_phi = np.zeros((nfreq, ninput), dtype=phi.dtype) std_phi = np.zeros((nfreq, ninput), dtype=phi.dtype) mad_amp_by_diff = np.zeros((nfreq, ninput, ndiff), dtype=amp.dtype) std_amp_by_diff = np.zeros((nfreq, ninput, ndiff), dtype=amp.dtype) mad_phi_by_diff = np.zeros((nfreq, ninput, ndiff), dtype=phi.dtype) std_phi_by_diff = np.zeros((nfreq, ninput, ndiff), dtype=phi.dtype) for ff in range(nfreq): for ii in range(ninput): this_flag = flag[ff, ii, :] if np.any(this_flag): std_amp[ff, ii] = np.std(damp[ff, ii, this_flag]) std_phi[ff, ii] = np.std(dphi[ff, ii, this_flag]) mad_amp[ff, ii] = 1.48625 * wq.median( np.abs(damp[ff, ii, :]), this_flag.astype(np.float32)) mad_phi[ff, ii] = 1.48625 * wq.median( np.abs(dphi[ff, ii, :]), this_flag.astype(np.float32)) for dd in range(ndiff): this_diff = this_flag & (lbl_diff == dd) if np.any(this_diff): std_amp_by_diff[ff, ii, dd] = np.std(damp[ff, ii, this_diff]) std_phi_by_diff[ff, ii, dd] = np.std(dphi[ff, ii, this_diff]) mad_amp_by_diff[ff, ii, dd] = 1.48625 * wq.median( np.abs(damp[ff, ii, :]), this_diff.astype(np.float32)) mad_phi_by_diff[ff, ii, dd] = 1.48625 * wq.median( np.abs(dphi[ff, ii, :]), this_diff.astype(np.float32)) ## Construct delay template if not config.fit_delay_before: logger.info("Fitting delay template.") omega = timing.FREQ_TO_OMEGA * freq tau, tau_flag, _ = construct_delay_template( omega, dphi, c_flag & flag, min_num_freq_for_delay_fit=config.min_num_freq_for_delay_fit) # Compute residuals logger.info("Subtracting delay template from phase.") resid = (dphi - tau[np.newaxis, :, :] * omega[:, np.newaxis, np.newaxis]) * flag.astype(np.float32) else: resid = dphi tau_count = np.sum(tau_flag, axis=-1, dtype=np.int) tau_stat_flag = tau_count > config.min_num_transit tau_count_by_diff = np.zeros((ninput, ndiff), dtype=np.int) tau_stat_flag_by_diff = np.zeros((ninput, ndiff), dtype=np.bool) for dd in range(ndiff): this_diff = np.flatnonzero(lbl_diff == dd) tau_count_by_diff[:, dd] = np.sum(tau_flag[:, this_diff], axis=-1, dtype=np.int) tau_stat_flag_by_diff[:, dd] = tau_count_by_diff[:, dd] > config.min_num_transit ## Calculate statistics of residuals std_resid = np.zeros((nfreq, ninput), dtype=phi.dtype) mad_resid = np.zeros((nfreq, ninput), dtype=phi.dtype) std_resid_by_diff = np.zeros((nfreq, ninput, ndiff), dtype=phi.dtype) mad_resid_by_diff = np.zeros((nfreq, ninput, ndiff), dtype=phi.dtype) for ff in range(nfreq): for ii in range(ninput): this_flag = flag[ff, ii, :] if np.any(this_flag): std_resid[ff, ii] = np.std(resid[ff, ii, this_flag]) mad_resid[ff, ii] = 1.48625 * wq.median( np.abs(resid[ff, ii, :]), this_flag.astype(np.float32)) for dd in range(ndiff): this_diff = this_flag & (lbl_diff == dd) if np.any(this_diff): std_resid_by_diff[ff, ii, dd] = np.std(resid[ff, ii, this_diff]) mad_resid_by_diff[ff, ii, dd] = 1.48625 * wq.median( np.abs(resid[ff, ii, :]), this_diff.astype(np.float32)) ## Calculate statistics of delay template mad_tau = np.zeros((ninput, ), dtype=phi.dtype) std_tau = np.zeros((ninput, ), dtype=phi.dtype) mad_tau_by_diff = np.zeros((ninput, ndiff), dtype=phi.dtype) std_tau_by_diff = np.zeros((ninput, ndiff), dtype=phi.dtype) for ii in range(ninput): this_flag = tau_flag[ii] if np.any(this_flag): std_tau[ii] = np.std(tau[ii, this_flag]) mad_tau[ii] = 1.48625 * wq.median(np.abs(tau[ii]), this_flag.astype(np.float32)) for dd in range(ndiff): this_diff = this_flag & (lbl_diff == dd) if np.any(this_diff): std_tau_by_diff[ii, dd] = np.std(tau[ii, this_diff]) mad_tau_by_diff[ii, dd] = 1.48625 * wq.median( np.abs(tau[ii]), this_diff.astype(np.float32)) ## Define output res = { "timestamp": { "data": timestamp, "axis": ["div", "time"] }, "is_daytime": { "data": is_daytime, "axis": ["div", "time"] }, "csd": { "data": kcsd, "axis": ["div", "time"] }, "pair_map": { "data": lbl_diff, "axis": ["time"] }, "pair_count": { "data": cnt_diff, "axis": ["pair"] }, "gain": { "data": gaincal, "axis": ["freq", "input", "time"] }, "frac_err": { "data": frac_err_cal, "axis": ["freq", "input", "time"] }, "flags/gain": { "data": flag, "axis": ["freq", "input", "time"], "flag": True }, "flags/gain_conservative": { "data": c_flag, "axis": ["freq", "input", "time"], "flag": True }, "flags/count": { "data": count, "axis": ["freq", "input"], "flag": True }, "flags/stat": { "data": stat_flag, "axis": ["freq", "input"], "flag": True }, "flags/count_by_pair": { "data": count_by_diff, "axis": ["freq", "input", "pair"], "flag": True }, "flags/stat_by_pair": { "data": stat_flag_by_diff, "axis": ["freq", "input", "pair"], "flag": True }, "med_amp": { "data": med_amp, "axis": ["freq", "input", "pair"] }, "med_phi": { "data": med_phi, "axis": ["freq", "input", "pair"] }, "flags/group_flag": { "data": group_flag, "axis": ["group", "input"], "flag": True }, "cmn_amp": { "data": cmn_amp, "axis": ["freq", "group", "time"] }, "cmn_phi": { "data": cmn_phi, "axis": ["freq", "group", "time"] }, "amp": { "data": damp, "axis": ["freq", "input", "time"] }, "phi": { "data": dphi, "axis": ["freq", "input", "time"] }, "std_amp": { "data": std_amp, "axis": ["freq", "input"] }, "std_amp_by_pair": { "data": std_amp_by_diff, "axis": ["freq", "input", "pair"] }, "mad_amp": { "data": mad_amp, "axis": ["freq", "input"] }, "mad_amp_by_pair": { "data": mad_amp_by_diff, "axis": ["freq", "input", "pair"] }, "std_phi": { "data": std_phi, "axis": ["freq", "input"] }, "std_phi_by_pair": { "data": std_phi_by_diff, "axis": ["freq", "input", "pair"] }, "mad_phi": { "data": mad_phi, "axis": ["freq", "input"] }, "mad_phi_by_pair": { "data": mad_phi_by_diff, "axis": ["freq", "input", "pair"] }, "tau": { "data": tau, "axis": ["input", "time"] }, "flags/tau": { "data": tau_flag, "axis": ["input", "time"], "flag": True }, "flags/tau_count": { "data": tau_count, "axis": ["input"], "flag": True }, "flags/tau_stat": { "data": tau_stat_flag, "axis": ["input"], "flag": True }, "flags/tau_count_by_pair": { "data": tau_count_by_diff, "axis": ["input", "pair"], "flag": True }, "flags/tau_stat_by_pair": { "data": tau_stat_flag_by_diff, "axis": ["input", "pair"], "flag": True }, "std_tau": { "data": std_tau, "axis": ["input"] }, "std_tau_by_pair": { "data": std_tau_by_diff, "axis": ["input", "pair"] }, "mad_tau": { "data": mad_tau, "axis": ["input"] }, "mad_tau_by_pair": { "data": mad_tau_by_diff, "axis": ["input", "pair"] }, "resid_phi": { "data": resid, "axis": ["freq", "input", "time"] }, "std_resid_phi": { "data": std_resid, "axis": ["freq", "input"] }, "std_resid_phi_by_pair": { "data": std_resid_by_diff, "axis": ["freq", "input", "pair"] }, "mad_resid_phi": { "data": mad_resid, "axis": ["freq", "input"] }, "mad_resid_phi_by_pair": { "data": mad_resid_by_diff, "axis": ["freq", "input", "pair"] }, } ## Create the output container logger.info("Creating StabilityData container.") data = StabilityData() data.create_index_map( "div", np.array(["numerator", "denominator"], dtype=np.string_)) data.create_index_map("pair", np.array(uniq_diff, dtype=np.string_)) data.create_index_map("group", np.array(group_str, dtype=np.string_)) data.create_index_map("freq", freq) data.create_index_map("input", inputs) data.create_index_map("time", timestamp[0, :]) logger.info("Writing datsets to container.") for name, dct in res.iteritems(): is_flag = dct.get('flag', False) if is_flag: dset = data.create_flag(name.split('/')[-1], data=dct['data']) else: dset = data.create_dataset(name, data=dct['data']) dset.attrs['axis'] = np.array(dct['axis'], dtype=np.string_) data.attrs['phase_ref'] = np.array( [iref for iref in index_phase_ref if iref is not None]) # Determine the output filename and save results start_time, end_time = ephemeris.unix_to_datetime( np.percentile(timestamp, [0, 100])) tfmt = "%Y%m%d" night_str = 'night_' if not np.any(is_daytime) else '' output_file = os.path.join( config.output_dir, "%s_%s_%sraw_stability_data.h5" % (start_time.strftime(tfmt), end_time.strftime(tfmt), night_str)) logger.info("Saving results to %s." % output_file) data.save(output_file)
def solve_ps_transit(filename, corrs, feeds, inp, src, nfreq=1024, transposed=False, nfeed=128): """ Function that fringestops time slice where point source is in the beam, takes all correlations for a given polarization, and then eigendecomposes the correlation matrix freq by freq after removing the fpga phases. It will also plot intermediate steps to verify the phase solution. Parameters ---------- filename : np.str Full-path filename corrs : list List of correlations to use in solver feeds : list List of feeds to use inp : Correlator inputs (output of ch_util.tools.get_correlator_inputs) src : ephem.FixedBody Source to calibrate off of. e.g. ch_util.ephemeris.TauA Returns ------- Gains : np.array Complex gain array (nfreq, nfeed) """ nsplit = 32 # Number of freq chunks to divide nfreq into del_t = 800 f = h5py.File(filename, 'r') # Add half an integration time to each. Hack. times = f['index_map']['time'].value['ctime'] + 10.50 src_trans = eph.transit_times(src, times[0]) # try to account for differential arrival time from # cylinder rotation. del_phi = (src._dec - np.radians(eph.CHIMELATITUDE)) \ * np.sin(np.radians(1.988)) del_phi *= (24 * 3600.0) / (2 * np.pi) # Adjust the transit time accordingly src_trans += del_phi # Select +- del_t of transit, accounting for the mispointing t_range = np.where((times < src_trans + del_t) & (times > src_trans - del_t))[0] print "\n...... This data is from %s starting at RA: %f ...... \n" \ % (eph.unix_to_datetime(times[0]), eph.transit_RA(times[0])) assert (len(t_range) > 0), "Source is not in this acq" # Create gains array to fill in solution Gains = np.zeros([nfreq, nfeed], np.complex128) print "Starting the solver" times = times[t_range[0]:t_range[-1]] k=0 # Start at a strong freq channel that can be plotted # and from which we can find the noise source on-sample for i in range(12, nsplit) + range(0, 12): k+=1 # Divides the arrays up into nfreq / nsplit freq chunks and solves those frq = range(i * nfreq // nsplit, (i+1) * nfreq // nsplit) print " %d:%d \n" % (frq[0], frq[-1]) # Read in time and freq slice if data has already been transposed if transposed is True: v = f['vis'][frq[0]:frq[-1]+1, corrs, :] v = v[..., t_range[0]:t_range[-1]] vis = v['r'] + 1j * v['i'] if k==1: autos = auto_corrs(nfeed) offp = (abs(vis[:, autos, 0::2]).mean() > \ (abs(vis[:, autos, 1::2]).mean())).astype(int) times = times[offp::2] vis = vis[..., offp::2] gg = f['gain_coeff'][frq[0]:frq[-1]+1, feeds, t_range[0]:t_range[-1]][..., offp::2] gain_coeff = gg['r'] + 1j * gg['i'] del gg # Read in time and freq slice if data has not yet been transposed if transposed is False: print "TRANSPOSED V OF CODE DOESN'T WORK YET!" v = f['vis'][t_range[0]:t_range[-1]:2, frq[0]:frq[-1]+1, corrs] vis = v['r'][:] + 1j * v['i'][:] del v gg = f['gain_coeff'][0, frq[0]:frq[-1]+1, feeds] gain_coeff = gg['r'][:] + 1j * gg['i'][:] vis = vis[..., offp::2] vis = np.transpose(vis, (1, 2, 0)) # Remove fpga gains from data vis = remove_fpga_gains(vis, gain_coeff, nfeed=nfeed, triu=False) # Remove offset from galaxy vis -= 0.5 * (vis[..., 0] + vis[..., -1])[..., np.newaxis] # Get physical freq for fringestopper freq_MHZ = 800.0 - np.array(frq) / 1024.0 * 400. baddies = np.where(np.isnan(tools.get_feed_positions(inp)[:, 0]))[0] a, b, c = select_corrs(baddies, nfeed=128) vis[:, a + b] = 0.0 # Fringestop to location of "src" data_fs = tools.fringestop_pathfinder(vis, eph.transit_RA(times), freq_MHZ, inp, src) del vis dr, sol_arr = solve_gain(data_fs) # Find index of point source transit drlist = np.argmax(dr, axis=-1) # If multiple freq channels are zerod, the trans_pix # will end up being 0. This is bad, so ensure that # you are only looking for non-zero transit pixels. drlist = [x for x in drlist if x != 0] trans_pix = np.argmax(np.bincount(drlist)) assert trans_pix != 0.0 Gains[frq] = sol_arr[..., trans_pix-3:trans_pix+4].mean(-1) zz = h5py.File('data' + str(i) + '.hdf5','w') zz.create_dataset('data', data=dr) zz.close() print "%f, %d Nans out of %d" % (np.isnan(sol_arr).sum(), np.isnan(Gains[frq]).sum(), np.isnan(Gains[frq]).sum()) print trans_pix, sol_arr[..., trans_pix-3:trans_pix+4].mean(-1).sum(), sol_arr.mean(-1).sum() # Plot up post-fs phases to see if everything has been fixed if frq[0] == 12 * nsplit: print "======================" print " Plotting up freq: %d" % frq[0] print "======================" img_nm = './phs_plots/dfs' + np.str(frq[17]) + np.str(np.int(time.time())) +'.png' img_nmcorr = './phs_plots/dfs' + np.str(frq[17]) + np.str(np.int(time.time())) +'corr.png' plt_gains(data_fs, 0, img_name=img_nm, bad_chans=baddies) dfs_corr = correct_dfs(data_fs, np.angle(Gains[frq])[..., np.newaxis], nfeed=128) plt_gains(dfs_corr, 0, img_name=img_nmcorr, bad_chans=baddies) del dfs_corr del data_fs, a return Gains
def fs_from_file(filename, frq, src, del_t=900, transposed=True, subtract_avg=False): f = h5py.File(filename, 'r') times = f['index_map']['time'].value['ctime'] + 10.6 src_trans = eph.transit_times(src, times[0]) # try to account for differential arrival time from cylinder rotation. del_phi = (src._dec - np.radians(eph.CHIMELATITUDE)) * np.sin( np.radians(1.988)) del_phi *= (24 * 3600.0) / (2 * np.pi) # Adjust the transit time accordingly src_trans += del_phi # Select +- del_t of transit, accounting for the mispointing t_range = np.where((times < src_trans + del_t) & (times > src_trans - del_t))[0] times = times[t_range[0]:t_range[-1]] #[offp::2] test print "Time range:", times[0], times[-1] print "\n...... This data is from %s starting at RA: %f ...... \n" \ % (eph.unix_to_datetime(times[0]), eph.transit_RA(times[0])) if transposed is True: v = f['vis'][frq[0]:frq[-1] + 1, :] v = v[..., t_range[0]:t_range[-1]] vis = v['r'] + 1j * v['i'] del v # Read in time and freq slice if data has not yet been transposed if transposed is False: v = f['vis'][t_range[0]:t_range[-1], frq[0]:frq[-1] + 1, :] vis = v['r'][:] + 1j * v['i'][:] del v vis = np.transpose(vis, (1, 2, 0)) inp = gen_inp()[0] # Remove offset from galaxy if subtract_avg is True: vis -= 0.5 * (vis[..., 0] + vis[..., -1])[..., np.newaxis] freq_MHZ = 800.0 - np.array(frq) / 1024.0 * 400. print len(inp) baddies = np.where(np.isnan(tools.get_feed_positions(inp)[:, 0]))[0] # Fringestop to location of "src" data_fs = tools.fringestop_pathfinder(vis, eph.transit_RA(times), freq_MHZ, inp, src) # data_fs = fringestop_pathfinder(vis, eph.transit_RA(times), freq_MHZ, inp, src) return data_fs
def solve_ps_transit(filename, corrs, feeds, inp, src, nfreq=1024, transposed=False, nfeed=128): """ Function that fringestops time slice where point source is in the beam, takes all correlations for a given polarization, and then eigendecomposes the correlation matrix freq by freq after removing the fpga phases. It will also plot intermediate steps to verify the phase solution. Parameters ---------- filename : np.str Full-path filename corrs : list List of correlations to use in solver feeds : list List of feeds to use inp : Correlator inputs (output of ch_util.tools.get_correlator_inputs) src : ephem.FixedBody Source to calibrate off of. e.g. ch_util.ephemeris.TauA Returns ------- Gains : np.array Complex gain array (nfreq, nfeed) """ nsplit = 32 # Number of freq chunks to divide nfreq into del_t = 800 f = h5py.File(filename, 'r') # Add half an integration time to each. Hack. times = f['index_map']['time'].value['ctime'] + 10.50 src_trans = eph.transit_times(src, times[0]) # try to account for differential arrival time from # cylinder rotation. del_phi = (src._dec - np.radians(eph.CHIMELATITUDE)) \ * np.sin(np.radians(1.988)) del_phi *= (24 * 3600.0) / (2 * np.pi) # Adjust the transit time accordingly src_trans += del_phi # Select +- del_t of transit, accounting for the mispointing t_range = np.where((times < src_trans + del_t) & (times > src_trans - del_t))[0] print "\n...... This data is from %s starting at RA: %f ...... \n" \ % (eph.unix_to_datetime(times[0]), eph.transit_RA(times[0])) assert (len(t_range) > 0), "Source is not in this acq" # Create gains array to fill in solution Gains = np.zeros([nfreq, nfeed], np.complex128) print "Starting the solver" times = times[t_range[0]:t_range[-1]] k = 0 # Start at a strong freq channel that can be plotted # and from which we can find the noise source on-sample for i in range(12, nsplit) + range(0, 12): k += 1 # Divides the arrays up into nfreq / nsplit freq chunks and solves those frq = range(i * nfreq // nsplit, (i + 1) * nfreq // nsplit) print " %d:%d \n" % (frq[0], frq[-1]) # Read in time and freq slice if data has already been transposed if transposed is True: v = f['vis'][frq[0]:frq[-1] + 1, corrs, :] v = v[..., t_range[0]:t_range[-1]] vis = v['r'] + 1j * v['i'] if k == 1: autos = auto_corrs(nfeed) offp = (abs(vis[:, autos, 0::2]).mean() > \ (abs(vis[:, autos, 1::2]).mean())).astype(int) times = times[offp::2] vis = vis[..., offp::2] gg = f['gain_coeff'][frq[0]:frq[-1] + 1, feeds, t_range[0]:t_range[-1]][..., offp::2] gain_coeff = gg['r'] + 1j * gg['i'] del gg # Read in time and freq slice if data has not yet been transposed if transposed is False: print "TRANSPOSED V OF CODE DOESN'T WORK YET!" v = f['vis'][t_range[0]:t_range[-1]:2, frq[0]:frq[-1] + 1, corrs] vis = v['r'][:] + 1j * v['i'][:] del v gg = f['gain_coeff'][0, frq[0]:frq[-1] + 1, feeds] gain_coeff = gg['r'][:] + 1j * gg['i'][:] vis = vis[..., offp::2] vis = np.transpose(vis, (1, 2, 0)) # Remove fpga gains from data vis = remove_fpga_gains(vis, gain_coeff, nfeed=nfeed, triu=False) # Remove offset from galaxy vis -= 0.5 * (vis[..., 0] + vis[..., -1])[..., np.newaxis] # Get physical freq for fringestopper freq_MHZ = 800.0 - np.array(frq) / 1024.0 * 400. baddies = np.where(np.isnan(tools.get_feed_positions(inp)[:, 0]))[0] a, b, c = select_corrs(baddies, nfeed=128) vis[:, a + b] = 0.0 # Fringestop to location of "src" data_fs = tools.fringestop_pathfinder(vis, eph.transit_RA(times), freq_MHZ, inp, src) del vis dr, sol_arr = solve_gain(data_fs) # Find index of point source transit drlist = np.argmax(dr, axis=-1) # If multiple freq channels are zerod, the trans_pix # will end up being 0. This is bad, so ensure that # you are only looking for non-zero transit pixels. drlist = [x for x in drlist if x != 0] trans_pix = np.argmax(np.bincount(drlist)) assert trans_pix != 0.0 Gains[frq] = sol_arr[..., trans_pix - 3:trans_pix + 4].mean(-1) zz = h5py.File('data' + str(i) + '.hdf5', 'w') zz.create_dataset('data', data=dr) zz.close() print "%f, %d Nans out of %d" % (np.isnan(sol_arr).sum(), np.isnan(Gains[frq]).sum(), np.isnan(Gains[frq]).sum()) print trans_pix, sol_arr[..., trans_pix - 3:trans_pix + 4].mean(-1).sum(), sol_arr.mean(-1).sum() # Plot up post-fs phases to see if everything has been fixed if frq[0] == 12 * nsplit: print "======================" print " Plotting up freq: %d" % frq[0] print "======================" img_nm = './phs_plots/dfs' + np.str(frq[17]) + np.str( np.int(time.time())) + '.png' img_nmcorr = './phs_plots/dfs' + np.str(frq[17]) + np.str( np.int(time.time())) + 'corr.png' plt_gains(data_fs, 0, img_name=img_nm, bad_chans=baddies) dfs_corr = correct_dfs(data_fs, np.angle(Gains[frq])[..., np.newaxis], nfeed=128) plt_gains(dfs_corr, 0, img_name=img_nmcorr, bad_chans=baddies) del dfs_corr del data_fs, a return Gains
def main(config_file=None, logging_params=DEFAULT_LOGGING): # Setup logging log.setup_logging(logging_params) mlog = log.get_logger(__name__) # Set config config = DEFAULTS.deepcopy() if config_file is not None: config.merge(NameSpace(load_yaml_config(config_file))) # Set niceness current_niceness = os.nice(0) os.nice(config.niceness - current_niceness) mlog.info('Changing process niceness from %d to %d. Confirm: %d' % (current_niceness, config.niceness, os.nice(0))) # Create output suffix output_suffix = config.output_suffix if config.output_suffix is not None else "jumps" # Calculate the wavelet transform for the following scales nwin = 2 * config.max_scale + 1 nhwin = nwin // 2 if config.log_scale: mlog.info("Using log scale.") scale = np.logspace(np.log10(config.min_scale), np.log10(nwin), num=config.num_points, dtype=np.int) else: mlog.info("Using linear scale.") scale = np.arange(config.min_scale, nwin, dtype=np.int) # Loop over acquisitions for acq in config.acq: # Find acquisition files all_data_files = sorted(glob(os.path.join(config.data_dir, acq, "*.h5"))) nfiles = len(all_data_files) if nfiles == 0: continue mlog.info("Now processing acquisition %s (%d files)" % (acq, nfiles)) # Determine list of feeds to examine dset = ['flags/inputs'] if config.use_input_flag else () rdr = andata.CorrData.from_acq_h5(all_data_files, datasets=dset, apply_gain=False, renormalize=False) inputmap = tools.get_correlator_inputs(ephemeris.unix_to_datetime(rdr.time[0]), correlator='chime') # Extract good inputs if config.use_input_flag: ifeed = np.flatnonzero((np.sum(rdr.flags['inputs'][:], axis=-1, dtype=np.int) / float(rdr.flags['inputs'].shape[-1])) > config.input_threshold) else: ifeed = np.array([ii for ii, inp in enumerate(inputmap) if tools.is_chime(inp)]) ninp = len(ifeed) mlog.info("Processing %d feeds." % ninp) # Create list of candidates cfreq, cinput, ctime, cindex = [], [], [], [] jump_flag, jump_time, jump_auto = [], [], [] ncandidate = 0 # Determine number of files to process at once if config.max_num_file is None: chunk_size = nfiles else: chunk_size = min(config.max_num_file, nfiles) # Loop over chunks of files for chnk, data_files in enumerate(chunks(all_data_files, chunk_size)): mlog.info("Now processing chunk %d (%d files)" % (chnk, len(data_files))) # Deteremine selections along the various axes rdr = andata.CorrData.from_acq_h5(data_files, datasets=()) auto_sel = np.array([ii for ii, pp in enumerate(rdr.prod) if pp[0] == pp[1]]) auto_sel = andata._convert_to_slice(auto_sel) if config.time_start is None: ind_start = 0 else: time_start = ephemeris.datetime_to_unix(datetime.datetime(*config.time_start)) ind_start = int(np.argmin(np.abs(rdr.time - time_start))) if config.time_stop is None: ind_stop = rdr.ntime else: time_stop = ephemeris.datetime_to_unix(datetime.datetime(*config.time_stop)) ind_stop = int(np.argmin(np.abs(rdr.time - time_stop))) if config.freq_physical is not None: if hasattr(config.freq_physical, '__iter__'): freq_physical = config.freq_physical else: freq_physical = [config.freq_physical] freq_sel = [np.argmin(np.abs(ff - rdr.freq)) for ff in freq_physical] freq_sel = andata._convert_to_slice(freq_sel) else: fstart = config.freq_start if config.freq_start is not None else 0 fstop = config.freq_stop if config.freq_stop is not None else rdr.freq.size freq_sel = slice(fstart, fstop) # Load autocorrelations t0 = time.time() data = andata.CorrData.from_acq_h5(data_files, datasets=['vis'], start=ind_start, stop=ind_stop, freq_sel=freq_sel, prod_sel=auto_sel, apply_gain=False, renormalize=False) mlog.info("Took %0.1f seconds to load autocorrelations." % (time.time() - t0,)) # If first chunk, save the frequencies that are being used if not chnk: all_freq = data.freq.copy() # If requested do not consider data during day or near bright source transits flag_quiet = np.ones(data.ntime, dtype=np.bool) if config.ignore_sun: flag_quiet &= ~transit_flag('sun', data.time, freq=np.min(data.freq), pol='X', nsig=1.0) if config.only_quiet: flag_quiet &= ~daytime_flag(data.time) for ss in ["CYG_A", "CAS_A", "TAU_A", "VIR_A"]: flag_quiet &= ~transit_flag(ss, data.time, freq=np.min(data.freq), pol='X', nsig=1.0) # Loop over frequencies for ff, freq in enumerate(data.freq): print_cnt = 0 mlog.info("FREQ %d (%0.2f MHz)" % (ff, freq)) auto = data.vis[ff, :, :].real fractional_auto = auto * tools.invert_no_zero(np.median(auto, axis=-1, keepdims=True)) - 1.0 # Loop over inputs for ii in ifeed: print_cnt += 1 do_print = not (print_cnt % 100) if do_print: mlog.info("INPUT %d" % ii) t0 = time.time() signal = fractional_auto[ii, :] # Perform wavelet transform coef, freqs = pywt.cwt(signal, scale, config.wavelet_name) if do_print: mlog.info("Took %0.1f seconds to perform wavelet transform." % (time.time() - t0,)) t0 = time.time() # Find local modulus maxima flg_mod_max, mod_max = mod_max_finder(scale, coef, threshold=config.thresh, search_span=config.search_span) if do_print: mlog.info("Took %0.1f seconds to find modulus maxima." % (time.time() - t0,)) t0 = time.time() # Find persisent modulus maxima across scales candidates, cmm, pdrift, start, stop, lbl = finger_finder(scale, flg_mod_max, mod_max, istart=max(config.min_rise - config.min_scale, 0), do_fill=False) if do_print: mlog.info("Took %0.1f seconds to find fingers." % (time.time() - t0,)) t0 = time.time() if candidates is None: continue # Cut bad candidates index_good_candidates = np.flatnonzero((scale[stop] >= config.max_scale) & flag_quiet[candidates[start, np.arange(start.size)]] & (pdrift <= config.psigma_max)) ngood = index_good_candidates.size if ngood == 0: continue mlog.info("Input %d has %d jumps" % (ii, ngood)) # Add remaining candidates to list ncandidate += ngood cfreq += [freq] * ngood cinput += [ii] * ngood for igc in index_good_candidates: icenter = candidates[start[igc], igc] cindex.append(icenter) ctime.append(data.time[icenter]) aa = max(0, icenter - nhwin) bb = min(data.ntime, icenter + nhwin + 1) ncut = bb - aa temp_var = np.zeros(nwin, dtype=np.bool) temp_var[0:ncut] = True jump_flag.append(temp_var) temp_var = np.zeros(nwin, dtype=data.time.dtype) temp_var[0:ncut] = data.time[aa:bb].copy() jump_time.append(temp_var) temp_var = np.zeros(nwin, dtype=auto.dtype) temp_var[0:ncut] = auto[ii, aa:bb].copy() jump_auto.append(temp_var) # Garbage collect del data gc.collect() # If we found any jumps, write them to a file. if ncandidate > 0: output_file = os.path.join(config.output_dir, "%s_%s.h5" % (acq, output_suffix)) mlog.info("Writing %d jumps to: %s" % (ncandidate, output_file)) # Write to output file with h5py.File(output_file, 'w') as handler: handler.attrs['files'] = all_data_files handler.attrs['chan_id'] = ifeed handler.attrs['freq'] = all_freq index_map = handler.create_group('index_map') index_map.create_dataset('jump', data=np.arange(ncandidate)) index_map.create_dataset('window', data=np.arange(nwin)) ax = np.array(['jump']) dset = handler.create_dataset('freq', data=np.array(cfreq)) dset.attrs['axis'] = ax dset = handler.create_dataset('input', data=np.array(cinput)) dset.attrs['axis'] = ax dset = handler.create_dataset('time', data=np.array(ctime)) dset.attrs['axis'] = ax dset = handler.create_dataset('time_index', data=np.array(cindex)) dset.attrs['axis'] = ax ax = np.array(['jump', 'window']) dset = handler.create_dataset('jump_flag', data=np.array(jump_flag)) dset.attrs['axis'] = ax dset = handler.create_dataset('jump_time', data=np.array(jump_time)) dset.attrs['axis'] = ax dset = handler.create_dataset('jump_auto', data=np.array(jump_auto)) dset.attrs['axis'] = ax else: mlog.info("No jumps found for %s acquisition." % acq)
def process(self, sstream, inputmap, inputmask): """Determine calibration from a timestream. Parameters ---------- sstream : andata.CorrData or containers.SiderealStream Timestream collected during the day. inputmap : list of :class:`CorrInput` A list describing the inputs as they are in the file. inputmask : containers.CorrInputMask Mask indicating which correlator inputs to use in the eigenvalue decomposition. Returns ------- suntrans : containers.SunTransit Response to the sun. """ from operator import itemgetter from itertools import groupby from .calibration import _extract_diagonal, solve_gain # Ensure that we are distributed over frequency sstream.redistribute("freq") # Find the local frequencies nfreq = sstream.vis.local_shape[0] sfreq = sstream.vis.local_offset[0] efreq = sfreq + nfreq # Get the local frequency axis freq = sstream.freq["centre"][sfreq:efreq] wv = 3e2 / freq # Get times if hasattr(sstream, "time"): time = sstream.time ra = ephemeris.transit_RA(time) else: ra = sstream.index_map["ra"][:] csd = (sstream.attrs["lsd"] if "lsd" in sstream.attrs else sstream.attrs["csd"]) csd = csd + ra / 360.0 time = ephemeris.csd_to_unix(csd) # Only examine data between sunrise and sunset time_flag = np.zeros(len(time), dtype=np.bool) rise = ephemeris.solar_rising(time[0] - 24.0 * 3600.0, end_time=time[-1]) for rr in rise: ss = ephemeris.solar_setting(rr)[0] time_flag |= (time >= rr) & (time <= ss) if not np.any(time_flag): self.log.debug( "No daytime data between %s and %s.", ephemeris.unix_to_datetime(time[0]).strftime("%b %d %H:%M"), ephemeris.unix_to_datetime(time[-1]).strftime("%b %d %H:%M"), ) return None # Convert boolean flag to slices time_index = np.where(time_flag)[0] time_slice = [] ntime = 0 for key, group in groupby( enumerate(time_index), lambda index_item: index_item[0] - index_item[1]): group = list(map(itemgetter(1), group)) ngroup = len(group) time_slice.append( (slice(group[0], group[-1] + 1), slice(ntime, ntime + ngroup))) ntime += ngroup time = np.concatenate([time[slc[0]] for slc in time_slice]) ra = np.concatenate([ra[slc[0]] for slc in time_slice]) # Get ra, dec, alt of sun sun_pos = np.array([ ra_dec_of(ephemeris.skyfield_wrapper.ephemeris["sun"], t) for t in time ]) # Convert from ra to hour angle sun_pos[:, 0] = np.radians(ra) - sun_pos[:, 0] # Determine good inputs nfeed = len(inputmap) good_input = np.arange( nfeed, dtype=np.int)[inputmask.datasets["input_mask"][:]] # Use input map to figure out which are the X and Y feeds xfeeds = np.array([ idx for idx, inp in enumerate(inputmap) if tools.is_chime_x(inp) and (idx in good_input) ]) yfeeds = np.array([ idx for idx, inp in enumerate(inputmap) if tools.is_chime_y(inp) and (idx in good_input) ]) self.log.debug( "Performing sun calibration with %d/%d good feeds (%d xpol, %d ypol).", len(good_input), nfeed, len(xfeeds), len(yfeeds), ) # Construct baseline vector for each visibility feed_pos = tools.get_feed_positions(inputmap) vis_pos = np.array([ feed_pos[ii] - feed_pos[ij] for ii, ij in sstream.index_map["prod"][:] ]) vis_pos = np.where(np.isnan(vis_pos), np.zeros_like(vis_pos), vis_pos) u = (vis_pos[np.newaxis, :, 0] / wv[:, np.newaxis])[:, :, np.newaxis] v = (vis_pos[np.newaxis, :, 1] / wv[:, np.newaxis])[:, :, np.newaxis] # Create container to hold results of fit suntrans = containers.SunTransit(time=time, pol_x=xfeeds, pol_y=yfeeds, axes_from=sstream) for key in suntrans.datasets.keys(): suntrans.datasets[key][:] = 0.0 # Set coordinates suntrans.coord[:] = sun_pos # Loop over time slices for slc_in, slc_out in time_slice: # Extract visibility slice vis_slice = sstream.vis[..., slc_in].copy() ha = (sun_pos[slc_out, 0])[np.newaxis, np.newaxis, :] dec = (sun_pos[slc_out, 1])[np.newaxis, np.newaxis, :] # Extract the diagonal (to be used for weighting) norm = (_extract_diagonal(vis_slice, axis=1).real)**0.5 norm = tools.invert_no_zero(norm) # Fringestop if self.fringestop: vis_slice *= tools.fringestop_phase( ha, np.radians(ephemeris.CHIMELATITUDE), dec, u, v) # Solve for the point source response of each set of polarisations ev_x, resp_x, err_resp_x = solve_gain(vis_slice, feeds=xfeeds, norm=norm[:, xfeeds]) ev_y, resp_y, err_resp_y = solve_gain(vis_slice, feeds=yfeeds, norm=norm[:, yfeeds]) # Save to container suntrans.evalue_x[..., slc_out] = ev_x suntrans.evalue_y[..., slc_out] = ev_y suntrans.response[:, xfeeds, slc_out] = resp_x suntrans.response[:, yfeeds, slc_out] = resp_y suntrans.response_error[:, xfeeds, slc_out] = err_resp_x suntrans.response_error[:, yfeeds, slc_out] = err_resp_y # If requested, fit a model to the primary beam of the sun transit if self.model_fit: # Estimate peak RA i_transit = np.argmin(np.abs(sun_pos[:, 0])) body = ephemeris.skyfield_wrapper.ephemeris["sun"] obs = ephemeris._get_chime() obs.date = ephemeris.unix_to_ephem_time(time[i_transit]) body.compute(obs) peak_ra = ephemeris.peak_RA(body) dra = ra - peak_ra dra = np.abs(dra - (dra > np.pi) * 2.0 * np.pi)[np.newaxis, np.newaxis, :] # Estimate FWHM sig_x = cal_utils.guess_fwhm(freq, pol="X", dec=body.dec, sigma=True)[:, np.newaxis, np.newaxis] sig_y = cal_utils.guess_fwhm(freq, pol="Y", dec=body.dec, sigma=True)[:, np.newaxis, np.newaxis] # Only fit ra values above the specified dynamic range threshold fit_flag = np.zeros([nfreq, nfeed, ntime], dtype=np.bool) fit_flag[:, xfeeds, :] = dra < (self.nsig * sig_x) fit_flag[:, yfeeds, :] = dra < (self.nsig * sig_y) # Fit model for the complex response of each feed to the point source param, param_cov = cal_utils.fit_point_source_transit( ra, suntrans.response[:], suntrans.response_error[:], flag=fit_flag) # Save to container suntrans.add_dataset("flag") suntrans.flag[:] = fit_flag suntrans.add_dataset("parameter") suntrans.parameter[:] = param suntrans.add_dataset("parameter_cov") suntrans.parameter_cov[:] = param_cov # Update attributes units = "sqrt(" + sstream.vis.attrs.get("units", "correlator-units") + ")" suntrans.response.attrs["units"] = units suntrans.response_error.attrs["units"] = units suntrans.attrs["source"] = "Sun" # Return sun transit return suntrans
def create_from_dict( cls, dict, notes=None, start_tol=60.0, dryrun=True, replace_dup=False, verbose=False, ): """ Create a holography database entry from a dictionary This routine checks for duplicates and overwrites duplicates if and only if `replace_dup = True` Parameters ---------- dict : dict src : :py:class:`HolographySource` A HolographySource object for the source start_time Start time as a Skyfield Time object finish_time Finish time as a Skyfield Time object """ DATE_FMT_STR = "%Y-%m-%d %H:%M:%S %Z" def check_for_duplicates(t, src, start_tol, ignore_src_mismatch=False): """ Check for duplicate holography observations, comparing the given observation to the existing database Inputs ------ t: Skyfield Time object beginning time of observation src: HolographySource target source start_tol: float Tolerance in seconds within which to search for duplicates ignore_src_mismatch: bool (default: False) If True, consider observations a match if the time matches but the source does not Outputs ------- If a duplicate is found: :py:class:`HolographyObservation` object for the existing entry in the database If no duplicate is found: None """ ts = ephemeris.skyfield_wrapper.timescale unixt = ephemeris.ensure_unix(t) dup_found = False existing_db_entry = cls.select().where( cls.start_time.between(unixt - start_tol, unixt + start_tol)) if len(existing_db_entry) > 0: if len(existing_db_entry) > 1: print("Multiple entries found.") for entry in existing_db_entry: tt = ts.utc(ephemeris.unix_to_datetime(entry.start_time)) # LST = GST + east longitude ttlst = np.mod(tt.gmst + DRAO_lon, 24.0) # Check if source name matches. If not, print a warning # but proceed anyway. if src.name.upper() == entry.source.name.upper(): dup_found = True if verbose: print("Observation is already in database.") else: if ignore_src_mismatch: dup_found = True print( "** Observation at same time but with different " + "sources in database: ", src.name, entry.source.name, tt.utc_datetime().isoformat(), ) # if the observations match in start time and source, # call them the same observation. Not the most strict # check possible. if dup_found: tf = ts.utc( ephemeris.unix_to_datetime(entry.finish_time)) print("Tried to add : {} {}; LST={:.3f}".format( src.name, t.utc_datetime().strftime(DATE_FMT_STR), ttlst)) print("Existing entry: {} {}; LST={:.3f}".format( entry.source.name, tt.utc_datetime().strftime(DATE_FMT_STR), ttlst, )) if dup_found: return existing_db_entry else: return None # DRAO longitude in hours DRAO_lon = ephemeris.chime.longitude * 24.0 / 360.0 if verbose: print(" ") addtodb = True dup_entries = check_for_duplicates(dict["start_time"], dict["src"], start_tol) if dup_entries is not None: if replace_dup: if not dryrun: for entry in dup_entries: cls.delete_instance(entry) if verbose: print( "Deleted observation from database and replacing." ) elif verbose: print( "Would have deleted observation and replaced (dry run)." ) addtodb = True else: addtodb = False for entry in dup_entries: print("Not replacing duplicate {} observation {}".format( entry.source.name, ephemeris.unix_to_datetime( entry.start_time).strftime(DATE_FMT_STR), )) # we've appended this observation to obslist. # Now add to the database, if we're supposed to. if addtodb: string = "Adding to database: {} {} to {}" print( string.format( dict["src"].name, dict["start_time"].utc_datetime().strftime(DATE_FMT_STR), dict["finish_time"].utc_datetime().strftime(DATE_FMT_STR), )) if dryrun: print("Dry run; doing nothing") else: cls.create( source=dict["src"], start_time=ephemeris.ensure_unix(dict["start_time"]), finish_time=ephemeris.ensure_unix(dict["finish_time"]), quality_flag=dict["quality_flag"], notes=notes, )
def main(config_file=None, logging_params=DEFAULT_LOGGING): # Setup logging log.setup_logging(logging_params) mlog = log.get_logger(__name__) # Set config config = DEFAULTS.deepcopy() if config_file is not None: config.merge(NameSpace(load_yaml_config(config_file))) # Set niceness current_niceness = os.nice(0) os.nice(config.niceness - current_niceness) mlog.info('Changing process niceness from %d to %d. Confirm: %d' % (current_niceness, config.niceness, os.nice(0))) # Find acquisition files acq_files = sorted(glob(os.path.join(config.data_dir, config.acq, "*.h5"))) nfiles = len(acq_files) # Determine time range of each file findex = [] tindex = [] for ii, filename in enumerate(acq_files): subdata = andata.CorrData.from_acq_h5(filename, datasets=()) findex += [ii] * subdata.ntime tindex += range(subdata.ntime) findex = np.array(findex) tindex = np.array(tindex) # Determine transits within these files transits = [] data = andata.CorrData.from_acq_h5(acq_files, datasets=()) solar_rise = ephemeris.solar_rising(data.time[0] - 24.0 * 3600.0, end_time=data.time[-1]) for rr in solar_rise: ss = ephemeris.solar_setting(rr)[0] solar_flag = np.flatnonzero((data.time >= rr) & (data.time <= ss)) if solar_flag.size > 0: solar_flag = solar_flag[::config.downsample] tval = data.time[solar_flag] this_findex = findex[solar_flag] this_tindex = tindex[solar_flag] file_list, tindices = [], [] for ii in range(nfiles): this_file = np.flatnonzero(this_findex == ii) if this_file.size > 0: file_list.append(acq_files[ii]) tindices.append(this_tindex[this_file]) date = ephemeris.unix_to_datetime(rr).strftime('%Y%m%dT%H%M%SZ') transits.append((date, tval, file_list, tindices)) # Specify some parameters for algorithm N = 2048 noffset = len(config.offsets) if config.sep_pol: rank = 1 cross_pol = False pol = np.array(['S', 'E']) pol_s = np.array( [rr + 256 * xx for xx in range(0, 8, 2) for rr in range(256)]) pol_e = np.array( [rr + 256 * xx for xx in range(1, 8, 2) for rr in range(256)]) prod_ss = [] prod_ee = [] else: rank = 8 cross_pol = config.cross_pol pol = np.array(['all']) npol = pol.size # Create file prefix and suffix prefix = [] prefix.append("gain_solutions") if config.output_prefix is not None: prefix.append(config.output_prefix) prefix = '_'.join(prefix) suffix = [] suffix.append("pol_%s" % '_'.join(pol)) suffix.append("niter_%d" % config.niter) if cross_pol: suffix.append("zerocross") else: suffix.append("keepcross") if config.normalize: suffix.append("normed") else: suffix.append("notnormed") suffix = '_'.join(suffix) # Loop over solar transits for date, timestamps, files, time_indices in transits: nfiles = len(files) mlog.info("%s (%d files) " % (date, nfiles)) output_file = os.path.join( config.output_dir, "%s_SUN_%s_%s.pickle" % (prefix, date, suffix)) mlog.info("Saving to: %s" % output_file) # Get info about this set of files data = andata.CorrData.from_acq_h5(files, datasets=['flags/inputs']) prod = data.prod coord = sun_coord(timestamps, deg=True) fstart = config.freq_start if config.freq_start is not None else 0 fstop = config.freq_stop if config.freq_stop is not None else data.freq.size freq_index = range(fstart, fstop) freq = data.freq[freq_index] ntime = timestamps.size nfreq = freq.size # Determind bad inputs if config.bad_input_file is None or not os.path.isfile( config.bad_input_file): bad_input = np.flatnonzero( ~np.all(data.flags['inputs'][:], axis=-1)) else: with open(config.bad_input_file, 'r') as handler: bad_input = pickle.load(handler) mlog.info("%d inputs flagged as bad." % bad_input.size) bad_prod = np.array([ ii for ii, pp in enumerate(prod) if (pp[0] in bad_input) or (pp[1] in bad_input) ]) # Create arrays to hold the results ores = {} ores['date'] = date ores['coord'] = coord ores['time'] = timestamps ores['freq'] = freq ores['offsets'] = config.offsets ores['pol'] = pol ores['evalue'] = np.zeros((noffset, nfreq, ntime, N), dtype=np.float32) ores['resp'] = np.zeros((noffset, nfreq, ntime, N, config.neigen), dtype=np.complex64) ores['resp_err'] = np.zeros((noffset, nfreq, ntime, N, config.neigen), dtype=np.float32) # Loop over frequencies for ff, find in enumerate(freq_index): mlog.info("Freq %d of %d. %0.2f MHz." % (ff + 1, nfreq, freq[ff])) cnt = 0 # Loop over files for ii, (filename, tind) in enumerate(zip(files, time_indices)): ntind = len(tind) mlog.info("Processing file %s (%d time samples)" % (filename, ntind)) # Loop over times for tt in tind: t0 = time.time() mlog.info("Time %d of %d. %d index of current file." % (cnt + 1, ntime, tt)) # Load visibilities with h5py.File(filename, 'r') as hf: vis = hf['vis'][find, :, tt] # Set bad products equal to zero vis[bad_prod] = 0.0 # Different code if we are separating polarisations if config.sep_pol: if not any(prod_ss): for pind, pp in enumerate(prod): if (pp[0] in pol_s) and (pp[1] in pol_s): prod_ss.append(pind) elif (pp[0] in pol_e) and (pp[1] in pol_e): prod_ee.append(pind) prod_ss = np.array(prod_ss) prod_ee = np.array(prod_ee) mlog.info("Product sizes: %d, %d" % (prod_ss.size, prod_ee.size)) # Loop over polarisations for pp, (input_pol, prod_pol) in enumerate([(pol_s, prod_ss), (pol_e, prod_ee)]): visp = vis[prod_pol] mlog.info("pol %s, visibility size: %d" % (pol[pp], visp.size)) # Loop over offsets for oo, off in enumerate(config.offsets): mlog.info( "pol %s, rank %d, niter %d, offset %d, cross_pol %s, neigen %d" % (pol[pp], rank, config.niter, off, cross_pol, config.neigen)) ev, rr, rre = solve_gain( visp, cutoff=off, cross_pol=cross_pol, normalize=config.normalize, rank=rank, niter=config.niter, neigen=config.neigen) ores['evalue'][oo, ff, cnt, input_pol] = ev ores['resp'][oo, ff, cnt, input_pol, :] = rr ores['resp_err'][oo, ff, cnt, input_pol, :] = rre else: # Loop over offsets for oo, off in enumerate(config.offsets): mlog.info( "rank %d, niter %d, offset %d, cross_pol %s, neigen %d" % (rank, config.niter, off, cross_pol, config.neigen)) ev, rr, rre = solve_gain( vis, cutoff=off, cross_pol=cross_pol, normalize=config.normalize, rank=rank, niter=config.niter, neigen=config.neigen) ores['evalue'][oo, ff, cnt, :] = ev ores['resp'][oo, ff, cnt, :, :] = rr ores['resp_err'][oo, ff, cnt, :, :] = rre # Increment time counter cnt += 1 # Print time elapsed mlog.info("Took %0.1f seconds." % (time.time() - t0, )) # Save to pickle file with open(output_file, 'w') as handle: pickle.dump(ores, handle)
def _create_plot(visi, tmstp, cut_tmstp, sky, popt, test_chans, good_gains, good_noise, good_fit): """Creates plot of the visibilities and the fits with labels for those that fail the tests """ import matplotlib matplotlib.use("PDF") import matplotlib.pyplot as plt import time # Visibilities to plot: visi1 = visi # Raw data tmstp1 = tmstp # Raw data visi2 = np.array([[ sky.fit_func(tt, popt[ii][0], popt[ii][1]) for tt in range(len(cut_tmstp)) ] for ii in range(len(popt))]) tmstp2 = cut_tmstp # For title, use start time stamp: title = "Good channels result for {0}".format( ch_eph.unix_to_datetime(tmstp1[0]).date()) # I need to know the slot for each channel: def get_slot(channel): slot_array = [4, 2, 16, 14, 3, 1, 15, 13, 8, 6, 12, 10, 7, 5, 11, 9] return slot_array[int(channel) // 16] fig = plt.figure(figsize=(8, 64)) fig.suptitle(title, fontsize=16) if (tmstp1[-1] - tmstp1[0]) / (24.0 * 3600.0) > 3.0: # Days since starting time # Notice: same starting time for both time_pl1 = (tmstp1 - tmstp1[0]) / (3600 * 24) time_pl2 = (tmstp2 - tmstp1[0]) / (3600 * 24) time_unit = "days" else: # Hours since starting time time_pl1 = (tmstp1 - tmstp1[0]) / (3600) time_pl2 = (tmstp2 - tmstp1[0]) / (3600) time_unit = "hours" for ii in range(len(visi1)): chan = test_chans[ii] # Determine position in subplot: if chan < 64: pos = chan * 4 + 1 elif chan < 128: pos = (chan - 64) * 4 + 2 elif chan < 192: pos = (chan - 128) * 4 + 3 elif chan < 256: pos = (chan - 192) * 4 + 4 # Create subplot: plt.subplot(64, 4, pos) lab = "" # Or print standard label: if good_gains is not None: if not good_gains[ii]: lab = lab + "bad gains | " if good_noise is not None: if not good_noise[ii]: lab = lab + "noisy | " if not good_fit[ii]: lab = lab + "bad fit" if lab != "": plt.plot([], [], "1.0", label=lab) plt.legend(loc="best", prop={"size": 6}) trace_pl1 = visi1[ii, :].real plt.plot(time_pl1, trace_pl1, "b-") trace_pl2 = visi2[ii, :].real plt.plot(time_pl2, trace_pl2, "r-") tm_brd = (time_pl1[-1] - time_pl1[0]) / 10.0 plt.xlim(time_pl1[0] - tm_brd, time_pl1[-1] + tm_brd) # Determine limits of plots: med = np.median(trace_pl1) mad = np.median([abs(entry - med) for entry in trace_pl1]) plt.ylim(med - 7.0 * mad, med + 7.0 * mad) # labels: plt.ylabel("Ch{0} (Sl.{1})".format(chan, get_slot(chan)), fontsize=8) # Hide numbering: frame = plt.gca() frame.axes.get_yaxis().set_ticks([]) if (chan != 63) and (chan != 127) and (chan != 191) and (chan != 255): # Remove x-axis, except on bottom plots: frame.axes.get_xaxis().set_ticks([]) else: # Change size of numbers in x axis: frame.tick_params(axis="both", which="major", labelsize=10) frame.tick_params(axis="both", which="minor", labelsize=8) if chan == 127: # Put x-labels on bottom plots: if time_unit == "days": plt.xlabel( "Time (days since {0} UTC)".format( ch_eph.unix_to_datetime(tmstp1[0])), fontsize=10, ) else: plt.xlabel( "Time (hours since {0} UTC)".format( ch_eph.unix_to_datetime(tmstp1[0])), fontsize=10, ) if chan == 0: plt.title("West cyl. P1(N-S)", fontsize=12) elif chan == 64: plt.title("West cyl. P2(E-W)", fontsize=12) elif chan == 128: plt.title("East cyl. P1(N-S)", fontsize=12) elif chan == 192: plt.title("East cyl. P2(E-W)", fontsize=12) filename = "plot_fit_{0}.pdf".format(int(time.time())) plt.savefig(filename) plt.close() print("Finished creating plot. File name: {0}".format(filename))