def _cut_non_chime(data, visi, chan_array, inputs=None): """ Remove non CHIME channels (noise injection, RFI antenna, 26m, etc...) from visibility. Also remove channels marked as powered-off in layout DB. """ # Map of channels to corr. inputs: input_map = data.input tmstp = data.index_map["time"]["ctime"] # time stamp # Datetime halfway through data: half_time = ch_eph.unix_to_datetime(tmstp[int(len(tmstp) // 2)]) # Get information on correlator inputs, if not already supplied if inputs is None: inputs = tools.get_correlator_inputs(half_time) # Reorder inputs to have sema order as input map (and data) inputs = tools.reorder_correlator_inputs(input_map, inputs) # Get noise source channel index: # Test if inputs are attached to CHIME antenna and powered on: pwds = tools.is_chime_on(inputs) for ii in range(len(inputs)): # if ( (not tools.is_chime(inputs[ii])) if (not pwds[ii]) and (ii in chan_array): # Remove non-CHIME-on channels from visibility matrix... idx = np.where(chan_array == ii)[0][0] # index of channel visi = np.delete(visi, idx, axis=0) # ...and from product array: chan_array = np.delete(chan_array, idx, axis=0) return visi, chan_array
def sat_phase(self,freq,offset=(0,0)): self.layout = array(tools.get_correlator_inputs(self.dt)) output = [] for baseline in self.bl: bl_vector = self.get_bl(*baseline) bdots = dot(self.all_coords(offset), bl_vector) output.append(2*constants.pi*bdots*(freq*10**6)/constants.c) return output
def set_metadata(self, tms, input_map): """Sets self.corr_inputs, self.pwds, self.pstns, self.p1_idx, self.p2_idx""" from ch_util import tools # Get CHIME ON channels: half_time = ephemeris.unix_to_datetime(tms[int(len(tms) // 2)]) corr_inputs = tools.get_correlator_inputs(half_time) self.corr_inputs = tools.reorder_correlator_inputs( input_map, corr_inputs) pwds = tools.is_chime_on( self.corr_inputs) # Which inputs are CHIME ON antennas self.pwds = np.array(pwds, dtype=bool) # Get cylinders and polarizations self.pstns, self.p1_idx, self.p2_idx = self.get_pos_pol( self.corr_inputs, self.pwds)
def gen_inp(nfeed=256): """ Generate input information for feeds Parameters ---------- feeds : list Feeds whose input info is needed nfeeds : int Number of feeds in total Returns ------- corrinput_real : All 256 inputs inpx : Only x feeds inpy : Only y feeds """ # Assumes a standard layout for 128 feeds on each cyl xfeeds = range(nfeed / 4) + range(2 * nfeed / 4, 3 * nfeed / 4) yfeeds = range(nfeed / 4, 2 * nfeed / 4) + range(3 * nfeed / 4, 4 * nfeed / 4) xcorrs = [] ycorrs = [] for ii in range(nfeed / 2): for jj in range(ii, nfeed / 2): xcorrs.append(misc.feed_map(xfeeds[ii], xfeeds[jj], nfeed)) ycorrs.append(misc.feed_map(yfeeds[ii], yfeeds[jj], nfeed)) corrinputs = tools.get_correlator_inputs(\ datetime.datetime(2015, 6, 1, 0, 0, 0), correlator='K7BP16-0004') # Need to rearrange to match order in the correlated data corrinput_real = rearrange_list(corrinputs, nfeeds=256) inpx = [] inpy = [] for i in range(nfeed / 2): inpx.append(corrinput_real[xfeeds[i]]) inpy.append(corrinput_real[yfeeds[i]]) return corrinput_real, inpx, inpy, xcorrs, ycorrs, xfeeds, yfeeds
def gen_inp(nfeed=256): """ Generate input information for feeds Parameters ---------- feeds : list Feeds whose input info is needed nfeeds : int Number of feeds in total Returns ------- corrinput_real : All 256 inputs inpx : Only x feeds inpy : Only y feeds """ # Assumes a standard layout for 128 feeds on each cyl xfeeds = range(nfeed/4) + range(2 * nfeed/4, 3 * nfeed/4) yfeeds = range(nfeed/4, 2 * nfeed/4) + range(3 * nfeed/4, 4 * nfeed/4) xcorrs = [] ycorrs = [] for ii in range(nfeed/2): for jj in range(ii, nfeed/2): xcorrs.append(misc.feed_map(xfeeds[ii], xfeeds[jj], nfeed)) ycorrs.append(misc.feed_map(yfeeds[ii], yfeeds[jj], nfeed)) corrinputs = tools.get_correlator_inputs(\ datetime.datetime(2015, 6, 1, 0, 0, 0), correlator='K7BP16-0004') # Need to rearrange to match order in the correlated data corrinput_real = rearrange_list(corrinputs, nfeeds=256) inpx = [] inpy = [] for i in range(nfeed/2): inpx.append(corrinput_real[xfeeds[i]]) inpy.append(corrinput_real[yfeeds[i]]) return corrinput_real, inpx, inpy, xcorrs, ycorrs, xfeeds, yfeeds
def get_prod_sel(self, data): """ """ from ch_util import tools input_map = data.input tms = data.time half_time = ephemeris.unix_to_datetime(tms[int(len(tms) // 2)]) corr_inputs = tools.get_correlator_inputs(half_time) corr_inputs = tools.reorder_correlator_inputs(input_map, corr_inputs) pwds = tools.is_chime_on( corr_inputs) # Which inputs are CHIME ON antennas wchp1, wchp2, echp1, echp2 = self.get_cyl_pol(corr_inputs, pwds) # Ensure base channels are CHIME and ON while not pwds[np.where(input_map["chan_id"] == self.bswp1)[0][0]]: self.bswp1 += 1 while not pwds[np.where(input_map["chan_id"] == self.bswp2)[0][0]]: self.bswp2 += 1 while not pwds[np.where(input_map["chan_id"] == self.bsep1)[0][0]]: self.bsep1 += 1 while not pwds[np.where(input_map["chan_id"] == self.bsep2)[0][0]]: self.bsep2 += 1 prod_sel = [] for (ii, prod) in enumerate(data.prod): add_prod = False add_prod = add_prod or ( (prod[0] == self.bswp1 and prod[1] in echp1) or (prod[1] == self.bswp1 and prod[0] in echp1)) add_prod = add_prod or ( (prod[0] == self.bswp2 and prod[1] in echp2) or (prod[1] == self.bswp2 and prod[0] in echp2)) add_prod = add_prod or ( (prod[0] == self.bsep1 and prod[1] in wchp1) or (prod[1] == self.bsep1 and prod[0] in wchp1)) add_prod = add_prod or ( (prod[0] == self.bsep2 and prod[1] in wchp2) or (prod[1] == self.bsep2 and prod[0] in wchp2)) if add_prod: prod_sel.append(ii) prod_sel.sort() return prod_sel, pwds
def _load_layout(self): """Load the CHIME/Pathfinder layout from the database. Generally this routine shouldn't be called directly. Use :method:`CHIME.from_layout` or configure from a YAML file. """ if self.layout is None: raise Exception("Layout attributes not set.") # Fetch feed layout from database feeds = tools.get_correlator_inputs(self.layout, self.correlator) if mpiutil.size > 1: feeds = mpiutil.world.bcast(feeds, root=0) if self.skip_non_chime: raise Exception("Not supported.") self._feeds = feeds
def expected_phase(self): output = [] self.layout = array(tools.get_correlator_inputs(self.dt)) if not self.set_up: self.transit_time() self.read_transit_data() if self.bl is not None: for baseline in self.bl: bl_vector = self.get_bl(*baseline) freqs = array([i[0] for i in self.data.freq]) bdots = dot(self.transit_coords(), bl_vector) output.append(2*constants.pi*bdots*(freq*10**6)/constants.c) else: for ii in xrange(256): for jj in xrange(i,256): bl_vector = self.get_bl(ii,jj) freqs = array([i[0] for i in self.data.freq]) bdots = dot(self.transit_coords(), bl_vector) output.append(2*constants.pi*bdots*(freqs*10**6)/constants.c) return output
def next(self, ts): """Generate an input description from the timestream passed in. Parameters ---------- ts : andata.CorrData Timestream container. Returns ------- inputs : list of :class:`CorrInput` A list of describing the inputs as they are in the file. """ # Fetch from the cache if we can if self.cache and self._cached_inputs: self.log.debug("Using cached inputs.") return self._cached_inputs inputs = None if mpiutil.rank0: # Get the datetime of the middle of the file time = ephemeris.unix_to_datetime(0.5 * (ts.time[0] + ts.time[-1])) inputs = tools.get_correlator_inputs(time) inputs = tools.reorder_correlator_inputs(ts.index_map["input"], inputs) # Broadcast input description to all ranks inputs = mpiutil.world.bcast(inputs, root=0) # Save into the cache for the next iteration if self.cache: self._cached_inputs = inputs # Make sure all nodes have container before return mpiutil.world.Barrier() return inputs
def offline_point_source_calibration(file_list, source, inputmap=None, start=None, stop=None, physical_freq=None, tcorr=None, logging_params=DEFAULT_LOGGING, **kwargs): # Load config config = DEFAULTS.deepcopy() config.merge(NameSpace(kwargs)) # Setup logging log.setup_logging(logging_params) mlog = log.get_logger(__name__) mlog.info("ephemeris file: %s" % ephemeris.__file__) # Set the model to use fitter_function = utils.fit_point_source_transit model_function = utils.model_point_source_transit farg = inspect.getargspec(fitter_function) defaults = { key: val for key, val in zip(farg.args[-len(farg.defaults):], farg.defaults) } poly_deg_amp = kwargs.get('poly_deg_amp', defaults['poly_deg_amp']) poly_deg_phi = kwargs.get('poly_deg_phi', defaults['poly_deg_phi']) poly_type = kwargs.get('poly_type', defaults['poly_type']) param_name = ([ '%s_poly_amp_coeff%d' % (poly_type, cc) for cc in range(poly_deg_amp + 1) ] + [ '%s_poly_phi_coeff%d' % (poly_type, cc) for cc in range(poly_deg_phi + 1) ]) model_kwargs = [('poly_deg_amp', poly_deg_amp), ('poly_deg_phi', poly_deg_phi), ('poly_type', poly_type)] model_name = '.'.join( [getattr(model_function, key) for key in ['__module__', '__name__']]) tval = {} # Set where to evaluate gain ha_eval_str = ['raw_transit'] if config.multi_sample: ha_eval_str += ['transit', 'peak'] ha_eval = [0.0, None] fitslc = slice(1, 3) ind_eval = ha_eval_str.index(config.evaluate_gain_at) # Determine dimensions direction = ['amp', 'phi'] nparam = len(param_name) ngain = len(ha_eval_str) ndir = len(direction) # Determine frequencies data = andata.CorrData.from_acq_h5(file_list, datasets=(), start=start, stop=stop) freq = data.freq if physical_freq is not None: index_freq = np.array( [np.argmin(np.abs(ff - freq)) for ff in physical_freq]) freq_sel = utils.convert_to_slice(index_freq) freq = freq[index_freq] else: index_freq = np.arange(freq.size) freq_sel = None nfreq = freq.size # Compute flux of source inv_rt_flux_density = tools.invert_no_zero( np.sqrt(FluxCatalog[source].predict_flux(freq))) # Read in the eigenvaluess for all frequencies data = andata.CorrData.from_acq_h5(file_list, datasets=['erms', 'eval'], freq_sel=freq_sel, start=start, stop=stop) # Determine source coordinates this_csd = np.floor(ephemeris.unix_to_csd(np.median(data.time))) timestamp0 = ephemeris.transit_times(FluxCatalog[source].skyfield, ephemeris.csd_to_unix(this_csd))[0] src_ra, src_dec = ephemeris.object_coords(FluxCatalog[source].skyfield, date=timestamp0, deg=True) ra = ephemeris.lsa(data.time) ha = ra - src_ra ha = ha - (ha > 180.0) * 360.0 + (ha < -180.0) * 360.0 ha = np.radians(ha) itrans = np.argmin(np.abs(ha)) window = 0.75 * np.max(np.abs(ha)) off_source = np.abs(ha) > window mlog.info("CSD %d" % this_csd) mlog.info("Hour angle at transit (%d of %d): %0.2f deg " % (itrans, len(ha), np.degrees(ha[itrans]))) mlog.info("Hour angle off source: %0.2f deg" % np.median(np.abs(np.degrees(ha[off_source])))) src_dec = np.radians(src_dec) lat = np.radians(ephemeris.CHIMELATITUDE) # Determine division of frequencies ninput = data.ninput ntime = data.ntime nblock_freq = int(np.ceil(nfreq / float(config.nfreq_per_block))) # Determine bad inputs eps = 10.0 * np.finfo(data['erms'].dtype).eps good_freq = np.flatnonzero(np.all(data['erms'][:] > eps, axis=-1)) ind_sub_freq = good_freq[slice(0, good_freq.size, max(int(good_freq.size / 10), 1))] tmp_data = andata.CorrData.from_acq_h5(file_list, datasets=['evec'], freq_sel=ind_sub_freq, start=start, stop=stop) eps = 10.0 * np.finfo(tmp_data['evec'].dtype).eps bad_input = np.flatnonzero( np.all(np.abs(tmp_data['evec'][:, 0]) < eps, axis=(0, 2))) input_axis = tmp_data.input.copy() del tmp_data # Query layout database for correlator inputs if inputmap is None: inputmap = tools.get_correlator_inputs( datetime.datetime.utcfromtimestamp(data.time[itrans]), correlator='chime') inputmap = tools.reorder_correlator_inputs(input_axis, inputmap) tools.change_chime_location(rotation=config.telescope_rotation) # Determine x and y pol index xfeeds = np.array([ idf for idf, inp in enumerate(inputmap) if (idf not in bad_input) and tools.is_array_x(inp) ]) yfeeds = np.array([ idf for idf, inp in enumerate(inputmap) if (idf not in bad_input) and tools.is_array_y(inp) ]) nfeed = xfeeds.size + yfeeds.size pol = [yfeeds, xfeeds] polstr = ['Y', 'X'] npol = len(pol) neigen = min(max(npol, config.neigen), data['eval'].shape[1]) phase_ref = config.phase_reference_index phase_ref_by_pol = [ pol[pp].tolist().index(phase_ref[pp]) for pp in range(npol) ] # Calculate dynamic range eval0_off_source = np.median(data['eval'][:, 0, off_source], axis=-1) dyn = data['eval'][:, 1, :] * tools.invert_no_zero( eval0_off_source[:, np.newaxis]) # Determine frequencies to mask not_rfi = np.ones((nfreq, 1), dtype=np.bool) if config.mask_rfi is not None: for frng in config.mask_rfi: not_rfi[:, 0] &= ((freq < frng[0]) | (freq > frng[1])) mlog.info("%0.1f percent of frequencies available after masking RFI." % (100.0 * np.sum(not_rfi, dtype=np.float32) / float(nfreq), )) #dyn_flg = utils.contiguous_flag(dyn > config.dyn_rng_threshold, centre=itrans) if source in config.dyn_rng_threshold: dyn_rng_threshold = config.dyn_rng_threshold[source] else: dyn_rng_threshold = config.dyn_rng_threshold.default mlog.info("Dynamic range threshold set to %0.1f." % dyn_rng_threshold) dyn_flg = dyn > dyn_rng_threshold # Calculate fit flag fit_flag = np.zeros((nfreq, npol, ntime), dtype=np.bool) for pp in range(npol): mlog.info("Dynamic Range Nsample, Pol %d: %s" % (pp, ','.join([ "%d" % xx for xx in np.percentile(np.sum(dyn_flg, axis=-1), [25, 50, 75, 100]) ]))) if config.nsigma1 is None: fit_flag[:, pp, :] = dyn_flg & not_rfi else: fit_window = config.nsigma1 * np.radians( utils.get_window(freq, pol=polstr[pp], dec=src_dec, deg=True)) win_flg = np.abs(ha)[np.newaxis, :] <= fit_window[:, np.newaxis] fit_flag[:, pp, :] = (dyn_flg & win_flg & not_rfi) # Calculate base error base_err = data['erms'][:, np.newaxis, :] # Check for sign flips ref_resp = andata.CorrData.from_acq_h5(file_list, datasets=['evec'], input_sel=config.eigen_reference, freq_sel=freq_sel, start=start, stop=stop)['evec'][:, 0:neigen, 0, :] sign0 = 1.0 - 2.0 * (ref_resp.real < 0.0) # Check that we have the correct reference feed if np.any(np.abs(ref_resp.imag) > 0.0): ValueError("Reference feed %d is incorrect." % config.eigen_reference) del ref_resp # Save index_map results = {} results['model'] = model_name results['param'] = param_name results['freq'] = data.index_map['freq'][:] results['input'] = input_axis results['eval'] = ha_eval_str results['dir'] = direction for key, val in model_kwargs: results[key] = val # Initialize numpy arrays to hold results if config.return_response: results['response'] = np.zeros((nfreq, ninput, ntime), dtype=np.complex64) results['response_err'] = np.zeros((nfreq, ninput, ntime), dtype=np.float32) results['fit_flag'] = fit_flag results['ha_axis'] = ha results['ra'] = ra else: results['gain_eval'] = np.zeros((nfreq, ninput, ngain), dtype=np.complex64) results['weight_eval'] = np.zeros((nfreq, ninput, ngain), dtype=np.float32) results['frac_gain_err'] = np.zeros((nfreq, ninput, ngain, ndir), dtype=np.float32) results['parameter'] = np.zeros((nfreq, ninput, nparam), dtype=np.float32) results['parameter_err'] = np.zeros((nfreq, ninput, nparam), dtype=np.float32) results['index_eval'] = np.full((nfreq, ninput), -1, dtype=np.int8) results['gain'] = np.zeros((nfreq, ninput), dtype=np.complex64) results['weight'] = np.zeros((nfreq, ninput), dtype=np.float32) results['ndof'] = np.zeros((nfreq, ninput, ndir), dtype=np.float32) results['chisq'] = np.zeros((nfreq, ninput, ndir), dtype=np.float32) results['timing'] = np.zeros((nfreq, ninput), dtype=np.complex64) # Initialize metric like variables results['runtime'] = np.zeros((nblock_freq, 2), dtype=np.float64) # Compute distances dist = tools.get_feed_positions(inputmap) for pp, feeds in enumerate(pol): dist[feeds, :] -= dist[phase_ref[pp], np.newaxis, :] # Loop over frequency blocks for gg in range(nblock_freq): mlog.info("Frequency block %d of %d." % (gg, nblock_freq)) fstart = gg * config.nfreq_per_block fstop = min((gg + 1) * config.nfreq_per_block, nfreq) findex = np.arange(fstart, fstop) ngroup = findex.size freq_sel = utils.convert_to_slice(index_freq[findex]) timeit_start_gg = time.time() # if config.return_response: gstart = start gstop = stop tslc = slice(0, ntime) else: good_times = np.flatnonzero(np.any(fit_flag[findex], axis=(0, 1))) if good_times.size == 0: continue gstart = int(np.min(good_times)) gstop = int(np.max(good_times)) + 1 tslc = slice(gstart, gstop) gstart += start gstop += start hag = ha[tslc] itrans = np.argmin(np.abs(hag)) # Load eigenvectors. nudata = andata.CorrData.from_acq_h5( file_list, datasets=['evec', 'vis', 'flags/vis_weight'], apply_gain=False, freq_sel=freq_sel, start=gstart, stop=gstop) # Save time to load data results['runtime'][gg, 0] = time.time() - timeit_start_gg timeit_start_gg = time.time() mlog.info("Time to load (per frequency): %0.3f sec" % (results['runtime'][gg, 0] / ngroup, )) # Loop over polarizations for pp, feeds in enumerate(pol): # Get timing correction if tcorr is not None: tgain = tcorr.get_gain(nudata.freq, nudata.input[feeds], nudata.time) tgain *= tgain[:, phase_ref_by_pol[pp], np.newaxis, :].conj() tgain_transit = tgain[:, :, itrans].copy() tgain *= tgain_transit[:, :, np.newaxis].conj() # Create the polarization masking vector P = np.zeros((1, ninput, 1), dtype=np.float64) P[:, feeds, :] = 1.0 # Loop over frequencies for gff, ff in enumerate(findex): flg = fit_flag[ff, pp, tslc] if (2 * int(np.sum(flg))) < (nparam + 1) and not config.return_response: continue # Normalize by eigenvalue and correct for pi phase flips in process. resp = (nudata['evec'][gff, 0:neigen, :, :] * np.sqrt(data['eval'][ff, 0:neigen, np.newaxis, tslc]) * sign0[ff, :, np.newaxis, tslc]) # Rotate to single-pol response # Move time to first axis for the matrix multiplication invL = tools.invert_no_zero( np.rollaxis(data['eval'][ff, 0:neigen, np.newaxis, tslc], -1, 0)) UT = np.rollaxis(resp, -1, 0) U = np.swapaxes(UT, -1, -2) mu, vp = np.linalg.eigh(np.matmul(UT.conj(), P * U)) rsign0 = (1.0 - 2.0 * (vp[:, 0, np.newaxis, :].real < 0.0)) resp = mu[:, np.newaxis, :] * np.matmul(U, rsign0 * vp * invL) # Extract feeds of this pol # Transpose so that time is back to last axis resp = resp[:, feeds, -1].T # Compute error on response dataflg = ((nudata.weight[gff, feeds, :] > 0.0) & np.isfinite(nudata.weight[gff, feeds, :])).astype( np.float32) resp_err = dataflg * base_err[ff, :, tslc] * np.sqrt( nudata.vis[gff, feeds, :].real) * tools.invert_no_zero( np.sqrt(mu[np.newaxis, :, -1])) # Reference to specific input resp *= np.exp( -1.0J * np.angle(resp[phase_ref_by_pol[pp], np.newaxis, :])) # Apply timing correction if tcorr is not None: resp *= tgain[gff] results['timing'][ff, feeds] = tgain_transit[gff] # Fringestop lmbda = scipy.constants.c * 1e-6 / nudata.freq[gff] resp *= tools.fringestop_phase( hag[np.newaxis, :], lat, src_dec, dist[feeds, 0, np.newaxis] / lmbda, dist[feeds, 1, np.newaxis] / lmbda) # Normalize by source flux resp *= inv_rt_flux_density[ff] resp_err *= inv_rt_flux_density[ff] # If requested, reference phase to the median value if config.med_phase_ref: phi0 = np.angle(resp[:, itrans, np.newaxis]) resp *= np.exp(-1.0J * phi0) resp *= np.exp( -1.0J * np.median(np.angle(resp), axis=0, keepdims=True)) resp *= np.exp(1.0J * phi0) # Check if return_response flag was set by user if not config.return_response: if config.multi_sample: moving_window = config.nsigma2 and config.nsigma2 * np.radians( utils.get_window(nudata.freq[gff], pol=polstr[pp], dec=src_dec, deg=True)) # Loop over inputs for pii, ii in enumerate(feeds): is_good = flg & (np.abs(resp[pii, :]) > 0.0) & (resp_err[pii, :] > 0.0) # Set the intial gains based on raw response at transit if is_good[itrans]: results['gain_eval'][ff, ii, 0] = tools.invert_no_zero( resp[pii, itrans]) results['frac_gain_err'][ff, ii, 0, :] = ( resp_err[pii, itrans] * tools.invert_no_zero( np.abs(resp[pii, itrans]))) results['weight_eval'][ff, ii, 0] = 0.5 * ( np.abs(resp[pii, itrans])**2 * tools.invert_no_zero(resp_err[pii, itrans]))**2 results['index_eval'][ff, ii] = 0 results['gain'][ff, ii] = results['gain_eval'][ff, ii, 0] results['weight'][ff, ii] = results['weight_eval'][ff, ii, 0] # Exit if not performing multi time sample fit if not config.multi_sample: continue if (2 * int(np.sum(is_good))) < (nparam + 1): continue try: param, param_err, gain, gain_err, ndof, chisq, tval = fitter_function( hag[is_good], resp[pii, is_good], resp_err[pii, is_good], ha_eval, window=moving_window, tval=tval, **config.fit) except Exception as rex: if config.verbose: mlog.info( "Frequency %0.2f, Feed %d failed with error: %s" % (nudata.freq[gff], ii, rex)) continue # Check for nan wfit = (np.abs(gain) * tools.invert_no_zero(np.abs(gain_err)))**2 if np.any(~np.isfinite(np.abs(gain))) or np.any( ~np.isfinite(wfit)): continue # Save to results using the convention that you should *multiply* the visibilites by the gains results['gain_eval'][ ff, ii, fitslc] = tools.invert_no_zero(gain) results['frac_gain_err'][ff, ii, fitslc, 0] = gain_err.real results['frac_gain_err'][ff, ii, fitslc, 1] = gain_err.imag results['weight_eval'][ff, ii, fitslc] = wfit results['parameter'][ff, ii, :] = param results['parameter_err'][ff, ii, :] = param_err results['ndof'][ff, ii, :] = ndof results['chisq'][ff, ii, :] = chisq # Check if the fit was succesful and update the gain evaluation index appropriately if np.all((chisq / ndof.astype(np.float32) ) <= config.chisq_per_dof_threshold): results['index_eval'][ff, ii] = ind_eval results['gain'][ff, ii] = results['gain_eval'][ ff, ii, ind_eval] results['weight'][ff, ii] = results['weight_eval'][ ff, ii, ind_eval] else: # Return response only (do not fit model) results['response'][ff, feeds, :] = resp results['response_err'][ff, feeds, :] = resp_err # Save time to fit data results['runtime'][gg, 1] = time.time() - timeit_start_gg mlog.info("Time to fit (per frequency): %0.3f sec" % (results['runtime'][gg, 1] / ngroup, )) # Clean up del nudata gc.collect() # Print total run time mlog.info("TOTAL TIME TO LOAD: %0.3f min" % (np.sum(results['runtime'][:, 0]) / 60.0, )) mlog.info("TOTAL TIME TO FIT: %0.3f min" % (np.sum(results['runtime'][:, 1]) / 60.0, )) # Set the best estimate of the gain if not config.return_response: flag = results['index_eval'] >= 0 gain = results['gain'] # Compute amplitude amp = np.abs(gain) # Hard cutoffs on the amplitude med_amp = np.median(amp[flag]) min_amp = med_amp * config.min_amp_scale_factor max_amp = med_amp * config.max_amp_scale_factor flag &= ((amp >= min_amp) & (amp <= max_amp)) # Flag outliers in amplitude for each frequency for pp, feeds in enumerate(pol): med_amp_by_pol = np.zeros(nfreq, dtype=np.float32) sig_amp_by_pol = np.zeros(nfreq, dtype=np.float32) for ff in range(nfreq): this_flag = flag[ff, feeds] if np.any(this_flag): med, slow, shigh = utils.estimate_directional_scale( amp[ff, feeds[this_flag]]) lower = med - config.nsigma_outlier * slow upper = med + config.nsigma_outlier * shigh flag[ff, feeds] &= ((amp[ff, feeds] >= lower) & (amp[ff, feeds] <= upper)) med_amp_by_pol[ff] = med sig_amp_by_pol[ff] = 0.5 * (shigh - slow) / np.sqrt( np.sum(this_flag, dtype=np.float32)) if config.nsigma_med_outlier: med_flag = med_amp_by_pol > 0.0 not_outlier = flag_outliers(med_amp_by_pol, med_flag, window=config.window_med_outlier, nsigma=config.nsigma_med_outlier) flag[:, feeds] &= not_outlier[:, np.newaxis] mlog.info("Pol %s: %d frequencies are outliers." % (polstr[pp], np.sum(~not_outlier & med_flag, dtype=np.int))) # Determine bad frequencies flag_freq = (np.sum(flag, axis=1, dtype=np.float32) / float(ninput)) > config.threshold_good_freq good_freq = np.flatnonzero(flag_freq) # Determine bad inputs fraction_good = np.sum(flag[good_freq, :], axis=0, dtype=np.float32) / float(good_freq.size) flag_input = fraction_good > config.threshold_good_input # Finalize flag flag &= (flag_freq[:, np.newaxis] & flag_input[np.newaxis, :]) # Interpolate gains interp_gain, interp_weight = interpolate_gain( freq, gain, results['weight'], flag=flag, length_scale=config.interpolation_length_scale, mlog=mlog) # Save gains to object results['flag'] = flag results['gain'] = interp_gain results['weight'] = interp_weight # Return results return results
def main(config_file=None, logging_params=DEFAULT_LOGGING): # Load config config = DEFAULTS.deepcopy() if config_file is not None: print(config_file) config.merge(NameSpace(load_yaml_config(config_file))) # Setup logging log.setup_logging(logging_params) logger = log.get_logger(__name__) timer = Timer(logger) # Load data sfile = config.data.filename if os.path.isabs( config.data.filename) else os.path.join(config.directory, config.data.filename) sdata = StabilityData.from_file(sfile) ninput, ntime = sdata['tau'].shape # Load temperature data tfile = (config.temperature.filename if os.path.isabs(config.temperature.filename) else os.path.join( config.directory, config.temperature.filename)) tkeys = ['flag', 'data_flag', 'outlier'] if config.temperature.load: tkeys += config.temperature.load tdata = TempData.from_acq_h5(tfile, datasets=tkeys) # Query layout database inputmap = tools.get_correlator_inputs(ephemeris.unix_to_datetime( np.median(sdata.time[:])), correlator='chime') good_input = np.flatnonzero(np.any(sdata['flags']['tau'][:], axis=-1)) pol = sutil.get_pol(sdata, inputmap) npol = len(pol) mezz_index, crate_index = sutil.get_mezz_and_crate(sdata, inputmap) if config.mezz_ref.enable: phase_ref = [ ipol[mezz_index[ipol] == iref] for ipol, iref in zip(pol, config.mezz_ref.mezz) ] else: phase_ref = config.data.phase_ref # Load timing if config.timing.enable: # Extract filenames from config timing_files = [ tf if os.path.isabs(tf) else os.path.join(config.directory, tf) for tf in config.timing.files ] timing_files_hpf = [ os.path.join(os.path.dirname(tf), 'hpf', os.path.basename(tf)) for tf in timing_files ] timing_files_lpf = [ os.path.join(os.path.dirname(tf), 'lpf', os.path.basename(tf)) for tf in timing_files ] # If requested, add the timing data back into the delay data if config.timing.add.enable: timer.start("Adding timing data to delay measurements.") ns_tau, _, ns_flag, ns_inputs = sutil.get_timing_correction( sdata, timing_files, **config.timing.add.kwargs) index = timing.map_input_to_noise_source(sdata.index_map['input'], ns_inputs) timing_tau = ns_tau[index, :] timing_flag = ns_flag[index, :] for ipol, iref in zip(pol, config.data.phase_ref): timing_tau[ipol, :] = timing_tau[ipol, :] - timing_tau[ iref, np.newaxis, :] timing_flag[ipol, :] = timing_flag[ipol, :] & timing_flag[ iref, np.newaxis, :] sdata['tau'][:] = sdata['tau'][:] + timing_tau sdata['flags']['tau'][:] = sdata['flags']['tau'][:] & timing_flag timer.stop() # Extract the dependent variables from the timing dataset timer.start("Calculating timing dependence.") if config.timing.sep_delay: logger.info("Fitting HPF and LPF timing correction separately.") files = timing_files_hpf files2 = timing_files_lpf else: files2 = None if config.timing.hpf_delay: logger.info("Using HPF timing correction for delay.") files = timing_files_hpf elif config.timing.lpf_delay: logger.info("Using LPF timing correction for delay.") files = timing_files_lpf else: logger.info("Using full timing correction for delay.") files = timing_files kwargs = {} if config.timing.lpf_amp: logger.info("Using LPF timing correction for amplitude.") kwargs['afiles'] = timing_files_lpf elif config.timing.hpf_amp: logger.info("Using HPF timing correction for amplitude.") kwargs['afiles'] = timing_files_hpf else: logger.info("Using full timing correction for amplitude.") kwargs['afiles'] = timing_files for key in ['ns_ref', 'inter_cmn', 'fit_amp', 'ref_amp', 'cmn_amp']: if key in config.timing: kwargs[key] = config.timing[key] xtiming, xtiming_flag, xtiming_group = sutil.timing_dependence( sdata, files, inputmap, **kwargs) if files2 is not None: logger.info("Calculating second timing dependence.") kwargs['fit_amp'] = False xtiming2, xtiming2_flag, xtiming2_group = sutil.timing_dependence( sdata, files2, inputmap, **kwargs) xtiming = np.concatenate((xtiming, xtiming2), axis=-1) xtiming_flag = np.concatenate((xtiming_flag, xtiming2_flag), axis=-1) xtiming_group = np.concatenate((xtiming_group, xtiming2_group), axis=-1) timer.stop() else: xtiming = None xtiming_flag = None xtiming_group = None # Reference delay data to mezzanine if config.mezz_ref.enable: timer.start("Referencing delay measurements to mezzanine.") for ipol, iref in zip(pol, config.mezz_ref.mezz): this_mezz = ipol[mezz_index[ipol] == iref] wmezz = sdata['flags']['tau'][this_mezz, :].astype(np.float32) norm = np.sum(wmezz, axis=0) taut_mezz = np.sum(wmezz * sdata['tau'][this_mezz, :], axis=0) * tools.invert_no_zero(norm) flagt_mezz = norm > 0.0 sdata['tau'][ ipol, :] = sdata['tau'][ipol, :] - taut_mezz[np.newaxis, :] sdata['flags']['tau'][ipol, :] = sdata['flags']['tau'][ ipol, :] & flagt_mezz[np.newaxis, :] timer.stop() # Load cable monitor if config.cable_monitor.enable: timer.start("Calculating cable monitor dependence.") cbl = timing.TimingCorrection.from_acq_h5( config.cable_monitor.filename) kwargs = {'include_diff': config.cable_monitor.include_diff} xcable, xcable_flag, xcable_group = sutil.cable_monitor_dependence( sdata, cbl, **kwargs) timer.stop() else: xcable = None xcable_flag = None xcable_group = None # Load NS distance if config.ns_distance.enable: timer.start("Calculating NS distance dependence.") kwargs = {} kwargs['phase_ref'] = phase_ref for key in [ 'sensor', 'temp_field', 'sep_cyl', 'sep_feed', 'include_offset', 'include_ha' ]: if key in config.ns_distance: kwargs[key] = config.ns_distance[key] if config.ns_distance.use_cable_monitor: kwargs['is_cable_monitor'] = True kwargs['use_alpha'] = config.ns_distance.use_alpha nsx = timing.TimingCorrection.from_acq_h5( config.cable_monitor.filename) else: kwargs['is_cable_monitor'] = False nsx = tdata xdist, xdist_flag, xdist_group = sutil.ns_distance_dependence( sdata, nsx, inputmap, **kwargs) if (config.ns_distance.deriv is not None) and (config.ns_distance.deriv > 0): for dd in range(1, config.ns_distance.deriv + 1): d_xdist, d_xdist_flag, d_xdist_group = sutil.ns_distance_dependence( sdata, tdata, inputmap, deriv=dd, **kwargs) tind = np.atleast_1d(1) xdist = np.concatenate((xdist, d_xdist[:, :, tind]), axis=-1) xdist_flag = xnp.concatenate( (xdist_flag, d_xdist_flag[:, :, tind]), axis=-1) xdist_group = np.concatenate( (xdist_group, d_xdist_group[:, tind]), axis=-1) timer.stop() else: xdist = None xdist_flag = None xdist_group = None # Load temperatures if config.temperature.enable: timer.start("Calculating temperature dependence.") xtemp, xtemp_flag, xtemp_group, xtemp_name = sutil.temperature_dependence( sdata, tdata, config.temperature.sensor, field=config.temperature.temp_field, inputmap=inputmap, phase_ref=phase_ref, check_hut=config.temperature.check_hut) if (config.temperature.deriv is not None) and (config.temperature.deriv > 0): for dd in range(1, config.temperature.deriv + 1): d_xtemp, d_xtemp_flag, d_xtemp_group, d_xtemp_name = sutil.temperature_dependence( sdata, tdata, config.temperature.sensor, field=config.temperature.temp_field, deriv=dd, inputmap=inputmap, phase_ref=phase_ref, check_hut=config.temperature.check_hut) xtemp = np.concatenate((xtemp, d_xtemp), axis=-1) xtemp_flag = xnp.concatenate((xtemp_flag, d_xtemp_flag), axis=-1) xtemp_group = np.concatenate((xtemp_group, d_xtemp_group), axis=-1) xtemp_name += d_xtemp_name timer.stop() else: xtemp = None xtemp_flag = None xtemp_group = None xtemp_name = None # Combine into single feature matrix x, coeff_name = _concatenate(xdist, xtemp, xcable, xtiming, name_xtemp=xtemp_name) x_group, _ = _concatenate(xdist_group, xtemp_group, xcable_group, xtiming_group) x_flag, _ = _concatenate(xdist_flag, xtemp_flag, xcable_flag, xtiming_flag) x_flag = np.all(x_flag, axis=-1) & sdata.flags['tau'][:] nfeature = x.shape[-1] logger.info("Fitting %d features." % nfeature) # Save data if config.preliminary_save.enable: if config.preliminary_save.filename is not None: ofile = (config.preliminary_save.filename if os.path.isabs( config.preliminary_save.filename) else os.path.join( config.directory, config.preliminary_save.filename)) else: ofile = os.path.splitext( sfile)[0] + '_%s.h5' % config.preliminary_save.suffix sdata.save(ofile, mode='w') # Subtract mean if config.mean_subtract: timer.start("Subtracting mean value.") tau, mu_tau, mu_tau_flag = sutil.mean_subtract(sdata, sdata['tau'][:], x_flag, use_calibrator=True) mu_x = np.zeros(mu_tau.shape + (nfeature, ), dtype=x.dtype) mu_x_flag = np.zeros(mu_tau.shape + (nfeature, ), dtype=np.bool) x_no_mu = x.copy() for ff in range(nfeature): x_no_mu[..., ff], mu_x[..., ff], mu_x_flag[..., ff] = sutil.mean_subtract( sdata, x[:, :, ff], x_flag, use_calibrator=True) timer.stop() else: x_no_mu = x.copy() tau = sdata['tau'][:].copy() # Calculate unique days csd_uniq, bmap = np.unique(sdata['csd'][:], return_inverse=True) ncsd = csd_uniq.size # Prepare unique sources classification = np.char.add(np.char.add(sdata['calibrator'][:], '/'), sdata['source'][:]) # If requested, load existing coefficients if config.coeff is not None: coeff = andata.BaseData.from_acq_h5(config.coeff) evaluate_only = True else: evaluate_only = False # If requested, set up boot strapping if config.bootstrap.enable: nboot = config.bootstrap.number nchoices = ncsd if config.bootstrap.by_transit else ntime nsample = int(config.bootstrap.fraction * nchoices) bindex = np.zeros((nboot, nsample), dtype=np.int) for roll in range(nboot): bindex[roll, :] = np.sort( np.random.choice(nchoices, size=nsample, replace=config.bootstrap.replace)) else: nboot = 1 bindex = np.arange(ntime, dtype=np.int)[np.newaxis, :] # Prepare output if config.output.directory is not None: output_dir = config.output.directory else: output_dir = config.data.directory if config.output.suffix is not None: output_suffix = config.output.suffix else: output_suffix = os.path.splitext(os.path.basename( config.data.filename))[0] # Perform joint fit for bb, bind in enumerate(bindex): if config.bootstrap.enable and config.bootstrap.by_transit: tind = np.concatenate( tuple([np.flatnonzero(bmap == ii) for ii in bind])) else: tind = bind ntime = tind.size if config.jackknife.enable: start = int( config.jackknife.start * ncsd ) if config.jackknife.start <= 1.0 else config.jackknife.start end = int( config.jackknife.end * ncsd) if config.jackknife.end <= 1.0 else config.jackknife.end time_flag_fit = (bmap >= start) & (bmap < end) if config.jackknife.restrict_stat: time_flag_stat = np.logical_not(time_flag_fit) else: time_flag_stat = np.ones(ntime, dtype=np.bool) else: time_flag_fit = np.ones(ntime, dtype=np.bool) time_flag_stat = np.ones(ntime, dtype=np.bool) logger.info( "Fitting data between %s (CSD %d) and %s (CSD %d)" % (ephemeris.unix_to_datetime(np.min( sdata.time[tind[time_flag_fit]])).strftime("%Y-%m-%d"), np.min(sdata['csd'][:][tind[time_flag_fit]]), ephemeris.unix_to_datetime(np.max( sdata.time[tind[time_flag_fit]])).strftime("%Y-%m-%d"), np.max(sdata['csd'][:][tind[time_flag_fit]]))) logger.info( "Calculating statistics from data between %s (CSD %d) and %s (CSD %d)" % (ephemeris.unix_to_datetime( np.min(sdata.time[tind[time_flag_stat]])).strftime("%Y-%m-%d"), np.min(sdata['csd'][:][tind[time_flag_stat]]), ephemeris.unix_to_datetime( np.max( sdata.time[tind[time_flag_stat]])).strftime("%Y-%m-%d"), np.max(sdata['csd'][:][tind[time_flag_stat]]))) if evaluate_only: timer.start("Evaluating coefficients provided.") fitter = sutil.JointTempEvaluation( x_no_mu[:, tind, :], tau[:, tind], coeff['coeff'][:], flag=x_flag[:, tind], coeff_name=coeff.index_map['feature'][:], feature_name=coeff_name, intercept=coeff['intercept'][:], intercept_name=coeff.index_map['classification'][:], classification=classification[tind]) timer.stop() else: timer.start("Setting up fit. Bootstrap %d of %d." % (bb + 1, nboot)) fitter = sutil.JointTempRegression( x_no_mu[:, tind, :], tau[:, tind], x_group, flag=x_flag[:, tind], classification=classification[tind], coeff_name=coeff_name) timer.stop() timer.start("Performing fit. Bootstrap %d of %d." % (bb + 1, nboot)) fitter.fit_temp(time_flag=time_flag_fit, **config.fit_options) timer.stop() # If bootstrapping, append counter to filename if config.bootstrap.enable: output_suffix_bb = output_suffix + "_bootstrap_%04d" % ( config.bootstrap.index_start + bb, ) with open( os.path.join(output_dir, "bootstrap_index_%s.json" % output_suffix_bb), 'w') as jhandler: json.dump({ "bind": bind.tolist(), "tind": tind.tolist() }, jhandler) else: output_suffix_bb = output_suffix # Save statistics to file if config.output.stat: # If requested, break the model up into its various components for calculating statistics stat_key = ['data', 'model', 'resid'] if config.refine_model.enable: stat_add = fitter.refine_model(config.refine_model.include) stat_key += stat_add # Redefine axes bdata = StabilityData() for dset in ["source", "csd", "calibrator", "calibrator_time"]: bdata.create_dataset(dset, data=sdata[dset][tind]) bdata.create_index_map("time", sdata.index_map["time"][tind]) bdata.create_index_map("input", sdata.index_map["input"][:]) bdata.attrs["calibrator"] = sdata.attrs.get("calibrator", "CYG_A") # Calculate statistics stat = {} for statistic in ['std', 'mad']: for attr in stat_key: for ref, ref_common in zip(['mezz', 'cmn'], [False, True]): stat[(statistic, attr, ref)] = sutil.short_long_stat( bdata, getattr(fitter, attr), fitter._flag & time_flag_stat[np.newaxis, :], stat=statistic, ref_common=ref_common, pol=pol) output_filename = os.path.join(output_dir, "stat_%s.h5" % output_suffix_bb) write_stat(bdata, stat, fitter, output_filename) # Save coefficients to file if config.output.coeff: output_filename = os.path.join(output_dir, "coeff_%s.h5" % output_suffix_bb) write_coeff(sdata, fitter, output_filename) # Save residuals to file if config.output.resid: output_filename = os.path.join(output_dir, "resid_%s.h5" % output_suffix_bb) write_resid(sdata, fitter, output_filename) del fitter gc.collect()
def main(config_file=None, logging_params=DEFAULT_LOGGING): # Setup logging log.setup_logging(logging_params) mlog = log.get_logger(__name__) # Set config config = DEFAULTS.deepcopy() if config_file is not None: config.merge(NameSpace(load_yaml_config(config_file))) # Set niceness current_niceness = os.nice(0) os.nice(config.niceness - current_niceness) mlog.info('Changing process niceness from %d to %d. Confirm: %d' % (current_niceness, config.niceness, os.nice(0))) # Find acquisition files acq_files = sorted(glob(os.path.join(config.data_dir, config.acq, "*.h5"))) nfiles = len(acq_files) # Determine time range of each file findex = [] tindex = [] for ii, filename in enumerate(acq_files): subdata = andata.CorrData.from_acq_h5(filename, datasets=()) findex += [ii] * subdata.ntime tindex += range(subdata.ntime) findex = np.array(findex) tindex = np.array(tindex) # Determine transits within these files transits = [] data = andata.CorrData.from_acq_h5(acq_files, datasets=()) solar_rise = ephemeris.solar_rising(data.time[0] - 24.0 * 3600.0, end_time=data.time[-1]) for rr in solar_rise: ss = ephemeris.solar_setting(rr)[0] solar_flag = np.flatnonzero((data.time >= rr) & (data.time <= ss)) if solar_flag.size > 0: solar_flag = solar_flag[::config.downsample] tval = data.time[solar_flag] this_findex = findex[solar_flag] this_tindex = tindex[solar_flag] file_list, tindices = [], [] for ii in range(nfiles): this_file = np.flatnonzero(this_findex == ii) if this_file.size > 0: file_list.append(acq_files[ii]) tindices.append(this_tindex[this_file]) date = ephemeris.unix_to_datetime(rr).strftime('%Y%m%dT%H%M%SZ') transits.append((date, tval, file_list, tindices)) # Create file prefix and suffix prefix = [] prefix.append("redundant_calibration") if config.output_prefix is not None: prefix.append(config.output_prefix) prefix = '_'.join(prefix) suffix = [] if config.include_auto: suffix.append("wauto") else: suffix.append("noauto") if config.include_intracyl: suffix.append("wintra") else: suffix.append("nointra") if config.fix_degen: suffix.append("fixed_degen") else: suffix.append("degen") suffix = '_'.join(suffix) # Loop over solar transits for date, timestamps, files, time_indices in transits: nfiles = len(files) mlog.info("%s (%d files) " % (date, nfiles)) output_file = os.path.join(config.output_dir, "%s_SUN_%s_%s.h5" % (prefix, date, suffix)) mlog.info("Saving to: %s" % output_file) # Get info about this set of files data = andata.CorrData.from_acq_h5(files, datasets=['flags/inputs'], apply_gain=False, renormalize=False) coord = sun_coord(timestamps, deg=True) fstart = config.freq_start if config.freq_start is not None else 0 fstop = config.freq_stop if config.freq_stop is not None else data.freq.size freq_index = range(fstart, fstop) freq = data.freq[freq_index] ntime = timestamps.size nfreq = freq.size # Determind bad inputs if config.bad_input_file is None or not os.path.isfile( config.bad_input_file): bad_input = np.flatnonzero( ~np.all(data.flags['inputs'][:], axis=-1)) else: with open(config.bad_input_file, 'r') as handler: bad_input = pickle.load(handler) mlog.info("%d inputs flagged as bad." % bad_input.size) nant = data.ninput # Determine polarization product maps dbinputs = tools.get_correlator_inputs(ephemeris.unix_to_datetime( timestamps[0]), correlator='chime') dbinputs = tools.reorder_correlator_inputs(data.input, dbinputs) feedpos = tools.get_feed_positions(dbinputs) prod = defaultdict(list) dist = defaultdict(list) for pp, this_prod in enumerate(data.prod): aa, bb = this_prod inp_aa = dbinputs[aa] inp_bb = dbinputs[bb] if (aa in bad_input) or (bb in bad_input): continue if not tools.is_chime(inp_aa) or not tools.is_chime(inp_bb): continue if not config.include_intracyl and (inp_aa.cyl == inp_bb.cyl): continue if not config.include_auto and (aa == bb): continue this_dist = list(feedpos[aa, :] - feedpos[bb, :]) if tools.is_array_x(inp_aa) and tools.is_array_x(inp_bb): key = 'XX' elif tools.is_array_y(inp_aa) and tools.is_array_y(inp_bb): key = 'YY' elif not config.include_crosspol: continue elif tools.is_array_x(inp_aa) and tools.is_array_y(inp_bb): key = 'XY' elif tools.is_array_y(inp_aa) and tools.is_array_x(inp_bb): key = 'YX' else: raise RuntimeError("CHIME feeds not polarized.") prod[key].append(pp) dist[key].append(this_dist) polstr = sorted(prod.keys()) polcnt = 0 pol_sky_id = [] bmap = {} for key in polstr: prod[key] = np.array(prod[key]) dist[key] = np.array(dist[key]) p_bmap, p_ubaseline = generate_mapping(dist[key]) nubase = p_ubaseline.shape[0] bmap[key] = p_bmap + polcnt if polcnt > 0: ubaseline = np.concatenate((ubaseline, p_ubaseline), axis=0) pol_sky_id += [key] * nubase else: ubaseline = p_ubaseline.copy() pol_sky_id = [key] * nubase polcnt += nubase mlog.info("%d unique baselines" % polcnt) nsky = ubaseline.shape[0] # Create arrays to hold the results ores = {} ores['freq'] = freq ores['input'] = data.input ores['time'] = timestamps ores['coord'] = coord ores['pol'] = np.array(pol_sky_id) ores['baseline'] = ubaseline # Create array to hold gain results ores['gain'] = np.zeros((nfreq, nant, ntime), dtype=np.complex) ores['sky'] = np.zeros((nfreq, nsky, ntime), dtype=np.complex) ores['err'] = np.zeros((nfreq, nant + nsky, ntime, 2), dtype=np.float) # Loop over polarisations for key in polstr: reverse_map = bmap[key] p_prod = prod[key] isort = np.argsort(reverse_map) p_prod = p_prod[isort] p_ant1 = data.prod['input_a'][p_prod] p_ant2 = data.prod['input_b'][p_prod] p_vismap = reverse_map[isort] # Find the redundant groups tmp = np.where(np.diff(p_vismap) != 0)[0] edges = np.zeros(2 + tmp.size, dtype='int') edges[0] = 0 edges[1:-1] = tmp + 1 edges[-1] = p_vismap.size kept_base = np.unique(p_vismap) # Determine the unique antennas kept_ants = np.unique(np.concatenate([p_ant1, p_ant2])) antmap = np.zeros(kept_ants.max() + 1, dtype='int') - 1 p_nant = kept_ants.size for i in range(p_nant): antmap[kept_ants[i]] = i p_ant1_use = antmap[p_ant1].copy() p_ant2_use = antmap[p_ant2].copy() # Create matrix p_nvis = p_prod.size nred = edges.size - 1 npar = p_nant + nred A = np.zeros((p_nvis, npar), dtype=np.float32) B = np.zeros((p_nvis, npar), dtype=np.float32) for kk in range(p_nant): flag_ant1 = p_ant1_use == kk if np.any(flag_ant1): A[flag_ant1, kk] = 1.0 B[flag_ant1, kk] = 1.0 flag_ant2 = p_ant2_use == kk if np.any(flag_ant2): A[flag_ant2, kk] = 1.0 B[flag_ant2, kk] = -1.0 for ee in range(nred): A[edges[ee]:edges[ee + 1], p_nant + ee] = 1.0 B[edges[ee]:edges[ee + 1], p_nant + ee] = 1.0 # Add equations to break degeneracy if config.fix_degen: A = np.concatenate((A, np.zeros((1, npar), dtype=np.float32))) A[-1, 0:p_nant] = 1.0 B = np.concatenate((B, np.zeros((3, npar), dtype=np.float32))) B[-3, 0:p_nant] = 1.0 B[-2, 0:p_nant] = feedpos[kept_ants, 0] B[-1, 0:p_nant] = feedpos[kept_ants, 1] # Loop over frequencies for ff, find in enumerate(freq_index): mlog.info("Freq %d of %d. %0.2f MHz." % (ff + 1, nfreq, freq[ff])) cnt = 0 # Loop over files for ii, (filename, tind) in enumerate(zip(files, time_indices)): ntind = len(tind) mlog.info("Processing file %s (%d time samples)" % (filename, ntind)) # Compute noise weight with h5py.File(filename, 'r') as hf: wnoise = np.median(hf['flags/vis_weight'][find, :, :], axis=-1) # Loop over times for tt in tind: t0 = time.time() mlog.info("Time %d of %d. %d index of current file." % (cnt + 1, ntime, tt)) # Load visibilities with h5py.File(filename, 'r') as hf: snap = hf['vis'][find, :, tt] wsnap = wnoise * ( (hf['flags/vis_weight'][find, :, tt] > 0.0) & (np.abs(snap) > 0.0)).astype(np.float32) # Extract relevant products for this polarization snap = snap[p_prod] wsnap = wsnap[p_prod] # Turn into amplitude and phase, avoiding NaN mask = (wsnap > 0.0) amp = np.where(mask, np.log(np.abs(snap)), 0.0) phi = np.where(mask, np.angle(snap), 0.0) # Deal with phase wrapping for aa, bb in zip(edges[:-1], edges[1:]): dphi = phi[aa:bb] - np.sort(phi[aa:bb])[int( (bb - aa) / 2)] phi[aa:bb] += (2.0 * np.pi * (dphi < -np.pi) - 2.0 * np.pi * (dphi > np.pi)) # Add elements to fix degeneracy if config.fix_degen: amp = np.concatenate((amp, np.zeros(1))) phi = np.concatenate((phi, np.zeros(3))) # Determine noise matrix inv_diagC = wsnap * np.abs(snap)**2 * 2.0 if config.fix_degen: inv_diagC = np.concatenate((inv_diagC, np.ones(1))) # Amplitude estimate and covariance amp_param_cov = np.linalg.inv( np.dot(A.T, inv_diagC[:, np.newaxis] * A)) amp_param = np.dot(amp_param_cov, np.dot(A.T, inv_diagC * amp)) # Phase estimate and covariance if config.fix_degen: inv_diagC = np.concatenate((inv_diagC, np.ones(2))) phi_param_cov = np.linalg.inv( np.dot(B.T, inv_diagC[:, np.newaxis] * B)) phi_param = np.dot(phi_param_cov, np.dot(B.T, inv_diagC * phi)) # Save to large array ores['gain'][ff, kept_ants, cnt] = np.exp(amp_param[0:p_nant] + 1.0J * phi_param[0:p_nant]) ores['sky'][ff, kept_base, cnt] = np.exp(amp_param[p_nant:] + 1.0J * phi_param[p_nant:]) ores['err'][ff, kept_ants, cnt, 0] = np.diag(amp_param_cov[0:p_nant, 0:p_nant]) ores['err'][ff, nant + kept_base, cnt, 0] = np.diag(amp_param_cov[p_nant:, p_nant:]) ores['err'][ff, kept_ants, cnt, 1] = np.diag(phi_param_cov[0:p_nant, 0:p_nant]) ores['err'][ff, nant + kept_base, cnt, 1] = np.diag(phi_param_cov[p_nant:, p_nant:]) # Increment time counter cnt += 1 # Print time elapsed mlog.info("Took %0.1f seconds." % (time.time() - t0, )) # Save to pickle file with h5py.File(output_file, 'w') as handler: handler.attrs['date'] = date for key, val in ores.iteritems(): handler.create_dataset(key, data=val)
def main(config_file=None, logging_params=DEFAULT_LOGGING): # Load config config = DEFAULTS.deepcopy() if config_file is not None: config.merge(NameSpace(load_yaml_config(config_file))) # Setup logging log.setup_logging(logging_params) logger = log.get_logger(__name__) ## Load data for flagging # Load fpga restarts time_fpga_restart = [] if config.fpga_restart_file is not None: with open(config.fpga_restart_file, 'r') as handler: for line in handler: time_fpga_restart.append( ephemeris.datetime_to_unix( ephemeris.timestr_to_datetime(line.split('_')[0]))) time_fpga_restart = np.array(time_fpga_restart) # Load housekeeping flag if config.housekeeping_file is not None: ftemp = TempData.from_acq_h5(config.housekeeping_file, datasets=["time_flag"]) else: ftemp = None # Load jump data if config.jump_file is not None: with h5py.File(config.jump_file, 'r') as handler: jump_time = handler["time"][:] jump_size = handler["jump_size"][:] else: jump_time = None jump_size = None # Load rain data if config.rain_file is not None: with h5py.File(config.rain_file, 'r') as handler: rain_ranges = handler["time_range_conservative"][:] else: rain_ranges = [] # Load data flags data_flags = {} if config.data_flags: finder.connect_database() flag_types = finder.DataFlagType.select() possible_data_flags = [] for ft in flag_types: possible_data_flags.append(ft.name) if ft.name in config.data_flags: new_data_flags = finder.DataFlag.select().where( finder.DataFlag.type == ft) data_flags[ft.name] = list(new_data_flags) # Set desired range of time start_time = (ephemeris.datetime_to_unix( datetime.datetime( *config.start_date)) if config.start_date is not None else None) end_time = (ephemeris.datetime_to_unix(datetime.datetime( *config.end_date)) if config.end_date is not None else None) ## Find gain files files = {} for src in config.sources: files[src] = sorted( glob.glob( os.path.join(config.directory, src.lower(), "%s_%s_lsd_*.h5" % ( config.prefix, src.lower(), )))) csd = {} for src in config.sources: csd[src] = np.array( [int(os.path.splitext(ff)[0][-4:]) for ff in files[src]]) for src in config.sources: logger.info("%s: %d files" % (src, len(csd[src]))) ## Remove files that occur during flag csd_flag = {} for src in config.sources: body = ephemeris.source_dictionary[src] csd_flag[src] = np.ones(csd[src].size, dtype=np.bool) for ii, cc in enumerate(csd[src][:]): ttrans = ephemeris.transit_times(body, ephemeris.csd_to_unix(cc))[0] if (start_time is not None) and (ttrans < start_time): csd_flag[src][ii] = False continue if (end_time is not None) and (ttrans > end_time): csd_flag[src][ii] = False continue # If requested, remove daytime transits if not config.include_daytime.get( src, config.include_daytime.default) and daytime_flag( ttrans)[0]: logger.info("%s CSD %d: daytime transit" % (src, cc)) csd_flag[src][ii] = False continue # Remove transits during HKP drop out if ftemp is not None: itemp = np.flatnonzero( (ftemp.time[:] >= (ttrans - config.transit_window)) & (ftemp.time[:] <= (ttrans + config.transit_window))) tempflg = ftemp['time_flag'][itemp] if (tempflg.size == 0) or ((np.sum(tempflg, dtype=np.float32) / float(tempflg.size)) < 0.50): logger.info("%s CSD %d: no housekeeping" % (src, cc)) csd_flag[src][ii] = False continue # Remove transits near jumps if jump_time is not None: njump = np.sum((jump_size > config.min_jump_size) & (jump_time > (ttrans - config.jump_window)) & (jump_time < ttrans)) if njump > config.max_njump: logger.info("%s CSD %d: %d jumps before" % (src, cc, njump)) csd_flag[src][ii] = False continue # Remove transits near rain for rng in rain_ranges: if (((ttrans - config.transit_window) <= rng[1]) and ((ttrans + config.transit_window) >= rng[0])): logger.info("%s CSD %d: during rain" % (src, cc)) csd_flag[src][ii] = False break # Remove transits during data flag for name, flag_list in data_flags.items(): if csd_flag[src][ii]: for flg in flag_list: if (((ttrans - config.transit_window) <= flg.finish_time) and ((ttrans + config.transit_window) >= flg.start_time)): logger.info("%s CSD %d: %s flag" % (src, cc, name)) csd_flag[src][ii] = False break # Print number of files left after flagging for src in config.sources: logger.info("%s: %d files (after flagging)" % (src, np.sum(csd_flag[src]))) ## Construct pair wise differences npair = len(config.diff_pair) shift = [nd * 24.0 * 3600.0 for nd in config.nday_shift] calmap = [] calpair = [] for (tsrc, csrc), sh in zip(config.diff_pair, shift): body_test = ephemeris.source_dictionary[tsrc] body_cal = ephemeris.source_dictionary[csrc] for ii, cc in enumerate(csd[tsrc]): if csd_flag[tsrc][ii]: test_transit = ephemeris.transit_times( body_test, ephemeris.csd_to_unix(cc))[0] cal_transit = ephemeris.transit_times(body_cal, test_transit + sh)[0] cal_csd = int(np.fix(ephemeris.unix_to_csd(cal_transit))) ttrans = np.sort([test_transit, cal_transit]) if cal_csd in csd[csrc]: jj = list(csd[csrc]).index(cal_csd) if csd_flag[csrc][jj] and not np.any( (time_fpga_restart >= ttrans[0]) & (time_fpga_restart <= ttrans[1])): calmap.append([ii, jj]) calpair.append([tsrc, csrc]) calmap = np.array(calmap) calpair = np.array(calpair) ntransit = calmap.shape[0] logger.info("%d total transit pairs" % ntransit) for ii in range(ntransit): t1 = ephemeris.transit_times( ephemeris.source_dictionary[calpair[ii, 0]], ephemeris.csd_to_unix(csd[calpair[ii, 0]][calmap[ii, 0]]))[0] t2 = ephemeris.transit_times( ephemeris.source_dictionary[calpair[ii, 1]], ephemeris.csd_to_unix(csd[calpair[ii, 1]][calmap[ii, 1]]))[0] logger.info("%s (%d) - %s (%d): %0.1f hr" % (calpair[ii, 0], csd_flag[calpair[ii, 0]][calmap[ii, 0]], calpair[ii, 1], csd_flag[calpair[ii, 1]][calmap[ii, 1]], (t1 - t2) / 3600.0)) # Determine unique diff pairs diff_name = np.array(['%s/%s' % tuple(cp) for cp in calpair]) uniq_diff, lbl_diff, cnt_diff = np.unique(diff_name, return_inverse=True, return_counts=True) ndiff = uniq_diff.size for ud, udcnt in zip(uniq_diff, cnt_diff): logger.info("%s: %d transit pairs" % (ud, udcnt)) ## Load gains inputmap = tools.get_correlator_inputs(datetime.datetime.utcnow(), correlator='chime') ninput = len(inputmap) nfreq = 1024 # Set up gain arrays gain = np.zeros((2, nfreq, ninput, ntransit), dtype=np.complex64) weight = np.zeros((2, nfreq, ninput, ntransit), dtype=np.float32) input_sort = np.zeros((2, ninput, ntransit), dtype=np.int) kcsd = np.zeros((2, ntransit), dtype=np.float32) timestamp = np.zeros((2, ntransit), dtype=np.float64) is_daytime = np.zeros((2, ntransit), dtype=np.bool) for tt in range(ntransit): for kk, (src, ind) in enumerate(zip(calpair[tt], calmap[tt])): body = ephemeris.source_dictionary[src] filename = files[src][ind] logger.info("%s: %s" % (src, filename)) temp = containers.StaticGainData.from_file(filename) freq = temp.freq[:] inputs = temp.input[:] isort = reorder_inputs(inputmap, inputs) inputs = inputs[isort] gain[kk, :, :, tt] = temp.gain[:, isort] weight[kk, :, :, tt] = temp.weight[:, isort] input_sort[kk, :, tt] = isort kcsd[kk, tt] = temp.attrs['lsd'] timestamp[kk, tt] = ephemeris.transit_times( body, ephemeris.csd_to_unix(kcsd[kk, tt]))[0] is_daytime[kk, tt] = daytime_flag(timestamp[kk, tt])[0] if np.any(isort != np.arange(isort.size)): logger.info("Input ordering has changed: %s" % ephemeris.unix_to_datetime( timestamp[kk, tt]).strftime("%Y-%m-%d")) logger.info("") inputs = np.array([(inp.id, inp.input_sn) for inp in inputmap], dtype=[('chan_id', 'u2'), ('correlator_input', 'S32')]) ## Load input flags inpflg = np.ones((2, ninput, ntransit), dtype=np.bool) min_flag_time = np.min(timestamp) - 7.0 * 24.0 * 60.0 * 60.0 max_flag_time = np.max(timestamp) + 7.0 * 24.0 * 60.0 * 60.0 flaginput_files = sorted( glob.glob( os.path.join(config.flaginput_dir, "*" + config.flaginput_suffix, "*.h5"))) if flaginput_files: logger.info("Found %d flaginput files." % len(flaginput_files)) tmp = andata.FlagInputData.from_acq_h5(flaginput_files, datasets=()) start, stop = [ int(yy) for yy in np.percentile( np.flatnonzero((tmp.time[:] >= min_flag_time) & (tmp.time[:] <= max_flag_time)), [0, 100]) ] cont = andata.FlagInputData.from_acq_h5(flaginput_files, start=start, stop=stop, datasets=['flag']) for kk in range(2): inpflg[kk, :, :] = cont.resample('flag', timestamp[kk], transpose=True) logger.info("Flaginput time offsets in minutes (pair %d):" % kk) logger.info( str( np.fix((cont.time[cont.search_update_time(timestamp[kk])] - timestamp[kk]) / 60.0).astype(np.int))) # Sort flags so they are in same order for tt in range(ntransit): for kk in range(2): inpflg[kk, :, tt] = inpflg[kk, input_sort[kk, :, tt], tt] # Do not apply input flag to phase reference for ii in config.index_phase_ref: inpflg[:, ii, :] = True ## Flag out gains with high uncertainty and frequencies with large fraction of data flagged frac_err = tools.invert_no_zero(np.sqrt(weight) * np.abs(gain)) flag = np.all((weight > 0.0) & (np.abs(gain) > 0.0) & (frac_err < config.max_uncertainty), axis=0) freq_flag = ((np.sum(flag, axis=(1, 2), dtype=np.float32) / float(np.prod(flag.shape[1:]))) > config.freq_threshold) if config.apply_rfi_mask: freq_flag &= np.logical_not(rfi.frequency_mask(freq)) flag = flag & freq_flag[:, np.newaxis, np.newaxis] good_freq = np.flatnonzero(freq_flag) logger.info("Number good frequencies %d" % good_freq.size) ## Generate flags with more conservative cuts on frequency c_flag = flag & np.all(frac_err < config.conservative.max_uncertainty, axis=0) c_freq_flag = ((np.sum(c_flag, axis=(1, 2), dtype=np.float32) / float(np.prod(c_flag.shape[1:]))) > config.conservative.freq_threshold) if config.conservative.apply_rfi_mask: c_freq_flag &= np.logical_not(rfi.frequency_mask(freq)) c_flag = c_flag & c_freq_flag[:, np.newaxis, np.newaxis] c_good_freq = np.flatnonzero(c_freq_flag) logger.info("Number good frequencies (conservative thresholds) %d" % c_good_freq.size) ## Apply input flags flag &= np.all(inpflg[:, np.newaxis, :, :], axis=0) ## Update flags based on beam flag if config.beam_flag_file is not None: dbeam = andata.BaseData.from_acq_h5(config.beam_flag_file) db_csd = np.floor(ephemeris.unix_to_csd(dbeam.index_map['time'][:])) for ii, name in enumerate(config.beam_flag_datasets): logger.info("Applying %s beam flag." % name) if not ii: db_flag = dbeam.flags[name][:] else: db_flag &= dbeam.flags[name][:] cnt = 0 for ii, dbc in enumerate(db_csd): this_csd = np.flatnonzero(np.any(kcsd == dbc, axis=0)) if this_csd.size > 0: logger.info("Beam flag for %d matches %s." % (dbc, str(kcsd[:, this_csd]))) flag[:, :, this_csd] &= db_flag[np.newaxis, :, ii, np.newaxis] cnt += 1 logger.info("Applied %0.1f percent of the beam flags" % (100.0 * cnt / float(db_csd.size), )) ## Flag inputs with large amount of missing data input_frac_flagged = ( np.sum(flag[good_freq, :, :], axis=(0, 2), dtype=np.float32) / float(good_freq.size * ntransit)) input_flag = input_frac_flagged > config.input_threshold for ii in config.index_phase_ref: logger.info("Phase reference %d has %0.3f fraction of data flagged." % (ii, input_frac_flagged[ii])) input_flag[ii] = True good_input = np.flatnonzero(input_flag) flag = flag & input_flag[np.newaxis, :, np.newaxis] logger.info("Number good inputs %d" % good_input.size) ## Calibrate gaincal = gain[0] * tools.invert_no_zero(gain[1]) frac_err_cal = np.sqrt(frac_err[0]**2 + frac_err[1]**2) count = np.sum(flag, axis=-1, dtype=np.int) stat_flag = count > config.min_num_transit ## Calculate phase amp = np.abs(gaincal) phi = np.angle(gaincal) ## Calculate polarisation groups pol_dict = {'E': 'X', 'S': 'Y'} cyl_dict = {2: 'A', 3: 'B', 4: 'C', 5: 'D'} if config.group_by_cyl: group_id = [ (inp.pol, inp.cyl) if tools.is_chime(inp) and (ii in good_input) else None for ii, inp in enumerate(inputmap) ] else: group_id = [ inp.pol if tools.is_chime(inp) and (ii in good_input) else None for ii, inp in enumerate(inputmap) ] ugroup_id = sorted([uidd for uidd in set(group_id) if uidd is not None]) ngroup = len(ugroup_id) group_list_noref = [ np.array([ gg for gg, gid in enumerate(group_id) if (gid == ugid) and gg not in config.index_phase_ref ]) for ugid in ugroup_id ] group_list = [ np.array([gg for gg, gid in enumerate(group_id) if gid == ugid]) for ugid in ugroup_id ] if config.group_by_cyl: group_str = [ "%s-%s" % (pol_dict[pol], cyl_dict[cyl]) for pol, cyl in ugroup_id ] else: group_str = [pol_dict[pol] for pol in ugroup_id] index_phase_ref = [] for gstr, igroup in zip(group_str, group_list): candidate = [ii for ii in config.index_phase_ref if ii in igroup] if len(candidate) != 1: index_phase_ref.append(None) else: index_phase_ref.append(candidate[0]) logger.info( "Phase reference: %s" % ', '.join(['%s = %s' % tpl for tpl in zip(group_str, index_phase_ref)])) ## Apply thermal correction to amplitude if config.amp_thermal.enabled: logger.info("Applying thermal correction.") # Load the temperatures tdata = TempData.from_acq_h5(config.amp_thermal.filename) index = tdata.search_sensors(config.amp_thermal.sensor)[0] temp = tdata.datasets[config.amp_thermal.field][index] temp_func = scipy.interpolate.interp1d(tdata.time, temp, **config.amp_thermal.interp) itemp = temp_func(timestamp) dtemp = itemp[0] - itemp[1] flag_func = scipy.interpolate.interp1d( tdata.time, tdata.datasets['flag'][index].astype(np.float32), **config.amp_thermal.interp) dtemp_flag = np.all(flag_func(timestamp) == 1.0, axis=0) flag &= dtemp_flag[np.newaxis, np.newaxis, :] for gstr, igroup in zip(group_str, group_list): pstr = gstr[0] thermal_coeff = np.polyval(config.amp_thermal.coeff[pstr], freq) gthermal = 1.0 + thermal_coeff[:, np.newaxis, np.newaxis] * dtemp[ np.newaxis, np.newaxis, :] amp[:, igroup, :] *= tools.invert_no_zero(gthermal) ## Compute common mode if config.subtract_common_mode_before: logger.info("Calculating common mode amplitude and phase.") cmn_amp, flag_cmn_amp = compute_common_mode(amp, flag, group_list_noref, median=False) cmn_phi, flag_cmn_phi = compute_common_mode(phi, flag, group_list_noref, median=False) # Subtract common mode (from phase only) logger.info("Subtracting common mode phase.") group_flag = np.zeros((ngroup, ninput), dtype=np.bool) for gg, igroup in enumerate(group_list): group_flag[gg, igroup] = True phi[:, igroup, :] = phi[:, igroup, :] - cmn_phi[:, gg, np.newaxis, :] for iref in index_phase_ref: if (iref is not None) and (iref in igroup): flag[:, iref, :] = flag_cmn_phi[:, gg, :] ## If requested, determine and subtract a delay template if config.fit_delay_before: logger.info("Fitting delay template.") omega = timing.FREQ_TO_OMEGA * freq tau, tau_flag, _ = construct_delay_template( omega, phi, c_flag & flag, min_num_freq_for_delay_fit=config.min_num_freq_for_delay_fit) # Compute residuals logger.info("Subtracting delay template.") phi = phi - tau[np.newaxis, :, :] * omega[:, np.newaxis, np.newaxis] ## Normalize by median over time logger.info("Calculating median amplitude and phase.") med_amp = np.zeros((nfreq, ninput, ndiff), dtype=amp.dtype) med_phi = np.zeros((nfreq, ninput, ndiff), dtype=phi.dtype) count_by_diff = np.zeros((nfreq, ninput, ndiff), dtype=np.int) stat_flag_by_diff = np.zeros((nfreq, ninput, ndiff), dtype=np.bool) def weighted_mean(yy, ww, axis=-1): return np.sum(ww * yy, axis=axis) * tools.invert_no_zero( np.sum(ww, axis=axis)) for dd in range(ndiff): this_diff = np.flatnonzero(lbl_diff == dd) this_flag = flag[:, :, this_diff] this_amp = amp[:, :, this_diff] this_amp_err = this_amp * frac_err_cal[:, :, this_diff] * this_flag.astype( np.float32) this_phi = phi[:, :, this_diff] this_phi_err = frac_err_cal[:, :, this_diff] * this_flag.astype( np.float32) count_by_diff[:, :, dd] = np.sum(this_flag, axis=-1, dtype=np.int) stat_flag_by_diff[:, :, dd] = count_by_diff[:, :, dd] > config.min_num_transit if config.weighted_mean == 2: logger.info("Calculating inverse variance weighted mean.") med_amp[:, :, dd] = weighted_mean(this_amp, tools.invert_no_zero(this_amp_err**2), axis=-1) med_phi[:, :, dd] = weighted_mean(this_phi, tools.invert_no_zero(this_phi_err**2), axis=-1) elif config.weighted_mean == 1: logger.info("Calculating uniform weighted mean.") med_amp[:, :, dd] = weighted_mean(this_amp, this_flag.astype(np.float32), axis=-1) med_phi[:, :, dd] = weighted_mean(this_phi, this_flag.astype(np.float32), axis=-1) else: logger.info("Calculating median value.") for ff in range(nfreq): for ii in range(ninput): if np.any(this_flag[ff, ii, :]): med_amp[ff, ii, dd] = wq.median( this_amp[ff, ii, :], this_flag[ff, ii, :].astype(np.float32)) med_phi[ff, ii, dd] = wq.median( this_phi[ff, ii, :], this_flag[ff, ii, :].astype(np.float32)) damp = np.zeros_like(amp) dphi = np.zeros_like(phi) for dd in range(ndiff): this_diff = np.flatnonzero(lbl_diff == dd) damp[:, :, this_diff] = amp[:, :, this_diff] * tools.invert_no_zero( med_amp[:, :, dd, np.newaxis]) - 1.0 dphi[:, :, this_diff] = phi[:, :, this_diff] - med_phi[:, :, dd, np.newaxis] # Compute common mode if not config.subtract_common_mode_before: logger.info("Calculating common mode amplitude and phase.") cmn_amp, flag_cmn_amp = compute_common_mode(damp, flag, group_list_noref, median=True) cmn_phi, flag_cmn_phi = compute_common_mode(dphi, flag, group_list_noref, median=True) # Subtract common mode (from phase only) logger.info("Subtracting common mode phase.") group_flag = np.zeros((ngroup, ninput), dtype=np.bool) for gg, igroup in enumerate(group_list): group_flag[gg, igroup] = True dphi[:, igroup, :] = dphi[:, igroup, :] - cmn_phi[:, gg, np.newaxis, :] for iref in index_phase_ref: if (iref is not None) and (iref in igroup): flag[:, iref, :] = flag_cmn_phi[:, gg, :] ## Compute RMS logger.info("Calculating RMS of amplitude and phase.") mad_amp = np.zeros((nfreq, ninput), dtype=amp.dtype) std_amp = np.zeros((nfreq, ninput), dtype=amp.dtype) mad_phi = np.zeros((nfreq, ninput), dtype=phi.dtype) std_phi = np.zeros((nfreq, ninput), dtype=phi.dtype) mad_amp_by_diff = np.zeros((nfreq, ninput, ndiff), dtype=amp.dtype) std_amp_by_diff = np.zeros((nfreq, ninput, ndiff), dtype=amp.dtype) mad_phi_by_diff = np.zeros((nfreq, ninput, ndiff), dtype=phi.dtype) std_phi_by_diff = np.zeros((nfreq, ninput, ndiff), dtype=phi.dtype) for ff in range(nfreq): for ii in range(ninput): this_flag = flag[ff, ii, :] if np.any(this_flag): std_amp[ff, ii] = np.std(damp[ff, ii, this_flag]) std_phi[ff, ii] = np.std(dphi[ff, ii, this_flag]) mad_amp[ff, ii] = 1.48625 * wq.median( np.abs(damp[ff, ii, :]), this_flag.astype(np.float32)) mad_phi[ff, ii] = 1.48625 * wq.median( np.abs(dphi[ff, ii, :]), this_flag.astype(np.float32)) for dd in range(ndiff): this_diff = this_flag & (lbl_diff == dd) if np.any(this_diff): std_amp_by_diff[ff, ii, dd] = np.std(damp[ff, ii, this_diff]) std_phi_by_diff[ff, ii, dd] = np.std(dphi[ff, ii, this_diff]) mad_amp_by_diff[ff, ii, dd] = 1.48625 * wq.median( np.abs(damp[ff, ii, :]), this_diff.astype(np.float32)) mad_phi_by_diff[ff, ii, dd] = 1.48625 * wq.median( np.abs(dphi[ff, ii, :]), this_diff.astype(np.float32)) ## Construct delay template if not config.fit_delay_before: logger.info("Fitting delay template.") omega = timing.FREQ_TO_OMEGA * freq tau, tau_flag, _ = construct_delay_template( omega, dphi, c_flag & flag, min_num_freq_for_delay_fit=config.min_num_freq_for_delay_fit) # Compute residuals logger.info("Subtracting delay template from phase.") resid = (dphi - tau[np.newaxis, :, :] * omega[:, np.newaxis, np.newaxis]) * flag.astype(np.float32) else: resid = dphi tau_count = np.sum(tau_flag, axis=-1, dtype=np.int) tau_stat_flag = tau_count > config.min_num_transit tau_count_by_diff = np.zeros((ninput, ndiff), dtype=np.int) tau_stat_flag_by_diff = np.zeros((ninput, ndiff), dtype=np.bool) for dd in range(ndiff): this_diff = np.flatnonzero(lbl_diff == dd) tau_count_by_diff[:, dd] = np.sum(tau_flag[:, this_diff], axis=-1, dtype=np.int) tau_stat_flag_by_diff[:, dd] = tau_count_by_diff[:, dd] > config.min_num_transit ## Calculate statistics of residuals std_resid = np.zeros((nfreq, ninput), dtype=phi.dtype) mad_resid = np.zeros((nfreq, ninput), dtype=phi.dtype) std_resid_by_diff = np.zeros((nfreq, ninput, ndiff), dtype=phi.dtype) mad_resid_by_diff = np.zeros((nfreq, ninput, ndiff), dtype=phi.dtype) for ff in range(nfreq): for ii in range(ninput): this_flag = flag[ff, ii, :] if np.any(this_flag): std_resid[ff, ii] = np.std(resid[ff, ii, this_flag]) mad_resid[ff, ii] = 1.48625 * wq.median( np.abs(resid[ff, ii, :]), this_flag.astype(np.float32)) for dd in range(ndiff): this_diff = this_flag & (lbl_diff == dd) if np.any(this_diff): std_resid_by_diff[ff, ii, dd] = np.std(resid[ff, ii, this_diff]) mad_resid_by_diff[ff, ii, dd] = 1.48625 * wq.median( np.abs(resid[ff, ii, :]), this_diff.astype(np.float32)) ## Calculate statistics of delay template mad_tau = np.zeros((ninput, ), dtype=phi.dtype) std_tau = np.zeros((ninput, ), dtype=phi.dtype) mad_tau_by_diff = np.zeros((ninput, ndiff), dtype=phi.dtype) std_tau_by_diff = np.zeros((ninput, ndiff), dtype=phi.dtype) for ii in range(ninput): this_flag = tau_flag[ii] if np.any(this_flag): std_tau[ii] = np.std(tau[ii, this_flag]) mad_tau[ii] = 1.48625 * wq.median(np.abs(tau[ii]), this_flag.astype(np.float32)) for dd in range(ndiff): this_diff = this_flag & (lbl_diff == dd) if np.any(this_diff): std_tau_by_diff[ii, dd] = np.std(tau[ii, this_diff]) mad_tau_by_diff[ii, dd] = 1.48625 * wq.median( np.abs(tau[ii]), this_diff.astype(np.float32)) ## Define output res = { "timestamp": { "data": timestamp, "axis": ["div", "time"] }, "is_daytime": { "data": is_daytime, "axis": ["div", "time"] }, "csd": { "data": kcsd, "axis": ["div", "time"] }, "pair_map": { "data": lbl_diff, "axis": ["time"] }, "pair_count": { "data": cnt_diff, "axis": ["pair"] }, "gain": { "data": gaincal, "axis": ["freq", "input", "time"] }, "frac_err": { "data": frac_err_cal, "axis": ["freq", "input", "time"] }, "flags/gain": { "data": flag, "axis": ["freq", "input", "time"], "flag": True }, "flags/gain_conservative": { "data": c_flag, "axis": ["freq", "input", "time"], "flag": True }, "flags/count": { "data": count, "axis": ["freq", "input"], "flag": True }, "flags/stat": { "data": stat_flag, "axis": ["freq", "input"], "flag": True }, "flags/count_by_pair": { "data": count_by_diff, "axis": ["freq", "input", "pair"], "flag": True }, "flags/stat_by_pair": { "data": stat_flag_by_diff, "axis": ["freq", "input", "pair"], "flag": True }, "med_amp": { "data": med_amp, "axis": ["freq", "input", "pair"] }, "med_phi": { "data": med_phi, "axis": ["freq", "input", "pair"] }, "flags/group_flag": { "data": group_flag, "axis": ["group", "input"], "flag": True }, "cmn_amp": { "data": cmn_amp, "axis": ["freq", "group", "time"] }, "cmn_phi": { "data": cmn_phi, "axis": ["freq", "group", "time"] }, "amp": { "data": damp, "axis": ["freq", "input", "time"] }, "phi": { "data": dphi, "axis": ["freq", "input", "time"] }, "std_amp": { "data": std_amp, "axis": ["freq", "input"] }, "std_amp_by_pair": { "data": std_amp_by_diff, "axis": ["freq", "input", "pair"] }, "mad_amp": { "data": mad_amp, "axis": ["freq", "input"] }, "mad_amp_by_pair": { "data": mad_amp_by_diff, "axis": ["freq", "input", "pair"] }, "std_phi": { "data": std_phi, "axis": ["freq", "input"] }, "std_phi_by_pair": { "data": std_phi_by_diff, "axis": ["freq", "input", "pair"] }, "mad_phi": { "data": mad_phi, "axis": ["freq", "input"] }, "mad_phi_by_pair": { "data": mad_phi_by_diff, "axis": ["freq", "input", "pair"] }, "tau": { "data": tau, "axis": ["input", "time"] }, "flags/tau": { "data": tau_flag, "axis": ["input", "time"], "flag": True }, "flags/tau_count": { "data": tau_count, "axis": ["input"], "flag": True }, "flags/tau_stat": { "data": tau_stat_flag, "axis": ["input"], "flag": True }, "flags/tau_count_by_pair": { "data": tau_count_by_diff, "axis": ["input", "pair"], "flag": True }, "flags/tau_stat_by_pair": { "data": tau_stat_flag_by_diff, "axis": ["input", "pair"], "flag": True }, "std_tau": { "data": std_tau, "axis": ["input"] }, "std_tau_by_pair": { "data": std_tau_by_diff, "axis": ["input", "pair"] }, "mad_tau": { "data": mad_tau, "axis": ["input"] }, "mad_tau_by_pair": { "data": mad_tau_by_diff, "axis": ["input", "pair"] }, "resid_phi": { "data": resid, "axis": ["freq", "input", "time"] }, "std_resid_phi": { "data": std_resid, "axis": ["freq", "input"] }, "std_resid_phi_by_pair": { "data": std_resid_by_diff, "axis": ["freq", "input", "pair"] }, "mad_resid_phi": { "data": mad_resid, "axis": ["freq", "input"] }, "mad_resid_phi_by_pair": { "data": mad_resid_by_diff, "axis": ["freq", "input", "pair"] }, } ## Create the output container logger.info("Creating StabilityData container.") data = StabilityData() data.create_index_map( "div", np.array(["numerator", "denominator"], dtype=np.string_)) data.create_index_map("pair", np.array(uniq_diff, dtype=np.string_)) data.create_index_map("group", np.array(group_str, dtype=np.string_)) data.create_index_map("freq", freq) data.create_index_map("input", inputs) data.create_index_map("time", timestamp[0, :]) logger.info("Writing datsets to container.") for name, dct in res.iteritems(): is_flag = dct.get('flag', False) if is_flag: dset = data.create_flag(name.split('/')[-1], data=dct['data']) else: dset = data.create_dataset(name, data=dct['data']) dset.attrs['axis'] = np.array(dct['axis'], dtype=np.string_) data.attrs['phase_ref'] = np.array( [iref for iref in index_phase_ref if iref is not None]) # Determine the output filename and save results start_time, end_time = ephemeris.unix_to_datetime( np.percentile(timestamp, [0, 100])) tfmt = "%Y%m%d" night_str = 'night_' if not np.any(is_daytime) else '' output_file = os.path.join( config.output_dir, "%s_%s_%sraw_stability_data.h5" % (start_time.strftime(tfmt), end_time.strftime(tfmt), night_str)) logger.info("Saving results to %s." % output_file) data.save(output_file)
def main(config_file=None, logging_params=DEFAULT_LOGGING): # Setup logging log.setup_logging(logging_params) mlog = log.get_logger(__name__) # Set config config = DEFAULTS.deepcopy() if config_file is not None: config.merge(NameSpace(load_yaml_config(config_file))) # Create transit tracker source_list = FluxCatalog.sort( ) if not config.source_list else config.source_list cal_list = [ name for name, obj in FluxCatalog.iteritems() if (obj.dec >= config.min_dec) and ( obj.predict_flux(config.freq_nominal) >= config.min_flux) and ( name in source_list) ] if not cal_list: raise RuntimeError("No calibrators found.") # Sort list by flux at nominal frequency cal_list.sort( key=lambda name: FluxCatalog[name].predict_flux(config.freq_nominal)) # Add to transit tracker transit_tracker = containers.TransitTrackerOffline( nsigma=config.nsigma_source, extend_night=config.extend_night) for name in cal_list: transit_tracker[name] = FluxCatalog[name].skyfield mlog.info("Initializing offline point source processing.") search_time = config.start_time or 0 # Find all calibration files all_files = sorted( glob.glob( os.path.join(config.acq_dir, '*' + config.correlator + config.acq_suffix, '*.h5'))) if not all_files: return # Remove files whose last modified time is before the time of the most recent update all_files = [ ff for ff in all_files if (os.path.getmtime(ff) > search_time) ] if not all_files: return # Remove files that are currently locked all_files = [ ff for ff in all_files if not os.path.isfile(os.path.splitext(ff)[0] + '.lock') ] if not all_files: return # Add files to transit tracker for ff in all_files: transit_tracker.add_file(ff) # Extract point source transits ready for analysis all_transits = transit_tracker.get_transits() # Create dictionary to hold results h5_psrc_fit = {} inputmap = None # Loop over transits for transit in all_transits: src, csd, is_day, files, start, stop = transit # Discard any point sources with unusual csd value if (csd < config.min_csd) or (csd > config.max_csd): continue # Discard any point sources transiting during the day if is_day > config.process_daytime: continue mlog.info( 'Processing %s transit on CSD %d (%d files, %d time samples)' % (src, csd, len(files), stop - start + 1)) # Load inputmap if inputmap is None: if config.inputmap is None: inputmap = tools.get_correlator_inputs( ephemeris.unix_to_datetime(ephemeris.csd_to_unix(csd)), correlator=config.correlator) else: with open(config.inputmap, 'r') as handler: inputmap = pickle.load(handler) # Grab the timing correction for this transit tcorr = None if config.apply_timing: if config.timing_glob is not None: mlog.info( "Loading timing correction from extended timing solutions." ) timing_files = sorted(glob.glob(config.timing_glob)) if timing_files: try: tcorr = search_extended_timing_solutions( timing_files, ephemeris.csd_to_unix(csd)) except Exception as e: mlog.error( 'search_extended_timing_solutions failed with error: %s' % e) else: mlog.info(str(tcorr)) if tcorr is None: mlog.info( "Loading timing correction from chimetiming acquisitions.") try: tcorr = timing.load_timing_correction( files, start=start, stop=stop, window=config.timing_window, instrument=config.correlator) except Exception as e: mlog.error( 'timing.load_timing_correction failed with error: %s' % e) mlog.warning( 'No timing correction applied to %s transit on CSD %d.' % (src, csd)) else: mlog.info(str(tcorr)) # Call the main routine to process data try: outdct = offline_cal.offline_point_source_calibration( files, src, start=start, stop=stop, inputmap=inputmap, tcorr=tcorr, logging_params=logging_params, **config.analysis.as_dict()) except Exception as e: msg = 'offline_cal.offline_point_source_calibration failed with error: %s' % e mlog.error(msg) continue #raise RuntimeError(msg) # Find existing gain files for this particular point source if src not in h5_psrc_fit: output_files = find_files(config, psrc=src) if output_files is not None: output_files = output_files[-1] mlog.info('Writing %s transit on CSD %d to existing file %s.' % (src, csd, output_files)) h5_psrc_fit[src] = containers.PointSourceWriter( src, output_file=output_files, output_dir=config.output_dir, output_suffix=point_source_name_to_file_suffix(src), instrument=config.correlator, max_file_size=config.max_file_size, max_num=config.max_num_time, memory_size=0) # Associate this gain calibration to the transit time this_time = ephemeris.transit_times(FluxCatalog[src].skyfield, ephemeris.csd_to_unix(csd))[0] outdct['csd'] = csd outdct['is_daytime'] = is_day outdct['acquisition'] = os.path.basename(os.path.dirname(files[0])) # Write to output file mlog.info('Writing to disk results from %s transit on CSD %d.' % (src, csd)) h5_psrc_fit[src].write(this_time, **outdct) # Dump an individual file for this point source transit mlog.info('Dumping to disk single file for %s transit on CSD %d.' % (src, csd)) dump_dir = os.path.join(config.output_dir, 'point_source_gains') containers.mkdir(dump_dir) dump_file = os.path.join(dump_dir, '%s_csd_%d.h5' % (src.lower(), csd)) h5_psrc_fit[src].dump(dump_file, datasets=[ 'csd', 'acquisition', 'is_daytime', 'gain', 'weight', 'timing', 'model' ]) mlog.info('Finished analysis of %s transit on CSD %d.' % (src, csd))
def main(config_file=None, logging_params=DEFAULT_LOGGING): # Setup logging log.setup_logging(logging_params) mlog = log.get_logger(__name__) # Set config config = DEFAULTS.deepcopy() if config_file is not None: config.merge(NameSpace(load_yaml_config(config_file))) # Set niceness current_niceness = os.nice(0) os.nice(config.niceness - current_niceness) mlog.info('Changing process niceness from %d to %d. Confirm: %d' % (current_niceness, config.niceness, os.nice(0))) # Create output suffix output_suffix = config.output_suffix if config.output_suffix is not None else "jumps" # Calculate the wavelet transform for the following scales nwin = 2 * config.max_scale + 1 nhwin = nwin // 2 if config.log_scale: mlog.info("Using log scale.") scale = np.logspace(np.log10(config.min_scale), np.log10(nwin), num=config.num_points, dtype=np.int) else: mlog.info("Using linear scale.") scale = np.arange(config.min_scale, nwin, dtype=np.int) # Loop over acquisitions for acq in config.acq: # Find acquisition files all_data_files = sorted(glob(os.path.join(config.data_dir, acq, "*.h5"))) nfiles = len(all_data_files) if nfiles == 0: continue mlog.info("Now processing acquisition %s (%d files)" % (acq, nfiles)) # Determine list of feeds to examine dset = ['flags/inputs'] if config.use_input_flag else () rdr = andata.CorrData.from_acq_h5(all_data_files, datasets=dset, apply_gain=False, renormalize=False) inputmap = tools.get_correlator_inputs(ephemeris.unix_to_datetime(rdr.time[0]), correlator='chime') # Extract good inputs if config.use_input_flag: ifeed = np.flatnonzero((np.sum(rdr.flags['inputs'][:], axis=-1, dtype=np.int) / float(rdr.flags['inputs'].shape[-1])) > config.input_threshold) else: ifeed = np.array([ii for ii, inp in enumerate(inputmap) if tools.is_chime(inp)]) ninp = len(ifeed) mlog.info("Processing %d feeds." % ninp) # Create list of candidates cfreq, cinput, ctime, cindex = [], [], [], [] jump_flag, jump_time, jump_auto = [], [], [] ncandidate = 0 # Determine number of files to process at once if config.max_num_file is None: chunk_size = nfiles else: chunk_size = min(config.max_num_file, nfiles) # Loop over chunks of files for chnk, data_files in enumerate(chunks(all_data_files, chunk_size)): mlog.info("Now processing chunk %d (%d files)" % (chnk, len(data_files))) # Deteremine selections along the various axes rdr = andata.CorrData.from_acq_h5(data_files, datasets=()) auto_sel = np.array([ii for ii, pp in enumerate(rdr.prod) if pp[0] == pp[1]]) auto_sel = andata._convert_to_slice(auto_sel) if config.time_start is None: ind_start = 0 else: time_start = ephemeris.datetime_to_unix(datetime.datetime(*config.time_start)) ind_start = int(np.argmin(np.abs(rdr.time - time_start))) if config.time_stop is None: ind_stop = rdr.ntime else: time_stop = ephemeris.datetime_to_unix(datetime.datetime(*config.time_stop)) ind_stop = int(np.argmin(np.abs(rdr.time - time_stop))) if config.freq_physical is not None: if hasattr(config.freq_physical, '__iter__'): freq_physical = config.freq_physical else: freq_physical = [config.freq_physical] freq_sel = [np.argmin(np.abs(ff - rdr.freq)) for ff in freq_physical] freq_sel = andata._convert_to_slice(freq_sel) else: fstart = config.freq_start if config.freq_start is not None else 0 fstop = config.freq_stop if config.freq_stop is not None else rdr.freq.size freq_sel = slice(fstart, fstop) # Load autocorrelations t0 = time.time() data = andata.CorrData.from_acq_h5(data_files, datasets=['vis'], start=ind_start, stop=ind_stop, freq_sel=freq_sel, prod_sel=auto_sel, apply_gain=False, renormalize=False) mlog.info("Took %0.1f seconds to load autocorrelations." % (time.time() - t0,)) # If first chunk, save the frequencies that are being used if not chnk: all_freq = data.freq.copy() # If requested do not consider data during day or near bright source transits flag_quiet = np.ones(data.ntime, dtype=np.bool) if config.ignore_sun: flag_quiet &= ~transit_flag('sun', data.time, freq=np.min(data.freq), pol='X', nsig=1.0) if config.only_quiet: flag_quiet &= ~daytime_flag(data.time) for ss in ["CYG_A", "CAS_A", "TAU_A", "VIR_A"]: flag_quiet &= ~transit_flag(ss, data.time, freq=np.min(data.freq), pol='X', nsig=1.0) # Loop over frequencies for ff, freq in enumerate(data.freq): print_cnt = 0 mlog.info("FREQ %d (%0.2f MHz)" % (ff, freq)) auto = data.vis[ff, :, :].real fractional_auto = auto * tools.invert_no_zero(np.median(auto, axis=-1, keepdims=True)) - 1.0 # Loop over inputs for ii in ifeed: print_cnt += 1 do_print = not (print_cnt % 100) if do_print: mlog.info("INPUT %d" % ii) t0 = time.time() signal = fractional_auto[ii, :] # Perform wavelet transform coef, freqs = pywt.cwt(signal, scale, config.wavelet_name) if do_print: mlog.info("Took %0.1f seconds to perform wavelet transform." % (time.time() - t0,)) t0 = time.time() # Find local modulus maxima flg_mod_max, mod_max = mod_max_finder(scale, coef, threshold=config.thresh, search_span=config.search_span) if do_print: mlog.info("Took %0.1f seconds to find modulus maxima." % (time.time() - t0,)) t0 = time.time() # Find persisent modulus maxima across scales candidates, cmm, pdrift, start, stop, lbl = finger_finder(scale, flg_mod_max, mod_max, istart=max(config.min_rise - config.min_scale, 0), do_fill=False) if do_print: mlog.info("Took %0.1f seconds to find fingers." % (time.time() - t0,)) t0 = time.time() if candidates is None: continue # Cut bad candidates index_good_candidates = np.flatnonzero((scale[stop] >= config.max_scale) & flag_quiet[candidates[start, np.arange(start.size)]] & (pdrift <= config.psigma_max)) ngood = index_good_candidates.size if ngood == 0: continue mlog.info("Input %d has %d jumps" % (ii, ngood)) # Add remaining candidates to list ncandidate += ngood cfreq += [freq] * ngood cinput += [ii] * ngood for igc in index_good_candidates: icenter = candidates[start[igc], igc] cindex.append(icenter) ctime.append(data.time[icenter]) aa = max(0, icenter - nhwin) bb = min(data.ntime, icenter + nhwin + 1) ncut = bb - aa temp_var = np.zeros(nwin, dtype=np.bool) temp_var[0:ncut] = True jump_flag.append(temp_var) temp_var = np.zeros(nwin, dtype=data.time.dtype) temp_var[0:ncut] = data.time[aa:bb].copy() jump_time.append(temp_var) temp_var = np.zeros(nwin, dtype=auto.dtype) temp_var[0:ncut] = auto[ii, aa:bb].copy() jump_auto.append(temp_var) # Garbage collect del data gc.collect() # If we found any jumps, write them to a file. if ncandidate > 0: output_file = os.path.join(config.output_dir, "%s_%s.h5" % (acq, output_suffix)) mlog.info("Writing %d jumps to: %s" % (ncandidate, output_file)) # Write to output file with h5py.File(output_file, 'w') as handler: handler.attrs['files'] = all_data_files handler.attrs['chan_id'] = ifeed handler.attrs['freq'] = all_freq index_map = handler.create_group('index_map') index_map.create_dataset('jump', data=np.arange(ncandidate)) index_map.create_dataset('window', data=np.arange(nwin)) ax = np.array(['jump']) dset = handler.create_dataset('freq', data=np.array(cfreq)) dset.attrs['axis'] = ax dset = handler.create_dataset('input', data=np.array(cinput)) dset.attrs['axis'] = ax dset = handler.create_dataset('time', data=np.array(ctime)) dset.attrs['axis'] = ax dset = handler.create_dataset('time_index', data=np.array(cindex)) dset.attrs['axis'] = ax ax = np.array(['jump', 'window']) dset = handler.create_dataset('jump_flag', data=np.array(jump_flag)) dset.attrs['axis'] = ax dset = handler.create_dataset('jump_time', data=np.array(jump_time)) dset.attrs['axis'] = ax dset = handler.create_dataset('jump_auto', data=np.array(jump_auto)) dset.attrs['axis'] = ax else: mlog.info("No jumps found for %s acquisition." % acq)