def process(self, mg): fig_prefix = self.params['fig_name'] # distributed along feed mg.dataset_common_to_distributed('eigval', distributed_axis=1) mg.dataset_common_to_distributed('gain', distributed_axis=1) feed = mpiutil.scatter_array(mg.attrs['feed']) for idx, fd in enumerate(feed): plt.figure() plt.subplot(411) plt.imshow(mg['gain'].local_data[:, idx, 0, :].T.real, origin='lower') # xx plt.colorbar() plt.subplot(412) plt.imshow(mg['gain'].local_data[:, idx, 0, :].T.imag, origin='lower') # xx plt.colorbar() plt.subplot(413) plt.imshow(mg['gain'].local_data[:, idx, 1, :].T.real, origin='lower') # yy plt.colorbar() plt.subplot(414) plt.imshow(mg['gain'].local_data[:, idx, 1, :].T.imag, origin='lower') # yy plt.colorbar() fig_name = '%s_%d.png' % (fig_prefix, fd) fig_name = output_path(fig_name) plt.savefig(fig_name) plt.clf() return mg
def __init__(self, parameter_file_or_dict=None, feedback=2): super(OneAndOne, self).__init__(parameter_file_or_dict, feedback) self.input_files = input_path(format_list(self.params['input_files'])) self.output_files = output_path(format_list(self.params['output_files']), mkdir=False) self.iterable = self.params['iterable'] if self.iterable: self.iter_start = self.params['iter_start'] self.iter_step = self.params['iter_step'] self.iter_num = self.params['iter_num'] self._iter_cnt = 0 # inner iter counter # Inspect the `process` method to see how many arguments it takes. pro_argspec = inspect.getargspec(self.process) n_args = len(pro_argspec.args) - 1 if n_args > 1: msg = ("`process` method takes more than 1 argument, which is not" " allowed") raise PipelineConfigError(msg) if pro_argspec.varargs or pro_argspec.keywords or pro_argspec.defaults: msg = ("`process` method may not have variable length or optional" " arguments") raise PipelineConfigError(msg) if n_args == 0: self._no_input = True else: # n_args == 1 self._no_input = False if len(self._in) != n_args and len(self.input_files) == 0: msg = ("No data to iterate over. There are no 'in' keys and no 'input_files'") raise PipelineConfigError(msg)
def write_output(self, output): """Method for writing time ordered data output. """ exclude = self.params['exclude'] check_status = self.params['check_status'] write_hints = self.params['write_hints'] libver = self.params['libver'] chunk_vis = self.params['chunk_vis'] chunk_shape = self.params['chunk_shape'] chunk_size = self.params['chunk_size'] output_failed_continue = self.params['output_failed_continue'] tag_output_iter = self.params['tag_output_iter'] if self.iterable and tag_output_iter: output_files = output_path(self.output_files, relative=False, iteration=self.iteration) else: output_files = self.output_files try: output.to_files(output_files, exclude, check_status, write_hints, libver, chunk_vis, chunk_shape, chunk_size) except Exception as e: if output_failed_continue: msg = 'Process %d writing output to files failed...' % mpiutil.rank logger.warning(msg) traceback.print_exc(file=sys.stdout) else: raise e
def __init__(self, pipefile=None, feedback=2): # Read in the parameters. self.params, self.task_params = parse_ini.parse(pipefile, self.params_init, prefix=self.prefix, return_undeclared=True, feedback=feedback) self.tasks = self.params['tasks'] # timing the running if self.params['timing']: self.start_time = datetime.datetime.now() if mpiutil.rank0: print 'Start the pipeline at %s...' % self.start_time # set environment var os.environ['TL_OUTPUT'] = self.params['outdir'] + '/' # copy pipefile to outdir if required if self.params['copy']: base_name = path.basename(pipefile) dst_file = '%s/%s' % (self.params['outdir'], base_name) outdir = output_path(dst_file, relative=False, mkdir=True) if mpiutil.rank0: if self.params['overwrite']: shutil.copy2(pipefile, dst_file) else: if not path.exists(dst_file): shutil.copy2(pipefile, dst_file) else: base, ext = path.splitext(dst_file) for cnt in itertools.count(1): dst = '%s_%d%s' % (base, cnt, ext) # add cnt to file name if not path.exists(dst): shutil.copy2(pipefile, dst) break mpiutil.barrier()
def combine_results(self): output_combined = self.params['output_combined'] output_combined = output_path( output_combined, relative=not output_combined.startswith('/')) self.allocate_output(output_combined, 'w') self.df_out[-1]['mode_list'] = self.mode_list for _m in self.mode_list: mode_key = 'cleaned_%02dmode' % _m self.create_dataset(-1, mode_key, dset_shp=self.dset_shp, dset_info=self.map_info) self.create_dataset(-1, mode_key + '_weight', dset_shp=self.dset_shp, dset_info=self.map_info) mask = np.ones(self.dset_shp[:1]).astype('bool') for ii in range(len(self.output_files)): for key in self.df_out[ii][mode_key].keys(): self.df_out[-1][mode_key][:]\ += self.df_out[ii]['weight'][:]\ * self.df_out[ii]['%s/%s'%(mode_key, key)][:] self.df_out[-1][mode_key + '_weight'][:]\ += self.df_out[ii]['weight'][:] mask *= ~(self.df_out[ii]['mask'][:].astype('bool')) weight = self.df_out[-1][mode_key + '_weight'][:] weight[weight == 0] = np.inf self.df_out[-1][mode_key][:] /= weight self.df_out[-1][mode_key + '_mask'] = (~mask).astype('int')
def __init__(self, pipefile=None, feedback=2): # Read in the parameters. self.params, self.task_params = parse_ini.parse(pipefile, self.params_init, prefix=self.prefix, return_undeclared=True, feedback=feedback) self.tasks = self.params['tasks'] # set environment var os.environ['TL_OUTPUT'] = self.params['outdir'] + '/' # copy pipefile to outdir if required if self.params['copy']: base_name = path.basename(pipefile) dst_file = '%s/%s' % (self.params['outdir'], base_name) outdir = output_path(dst_file, relative=False, mkdir=True) if mpiutil.rank0: if self.params['overwrite']: shutil.copy2(pipefile, dst_file) else: if not path.exists(dst_file): shutil.copy2(pipefile, dst_file) else: base, ext = path.splitext(dst_file) for cnt in itertools.count(1): dst = '%s_%d%s' % (base, cnt, ext) # add cnt to file name if not path.exists(dst): shutil.copy2(pipefile, dst) break mpiutil.barrier()
def write_to_file_splitmock(self): output_prefix = '/%s_cube_%s' % (self.params['prefix'], self.params['scenario']) for outfile in self.outfiles_split: mock_idx = self.iter_list[self.iter] output_file = output_prefix + '_%03d_%s.h5' % (mock_idx, outfile) output_file = output_path(output_file, relative=True) #self.allocate_output(output_file, 'w') if outfile is 'optsim': map_key = 'delta' weight_key = 'separable' map_name = 'sim_map_optsim' weight_name = 'sel' else: map_key = self.params['map_tmp_key'] weight_key = self.params['map_tmp_weight'] map_name = 'sim_map_' + outfile weight_name = 'weight' map_pad = self.map_pad map_shp = self.map_tmp.shape _map_shp = (map_shp[0], map_shp[1] - 2 * map_pad, map_shp[2] - 2 * map_pad) map_slice = (slice(0, None), slice(map_pad, map_shp[1] - map_pad), slice(map_pad, map_shp[2] - map_pad)) with h5py.File(output_file, 'w') as f: d = f.create_dataset(map_key, _map_shp, dtype=self.map_tmp.dtype) _d = getattr(self, map_name) d[:] = _d[map_slice] for key, value in self.map_tmp.info.iteritems(): d.attrs[key] = repr(value) d.attrs['freq_delta'] = repr(self.map_tmp.info['freq_delta'] / 1.e6) d.attrs['freq_centre'] = repr( self.map_tmp.info['freq_centre'] / 1.e6) #d.attrs['ra_delta'] = repr(self.map_tmp.info['ra_delta']) #d.attrs['ra_centre'] = repr(self.map_tmp.info['ra_centre']) #d.attrs['dec_delta'] = repr(self.map_tmp.info['dec_delta']) #d.attrs['dec_centre'] = repr(self.map_tmp.info['dec_centre']) if hasattr(self, weight_name): #mask = #mask[mask!=0] = 1. #d[:] *= mask n = f.create_dataset(weight_key, _map_shp, dtype=self.map_tmp.dtype) n[:] = getattr(self, weight_name)[map_slice] for key, value in self.map_tmp.info.iteritems(): n.attrs[key] = repr(value) n.attrs['freq_delta'] = repr( self.map_tmp.info['freq_delta'] / 1.e6) n.attrs['freq_centre'] = repr( self.map_tmp.info['freq_centre'] / 1.e6)
def init_output(self): suffix = '_%s.h5'%self.params['data_sets'] output_file = self.output_files[0] output_file = output_path(output_file + suffix, relative = not output_file.startswith('/')) #if mpiutil.rank0: # self.df = h5py.File(output_file, mode='w') self.allocate_output(output_file, 'w')
def read_process_write(self, tstream): """Overwrite the method of superclass.""" if isinstance(tstream, timestream.Timestream): return self.process(tstream) else: ts_dir = output_path(self.params['ts_dir']) ts_name = self.params['ts_name'] if mpiutil.rank0: print 'Try to load tstream from %s/%s' % (ts_dir, ts_name) tstream = timestream.Timestream.load(ts_dir, ts_name) return self.process(tstream)
def write_output(self, output): """Method for writing time ordered data output. """ exclude = self.params['exclude'] check_status = self.params['check_status'] libver = self.params['libver'] tag_output_iter = self.params['tag_output_iter'] if self.iterable and tag_output_iter: output_files = output_path(self.output_files, relative=False, iteration=self.iteration) else: output_files = self.output_files output.to_files(output_files, exclude, check_status, libver)
def read_input(self): for input_file in self.input_files: print input_file self.open(input_file) self.map_tmp = al.make_vect(al.load_h5(self.df_in[0], 'dirty_map')) self.map_shp = self.map_tmp.shape for output_file in self.output_files: output_file = output_path(output_file, relative=not output_file.startswith('/')) self.allocate_output(output_file, 'w') self.create_dataset_like(-1, 'clean_map', self.map_tmp) self.create_dataset_like(-1, 'noise_diag', self.map_tmp) return 1
def write_output(self, output): norm = self.hitmap norm[norm==0] = np.inf self.avgmap /= norm nside = self.params['nside'] map_name = 'avgmap_nside%d_f%6.2fMHz.fits'%(nside, self.freq) map_name = output_path(map_name) hp.write_map(map_name, self.avgmap, coord='C', overwrite=True) self.avgmap[self.avgmap==0] = hp.UNSEEN hp.mollview(self.avgmap, title='Map @ %6.2f MHz'%self.freq, badcolor='none') hp.graticule(coord='C', dpar=30, dmer=30) plt.show()
def write_output(self, output): #super(PlotSpectrum, self).write_output(output) fig_prefix = self.params['output_files'][0] main_data = self.params['main_data'] ymin = self.params['ymin'] ymax = self.params['ymax'] axhh = self.axhh axvv = self.axvv fig = self.fig x_label = self.x_label axhh.set_xticklabels([]) #axhh.set_ylabel(r'${\rm frequency\, [GHz]\, HH}$') #axhh.set_ylabel('HH Polarization') #if xmin is None: xmin = x_axis[0] #if xmax is None: xmax = x_axis[-1] xmin = self.xmin xmax = self.xmax axhh.set_xlim(xmin=self.xmin, xmax=self.xmax) axhh.set_ylim(ymin=ymin, ymax=ymax) axhh.minorticks_on() axhh.tick_params(length=4, width=1, direction='in') axhh.tick_params(which='minor', length=2, width=1, direction='in') axhh.legend(title=self.params['legend_title']) axhh.set_ylabel('HH Polarization') #axvv.set_xlabel(r'$({\rm time} - {\rm UT}\quad %s\,) [{\rm hour}]$'%t_start) axvv.set_xlabel(x_label) #axvv.set_ylabel(r'${\rm frequency\, [GHz]\, VV}$') #axvv.set_ylabel('VV Polarization') axvv.set_xlim(xmin=self.xmin, xmax=self.xmax) axvv.set_ylim(ymin=ymin, ymax=ymax) axvv.minorticks_on() axvv.tick_params(length=4, width=1, direction='in') axvv.tick_params(which='minor', length=2, width=1, direction='in') axvv.set_ylabel('VV Polarization') if fig_prefix is not None: fig_name = '%s_%s_Spec.png' % (fig_prefix, main_data) fig_name = output_path(fig_name) plt.savefig(fig_name, formate='png') #, dpi=100)
def write_output(self, output): super(PlotPointingvsTime, self).write_output(output) fig_prefix = self.params['output_files'][0] main_data = self.params['main_data'] axhh = self.axhh axvv = self.axvv fig = self.fig axhh.set_ylabel('Azimuth [arcmin]') axvv.set_ylabel('Elevation [arcmin]') if fig_prefix is not None: fig_name = '%s_%s_AzEl.png' % (fig_prefix, main_data) fig_name = output_path(fig_name) plt.savefig(fig_name, formate='png') #, dpi=100)
def write_output(self, output): super(PlotVvsTime, self).write_output(output) fig_prefix = self.params['output_files'][0] main_data = self.params['main_data'] axhh = self.axhh axvv = self.axvv fig = self.fig axhh.set_ylabel('HH Polarization') axvv.set_ylabel('VV Polarization') if fig_prefix is not None: fig_name = '%s_%s_TS.png' % (fig_prefix, main_data) fig_name = output_path(fig_name) plt.savefig(fig_name, formate='png') #, dpi=100)
def read_input(self): for input_file in self.input_files: if mpiutil.rank0: logger.info('%s' % input_file) self.open(input_file) map_tmp = al.load_h5(self.df_in[0], 'dirty_map') self.map_tmp = al.make_vect(map_tmp, axis_names=map_tmp.info['axes']) self.map_shp = self.map_tmp.shape for output_file in self.output_files: output_file = output_path(output_file, relative=not output_file.startswith('/')) self.allocate_output(output_file, 'w') self.create_dataset_like(-1, 'clean_map', self.map_tmp) self.create_dataset_like(-1, 'noise_diag', self.map_tmp) self.create_dataset_like(-1, 'dirty_map', self.map_tmp) return 1
def init_output(self): output_file = self.output_files[0] output_file = output_path(output_file, relative= not output_file.startswith('/')) self.allocate_output(output_file, 'w') # load one file to get the ant_n and pol_n self.create_dataset('binavg_1d', self.dset_shp + (self.knum,)) self.create_dataset('counts_1d', self.dset_shp + (self.knum,)) self.create_dataset('binavg_2d', self.dset_shp + (self.knum_x, self.knum_y)) self.create_dataset('counts_2d', self.dset_shp + (self.knum_x, self.knum_y)) self.df['kbin'] = self.kbin self.df['kbin_x'] = self.kbin_x self.df['kbin_y'] = self.kbin_y self.df['kbin_edges'] = self.kbin_edges self.df['kbin_x_edges'] = self.kbin_x_edges self.df['kbin_y_edges'] = self.kbin_y_edges self.df['sim'] = self.params['sim']
def write_to_file(self): output_prefix = '/%s_cube_%s' % (self.params['prefix'], self.params['scenario']) for outfile in self.outfiles: mock_idx = self.iter_list[self.iter] output_file = output_prefix + '_%03d_%s.h5' % (mock_idx, outfile) output_file = output_path(output_file, relative=True) #self.allocate_output(output_file, 'w') if outfile is 'optsim': map_key = 'delta' weight_key = 'separable' map_name = 'sim_map_optsim' weight_name = 'sel' else: map_key = self.params['map_tmp_key'] weight_key = self.params['map_tmp_weight'] map_name = 'sim_map_' + outfile weight_name = 'weight' with h5py.File(output_file, 'w') as f: d = f.create_dataset(map_key, self.map_tmp.shape, dtype=self.map_tmp.dtype) d[:] = getattr(self, map_name) for key, value in self.map_tmp.info.iteritems(): d.attrs[key] = repr(value) if hasattr(self, weight_name): #mask = #mask[mask!=0] = 1. #d[:] *= mask n = f.create_dataset(weight_key, self.map_tmp.shape, dtype=self.map_tmp.dtype) n[:] = getattr(self, weight_name) for key, value in self.map_tmp.info.iteritems(): n.attrs[key] = repr(value)
def open_outputfiles(self): output_prefix = '/%s_cube_%s' % (self.params['prefix'], self.params['scenario']) output_file = output_prefix + '.h5' output_file = output_path(output_file, relative=True) self.allocate_output(output_file, 'w') dset_shp = (self.params['mock_n'], ) + self.map_tmp.shape dset_info = {} dset_info['axes'] = ('mock', 'freq', 'ra', 'dec') dset_info['type'] = 'vect' dset_info['mock_delta'] = 1 dset_info['mock_centre'] = self.params['mock_n'] // 2 dset_info['freq_delta'] = self.map_tmp.info['freq_delta'] / 1.e6 dset_info['freq_centre'] = self.map_tmp.info['freq_centre'] / 1.e6 dset_info['ra_delta'] = self.map_tmp.info['ra_delta'] dset_info['ra_centre'] = self.map_tmp.info['ra_centre'] dset_info['dec_delta'] = self.map_tmp.info['dec_delta'] dset_info['dec_centre'] = self.map_tmp.info['dec_centre'] #if self.params['outfile_raw']: for outfile in self.outfiles: self.create_dataset(outfile, dset_shp, dset_info)
def setup(self): super(SurveySimToMap, self).setup() params = self.params freq = params['freq'] self.n_freq = freq.shape[0] self.freq_spacing = freq[1] - freq[0] self.n_ra, self.n_dec = params['map_shape'] self.map_shp = (self.n_freq, self.n_ra, self.n_dec) self.spacing = params['pixel_spacing'] self.dec_spacing = self.spacing self.ra_spacing = self.spacing / sp.cos( params['field_centre'][1] * sp.pi / 180.) axis_names = ('freq', 'ra', 'dec') map_tmp = np.zeros(self.map_shp, dtype=__dtype__) map_tmp = al.make_vect(map_tmp, axis_names=axis_names) map_tmp.set_axis_info('freq', freq[self.n_freq // 2], self.freq_spacing) map_tmp.set_axis_info('ra', params['field_centre'][0], self.ra_spacing) map_tmp.set_axis_info('dec', params['field_centre'][1], self.dec_spacing) self.map_tmp = map_tmp mock_n = self.mock_n #for ii in range(mock_n): for ii in self.HI_mock_ids: output_file = 'sim_mock%03d_%s_%s_%s_%s.h5' % ( ii, self.params['prefix'], self.params['survey_mode'], self.params['HI_scenario'], self.params['HI_model_type']) output_file = output_path(output_file, relative=True) self.allocate_output(output_file, 'w') self.create_dataset_like(-1, 'dirty_map', map_tmp) self.create_dataset_like(-1, 'clean_map', map_tmp) self.create_dataset_like(-1, 'count_map', map_tmp)
def flag(self, vis, vis_mask, li, gi, bl, ts, **kwargs): """Function that does the actual flag.""" freq_window = self.params['freq_window'] time_window = self.params['time_window'] freq_sigma = self.params['freq_sigma'] time_sigma = self.params['time_sigma'] plot_fit = self.params['plot_fit'] freq_fig_prefix = self.params['freq_fig_name'] time_fig_prefix = self.params['time_fig_name'] tag_output_iter = self.params['tag_output_iter'] iteration = self.iteration freq_flag = kwargs.get('freq_flag') time_flag = kwargs.get('time_flag') time = ts.time[:] nt = len(time) freq = ts.freq[:] nfreq = len(freq) if isinstance(ts, Timestream): # for Timestream pol = bl[0] bl = tuple(bl[1]) elif isinstance(ts, RawTimestream): # for RawTimestream pol = None bl = tuple(bl) else: raise ValueError('Need either a RawTimestream or Timestream') if freq_flag: # time integration tm_vis = np.ma.mean(np.ma.array(vis, mask=vis_mask), axis=0) abs_vis = np.abs(tm_vis) if np.ma.count_masked(tm_vis) > 0: # has masked value abs_vis_valid = abs_vis[~abs_vis.mask] inds_valid = np.arange(nfreq)[~abs_vis.mask] itp = InterpolatedUnivariateSpline(inds_valid, abs_vis_valid) abs_vis_itp = itp(np.arange(nfreq)) abs_vis1 = abs_vis_itp.copy() else: abs_vis1 = abs_vis.copy() for cnt in xrange(10): if cnt != 0: abs_vis1[inds] = smooth[inds] smooth = savitzky_golay(abs_vis1, freq_window, 3) # flage RFI diff = abs_vis1 - smooth mean = np.mean(diff) std = np.std(diff) inds = np.where(np.abs(diff - mean) > freq_sigma*std)[0] if len(inds) == 0: break diff = abs_vis - smooth mean = np.mean(diff) std = np.std(diff) inds = np.where(np.abs(diff - mean) > freq_sigma*std)[0] # masked inds vis_mask[:, inds] = True # set mask if plot_fit: plt.figure() plt.plot(freq, abs_vis, label='data') plt.plot(freq[inds], abs_vis[inds], 'ro', label='flag') plt.plot(freq, smooth, label='smooth') plt.xlabel(r'$\nu$ / MHz') plt.legend(loc='best') if pol is None: fig_name = '%s_%d_%d.png' % (freq_fig_prefix, bl[0], bl[1]) else: fig_name = '%s_%d_%d_%s.png' % (freq_fig_prefix, bl[0], bl[1], pol) if tag_output_iter: fig_name = output_path(fig_name, iteration=iteration) else: fig_name = output_path(fig_name) plt.savefig(fig_name) plt.close() if time_flag: # freq integration fm_vis = np.ma.mean(np.ma.array(vis, mask=vis_mask), axis=1) abs_vis = np.abs(fm_vis) if np.ma.count_masked(fm_vis) > 0: # has masked value abs_vis_valid = abs_vis[~abs_vis.mask] inds_valid = np.arange(nt)[~abs_vis.mask] itp = InterpolatedUnivariateSpline(inds_valid, abs_vis_valid) abs_vis_itp = itp(np.arange(nt)) abs_vis1 = abs_vis_itp.copy() else: abs_vis1 = abs_vis.copy() for cnt in xrange(10): if cnt != 0: abs_vis1[inds] = smooth[inds] smooth = savitzky_golay(abs_vis1, time_window, 3) # flage RFI diff = abs_vis1 - smooth mean = np.mean(diff) std = np.std(diff) inds = np.where(np.abs(diff - mean) > time_sigma*std)[0] if len(inds) == 0: break diff = abs_vis - smooth mean = np.mean(diff) std = np.std(diff) inds = np.where(np.abs(diff - mean) > time_sigma*std)[0] # masked inds # Addtional threshold # inds1 = np.where(np.abs(diff[inds]) > 1.0e-2*np.abs(smooth[inds]))[0] # inds = inds[inds1] vis_mask[inds] = True # set mask if plot_fit: plt.figure() plt.plot(time, abs_vis, label='data') plt.plot(time[inds], abs_vis[inds], 'ro', label='flag') plt.plot(time, smooth, label='smooth') plt.xlabel(r'$t$ / Julian Date') plt.legend(loc='best') if pol is None: fig_name = '%s_%d_%d.png' % (time_fig_prefix, bl[0], bl[1]) else: fig_name = '%s_%d_%d_%s.png' % (time_fig_prefix, bl[0], bl[1], pol) if tag_output_iter: fig_name = output_path(fig_name, iteration=iteration) else: fig_name = output_path(fig_name) plt.savefig(fig_name) plt.close() return vis, vis_mask
def plot(self, vis, vis_mask, li, gi, bl, ts, **kwargs): """Function that does the actual plot work.""" plot_type = self.params['plot_type'] bl_incl = self.params['bl_incl'] bl_excl = self.params['bl_excl'] flag_mask = self.params['flag_mask'] flag_ns = self.params['flag_ns'] slices = self.params['slices'] fig_prefix = self.params['fig_name'] rotate_xdate = self.params['rotate_xdate'] feed_no = self.params['feed_no'] tag_output_iter = self.params['tag_output_iter'] iteration = self.iteration if isinstance(ts, Timestream): # for Timestream pol = bl[0] bl = tuple(bl[1]) feed_no = True elif isinstance(ts, RawTimestream): # for RawTimestream pol = None bl = tuple(bl) if feed_no: pol = ts['bl_pol'].local_data[li] bl = tuple(ts['true_blorder'].local_data[li]) else: raise ValueError('Need either a RawTimestream or Timestream') if bl_incl != 'all': bl1 = set(bl) bl_incl = [{f1, f2} for (f1, f2) in bl_incl] bl_excl = [{f1, f2} for (f1, f2) in bl_excl] if (not bl1 in bl_incl) or (bl1 in bl_excl): return if plot_type == 'time': nt = vis.shape[0] c = nt / 2 s = max(0, c - slices / 2) e = min(nt, s + slices) if flag_mask: vis1 = np.ma.array(vis[s:e], mask=vis_mask[s:e]) elif flag_ns: vis1 = vis[s:e].copy() if 'ns_on' in ts.iterkeys(): ns_on = ts['ns_on'][s:e] on = np.where(ns_on)[0] vis1[on] = complex(np.nan, np.nan) else: vis1 = vis[s:e] o = c - s shift = 0.1 * np.ma.max(np.abs(vis1[o])) ax_val = ts.freq[:] xlabel = r'$\nu$ / MHz' elif plot_type == 'freq': nfreq = vis.shape[1] c = nfreq / 2 s = max(0, c - slices / 2) e = min(nfreq, s + slices) if flag_mask: vis1 = np.ma.array(vis[:, s:e], mask=vis_mask[:, s:e]) elif flag_ns: vis1 = vis[:, s:e].copy() if 'ns_on' in ts.iterkeys(): ns_on = ts['ns_on'][:] on = np.where(ns_on)[0] vis1[on] = complex(np.nan, np.nan) else: vis1 = vis[:, s:e] o = c - s shift = 0.1 * np.ma.max(np.abs(vis1[:, o])) # ax_val = ts.time[:] # xlabel = r'$t$ / Julian Date' ax_val = np.array( [datetime.fromtimestamp(sec) for sec in ts['sec1970'][:]]) xlabel = '%s' % ax_val[0].date() ax_val = mdates.date2num(ax_val) else: raise ValueError( 'Unknown plot_type %s, must be either time or freq' % plot_type) plt.figure() f, axarr = plt.subplots(3, sharex=True) for i in range(e - s): if plot_type == 'time': axarr[0].plot(ax_val, vis1[i].real + (i - o) * shift, label='real') elif plot_type == 'freq': axarr[0].plot(ax_val, vis1[:, i].real + (i - o) * shift, label='real') if i == 0: axarr[0].legend() if plot_type == 'time': axarr[1].plot(ax_val, vis1[i].imag + (i - o) * shift, label='imag') elif plot_type == 'freq': axarr[1].plot(ax_val, vis1[:, i].imag + (i - o) * shift, label='imag') if i == 0: axarr[1].legend() if plot_type == 'time': axarr[2].plot(ax_val, np.abs(vis1[i]) + (i - o) * shift, label='abs') elif plot_type == 'freq': axarr[2].plot(ax_val, np.abs(vis1[:, i]) + (i - o) * shift, label='abs') if i == 0: axarr[2].legend() if plot_type == 'freq': axarr[2].xaxis_date() date_format = mdates.DateFormatter('%H:%M') axarr[2].xaxis.set_major_formatter(date_format) if rotate_xdate: # set the x-axis tick labels to diagonal so it fits better f.autofmt_xdate() else: # half the number of date ticks so they do not overlap # axarr[2].set_xticks(axarr[2].get_xticks()[::2]) # reduce the number of tick locators locator = MaxNLocator(nbins=6) axarr[2].xaxis.set_major_locator(locator) axarr[2].xaxis.set_minor_locator(AutoMinorLocator(2)) axarr[2].set_xlabel(xlabel) if feed_no: fig_name = '%s_%s_%d_%d_%s.png' % (fig_prefix, plot_type, bl[0], bl[1], ts.pol_dict[pol]) else: fig_name = '%s_%s_%d_%d.png' % (fig_prefix, plot_type, bl[0], bl[1]) if tag_output_iter: fig_name = output_path(fig_name, iteration=iteration) else: fig_name = output_path(fig_name) plt.savefig(fig_name) plt.close()
def process(self, ts): mask_daytime = self.params['mask_daytime'] mask_time_range = self.params['mask_time_range'] tsys = self.params['tsys'] accuracy_boost = self.params['accuracy_boost'] l_boost = self.params['l_boost'] bl_range = self.params['bl_range'] auto_correlations = self.params['auto_correlations'] time_avg = self.params['time_avg'] pol = self.params['pol'] interp = self.params['interp'] beam_dir = output_path(self.params['beam_dir']) use_existed_beam = self.params['use_existed_beam'] gen_inv = self.params['gen_invbeam'] noise_weight = self.params['noise_weight'] ts_dir = output_path(self.params['ts_dir']) ts_name = self.params['ts_name'] no_m_zero = self.params['no_m_zero'] simulate = self.params['simulate'] input_maps = self.params['input_maps'] prior_map = self.params['prior_map'] add_noise = self.params['add_noise'] dirty_map = self.params['dirty_map'] nbin = self.params['nbin'] method = self.params['method'] normalize = self.params['normalize'] threshold = self.params['threshold'] eps = self.params['epsilon'] correct_order = self.params['correct_order'] if use_existed_beam: # load the saved telescope from disk tel = None else: assert isinstance(ts, Timestream), '%s only works for Timestream object' % self.__class__.__name__ ts.redistribute('baseline') lat = ts.attrs['sitelat'] # lon = ts.attrs['sitelon'] lon = 0.0 # lon = np.degrees(ts['ra_dec'][0, 0]) # the first ra local_origin = False freqs = ts.freq[:] # MHz nfreq = freqs.shape[0] band_width = ts.attrs['freqstep'] # MHz try: ndays = ts.attrs['ndays'] except KeyError: ndays = 1 feeds = ts['feedno'][:] bl_order = mpiutil.gather_array(ts.local_bl, axis=0, root=None, comm=ts.comm) bls = [ tuple(bl) for bl in bl_order ] az, alt = ts['az_alt'][0] az = np.degrees(az) alt = np.degrees(alt) pointing = [az, alt, 0.0] feedpos = ts['feedpos'][:] if ts.is_dish: from tlpipe.map.drift.telescope import tl_dish dish_width = ts.attrs['dishdiam'] tel = tl_dish.TlUnpolarisedDishArray(lat, lon, freqs, band_width, tsys, ndays, accuracy_boost, l_boost, bl_range, auto_correlations, local_origin, dish_width, feedpos, pointing) elif ts.is_cylinder: from tlpipe.map.drift.telescope import tl_cylinder # factor = 1.2 # suppose an illumination efficiency, keep same with that in timestream_common factor = 0.79 # for xx # factor = 0.88 # for yy cyl_width = factor * ts.attrs['cywid'] tel = tl_cylinder.TlUnpolarisedCylinder(lat, lon, freqs, band_width, tsys, ndays, accuracy_boost, l_boost, bl_range, auto_correlations, local_origin, cyl_width, feedpos) else: raise RuntimeError('Unknown array type %s' % ts.attrs['telescope']) if not simulate: # select the corresponding vis and vis_mask if pol == 'xx': local_vis = ts.local_vis[:, :, 0, :] local_vis_mask = ts.local_vis_mask[:, :, 0, :] elif pol == 'yy': local_vis = ts.local_vis[:, :, 1, :] local_vis_mask = ts.local_vis_mask[:, :, 1, :] elif pol == 'I': xx_vis = ts.local_vis[:, :, 0, :] xx_vis_mask = ts.local_vis_mask[:, :, 0, :] yy_vis = ts.local_vis[:, :, 1, :] yy_vis_mask = ts.local_vis_mask[:, :, 1, :] local_vis = np.zeros_like(xx_vis) for ti in xrange(local_vis.shape[0]): for fi in xrange(local_vis.shape[1]): for bi in xrange(local_vis.shape[2]): if xx_vis_mask[ti, fi, bi] != yy_vis_mask[ti, fi, bi]: if xx_vis_mask[ti, fi, bi]: local_vis[ti, fi, bi] = yy_vis[ti, fi, bi] else: local_vis[ti, fi, bi] = xx_vis[ti, fi, bi] else: local_vis[ti, fi, bi] = 0.5 * (xx_vis[ti, fi, bi] + yy_vis[ti, fi, bi]) local_vis_mask = xx_vis_mask | yy_vis_mask else: raise ValueError('Invalid pol: %s' % pol) if interp != 'none': for fi in xrange(local_vis.shape[1]): for bi in xrange(local_vis.shape[2]): # interpolate for local_vis true_inds = np.where(local_vis_mask[:, fi, bi])[0] # masked inds if len(true_inds) > 0: false_inds = np.where(~local_vis_mask[:, fi, bi])[0] # un-masked inds if len(false_inds) > 0.1 * local_vis.shape[0]: # nearest interpolate for local_vis if interp in ('linear', 'nearest'): itp_real = interp1d(false_inds, local_vis[false_inds, fi, bi].real, kind=interp, fill_value='extrapolate', assume_sorted=True) itp_imag = interp1d(false_inds, local_vis[false_inds, fi, bi].imag, kind=interp, fill_value='extrapolate', assume_sorted=True) elif interp == 'rbf': itp_real = Rbf(false_inds, local_vis[false_inds, fi, bi].real, smooth=10) itp_imag = Rbf(false_inds, local_vis[false_inds, fi, bi].imag, smooth=10) else: raise ValueError('Unknown interpolation method: %s' % interp) local_vis[true_inds, fi, bi] = itp_real(true_inds) + 1.0J * itp_imag(true_inds) # the interpolated vis else: local_vis[:, fi, bi] = 0 # TODO: may need to take special care # average data nt = ts['sec1970'].shape[0] phi_size = 2*tel.mmax + 1 # phi = np.zeros((phi_size,), dtype=ts['ra_dec'].dtype) phi = np.linspace(0, 2*np.pi, phi_size, endpoint=False) vis = np.zeros((phi_size,)+local_vis.shape[1:], dtype=local_vis.dtype) if time_avg == 'avg': nt_m = float(nt) / phi_size # roll data to have phi=0 near the first roll_len = np.int(np.around(0.5*nt_m)) local_vis[:] = np.roll(local_vis[:], roll_len, axis=0) if interp == 'none': local_vis_mask[:] = np.roll(local_vis_mask[:], roll_len, axis=0) # ts['ra_dec'][:] = np.roll(ts['ra_dec'][:], roll_len, axis=0) repeat_inds = np.repeat(np.arange(nt), phi_size) num, start, end = mpiutil.split_m(nt*phi_size, phi_size) # average over time for idx in xrange(phi_size): inds, weight = unique(repeat_inds[start[idx]:end[idx]], return_counts=True) if interp == 'none': vis[idx] = average(np.ma.array(local_vis[inds], mask=local_vis_mask[inds]), axis=0, weights=weight) # time mean else: vis[idx] = average(local_vis[inds], axis=0, weights=weight) # time mean # phi[idx] = np.average(ts['ra_dec'][:, 0][inds], axis=0, weights=weight) elif time_avg == 'fft': if interp == 'none': raise ValueError('Can not do fft average without first interpolation') Vm = np.fft.fftshift(np.fft.fft(local_vis, axis=0), axes=0) vis[:] = np.fft.ifft(np.fft.ifftshift(Vm[nt/2-tel.mmax:nt/2+tel.mmax+1], axes=0), axis=0) / (1.0 * nt / phi_size) # for fi in xrange(vis.shape[1]): # for bi in xrange(vis.shape[2]): # # plot local_vis and vis # import matplotlib # matplotlib.use('Agg') # import matplotlib.pyplot as plt # phi0 = np.linspace(0, 2*np.pi, nt, endpoint=False) # phi1 = np.linspace(0, 2*np.pi, phi_size, endpoint=False) # plt.figure() # plt.subplot(211) # plt.plot(phi0, local_vis[:, fi, bi].real, label='v0.real') # plt.plot(phi1, vis[:, fi, bi].real, label='v1.real') # plt.legend() # plt.subplot(212) # plt.plot(phi0, local_vis[:, fi, bi].imag, label='v0.imag') # plt.plot(phi1, vis[:, fi, bi].imag, label='v1.imag') # plt.legend() # plt.savefig('vis_fft/vis_%d_%d.png' % (fi, bi)) # plt.close() else: raise ValueError('Unknown time_avg: %s' % time_avg) del local_vis del local_vis_mask # mask daytime data if mask_daytime: day_or_night = np.where(ts['local_hour'][:]>=mask_time_range[0] & ts['local_hour'][:]<=mask_time_range[1], True, False) day_inds = np.where(np.repeat(day_or_night, phi_size).reshape(nt, phi_size).astype(np.int).sum(axis=1).astype(bool))[0] vis[day_inds] = 0 del ts # no longer need ts # redistribute vis to time axis vis = mpiarray.MPIArray.wrap(vis, axis=2).redistribute(0).local_array allpairs = tel.allpairs redundancy = tel.redundancy nrd = len(redundancy) # reorder bls according to allpairs vis_tmp = np.zeros_like(vis) for ind, (a1, a2) in enumerate(allpairs): try: b_ind = bls.index((feeds[a1], feeds[a2])) vis_tmp[:, :, ind] = vis[:, :, b_ind] except ValueError: b_ind = bls.index((feeds[a2], feeds[a1])) vis_tmp[:, :, ind] = vis[:, :, b_ind].conj() del vis # average over redundancy vis_stream = np.zeros(vis_tmp.shape[:-1]+(nrd,), dtype=vis_tmp.dtype) red_bin = np.cumsum(np.insert(redundancy, 0, 0)) # redundancy bin # average over redundancy for ind in xrange(nrd): vis_stream[:, :, ind] = np.sum(vis_tmp[:, :, red_bin[ind]:red_bin[ind+1]], axis=2) / redundancy[ind] del vis_tmp # beamtransfer bt = beamtransfer.BeamTransfer(beam_dir, tel, noise_weight, True) if not use_existed_beam: bt.generate() if tel is None: tel = bt.telescope if simulate: ndays = 733 tstream = timestream.simulate(bt, ts_dir, ts_name, input_maps, ndays, add_noise=add_noise) else: # timestream and map-making tstream = timestream.Timestream(ts_dir, ts_name, bt, no_m_zero) parent_path = os.path.dirname(tstream._fdir(0)) if os.path.exists(parent_path + '/COMPLETED'): if mpiutil.rank0: print 'Use existed timestream_f files in %s' % parent_path else: for fi in mpiutil.mpirange(nfreq): # Make directory if required if not os.path.exists(tstream._fdir(fi)): os.makedirs(tstream._fdir(fi)) # create memh5 object and write data to temporary file vis_h5 = memh5.MemGroup(distributed=True) vis_h5.create_dataset('/timestream', data=mpiarray.MPIArray.wrap(vis_stream, axis=0)) tmp_file = parent_path +'/vis_stream_temp.hdf5' vis_h5.to_hdf5(tmp_file, hints=False) del vis_h5 # re-organize data as need for tstream # make load even among nodes for fi in mpiutil.mpirange(nfreq, method='rand'): # read the needed data from the temporary file with h5py.File(tmp_file, 'r') as f: vis_fi = f['/timestream'][:, fi, :] # Write file contents with h5py.File(tstream._ffile(fi), 'w') as f: # Timestream data # allocate space for vis_stream shp = (nrd, phi_size) f.create_dataset('/timestream', data=vis_fi.T) f.create_dataset('/phi', data=phi) # Telescope layout data f.create_dataset('/feedmap', data=tel.feedmap) f.create_dataset('/feedconj', data=tel.feedconj) f.create_dataset('/feedmask', data=tel.feedmask) f.create_dataset('/uniquepairs', data=tel.uniquepairs) f.create_dataset('/baselines', data=tel.baselines) # Telescope frequencies f.create_dataset('/frequencies', data=freqs) # Write metadata f.attrs['beamtransfer_path'] = os.path.abspath(bt.directory) f.attrs['ntime'] = phi_size mpiutil.barrier() # remove temp file if mpiutil.rank0: os.remove(tmp_file) # mark all frequencies tstream files are saved correctly open(parent_path + '/COMPLETED', 'a').close() tstream.generate_mmodes() nside = hputil.nside_for_lmax(tel.lmax, accuracy_boost=tel.accuracy_boost) if dirty_map: tstream.mapmake_full(nside, 'map_full_dirty.hdf5', nbin, dirty=True, method=method, normalize=normalize, threshold=threshold) else: tstream.mapmake_full(nside, 'map_full.hdf5', nbin, dirty=False, method=method, normalize=normalize, threshold=threshold, eps=eps, correct_order=correct_order, prior_map_file=prior_map) # ts.add_history(self.history) return tstream
def process(self, ts): assert isinstance(ts, Timestream), '%s only works for Timestream object' % self.__class__.__name__ mask_daytime = self.params['mask_daytime'] mask_time_range = self.params['mask_time_range'] beam_theta_range = self.params['beam_theta_range'] tsys = self.params['tsys'] accuracy_boost = self.params['accuracy_boost'] l_boost = self.params['l_boost'] bl_range = self.params['bl_range'] auto_correlations = self.params['auto_correlations'] pol = self.params['pol'] beam_dir = output_path(self.params['beam_dir']) gen_inv = self.params['gen_invbeam'] noise_weight = self.params['noise_weight'] ts_dir = output_path(self.params['ts_dir']) ts_name = self.params['ts_name'] simulate = self.params['simulate'] input_maps = self.params['input_maps'] add_noise = self.params['add_noise'] dirty_map = self.params['dirty_map'] nbin = self.params['nbin'] method = self.params['method'] normalize = self.params['normalize'] threshold = self.params['threshold'] ts.redistribute('frequency') lat = ts.attrs['sitelat'] # lon = ts.attrs['sitelon'] lon = 0.0 # lon = np.degrees(ts['ra_dec'][0, 0]) # the first ra local_origin = False freq = ts.freq freqs = ts.freq.data.to_numpy_array(root=None) # MHz band_width = ts.attrs['freqstep'] # MHz ndays = 1 feeds = ts['feedno'][:] az, alt = ts['az_alt'][0] az = np.degrees(az) alt = np.degrees(alt) pointing = [az, alt, 0.0] feedpos = ts['feedpos'][:] if ts.is_dish: dish_width = ts.attrs['dishdiam'] tel = tl_dish.TlUnpolarisedDishArray(lat, lon, freqs, band_width, tsys, ndays, accuracy_boost, l_boost, bl_range, auto_correlations, local_origin, dish_width, feedpos, pointing) elif ts.is_cylinder: factor = 1.2 # suppose an illumination efficiency, keep same with that in timestream_common cyl_width = factor * ts.attrs['cywid'] tel = tl_cylinder.TlUnpolarisedCylinder(lat, lon, freqs, band_width, tsys, ndays, accuracy_boost, l_boost, bl_range, auto_correlations, local_origin, cyl_width, feedpos) else: raise RuntimeError('Unknown array type %s' % ts.attrs['telescope']) if not simulate: # mask daytime data if mask_daytime: day_inds = np.where(np.logical_and(ts['local_hour'][:]>=mask_time_range[0], ts['local_hour'][:]<=mask_time_range[1]))[0] ts.local_vis_mask[day_inds] = True # do not change vis directly # average data nt = ts['sec1970'].shape[0] phi_size = 2*tel.mmax + 1 nt_m = float(nt) / phi_size # roll data to have phi=0 near the first roll_len = np.int(np.around(0.5*nt_m)) ts.local_vis[:] = np.roll(ts.local_vis[:], roll_len, axis=0) ts.local_vis_mask[:] = np.roll(ts.local_vis_mask[:], roll_len, axis=0) ts['ra_dec'][:] = np.roll(ts['ra_dec'][:], roll_len, axis=0) repeat_inds = np.repeat(np.arange(nt), phi_size) num, start, end = mpiutil.split_m(nt*phi_size, phi_size) # phi = np.zeros((phi_size,), dtype=ts['ra_dec'].dtype) phi = np.linspace(0, 2*np.pi, phi_size, endpoint=False) vis = np.zeros((phi_size,)+ts.local_vis.shape[1:], dtype=ts.vis.dtype) # average over time for idx in xrange(phi_size): inds, weight = unique(repeat_inds[start[idx]:end[idx]], return_counts=True) vis[idx] = average(np.ma.array(ts.local_vis[inds], mask=ts.local_vis_mask[inds]), axis=0, weights=weight) # time mean # phi[idx] = np.average(ts['ra_dec'][:, 0][inds], axis=0, weights=weight) if pol == 'xx': vis = vis[:, :, 0, :] elif pol == 'yy': vis = vis[:, :, 1, :] elif pol == 'I': vis = 0.5 * (vis[:, :, 0, :] + vis[:, :, 1, :]) elif pol == 'all': vis = np.sum(vis, axis=2) # sum over all pol else: raise ValueError('Invalid pol: %s' % pol) allpairs = tel.allpairs redundancy = tel.redundancy # reorder bls according to allpairs vis_tmp = np.zeros_like(vis) bls = [ tuple(bl) for bl in ts['blorder'][:] ] for ind, (a1, a2) in enumerate(allpairs): try: b_ind = bls.index((feeds[a1], feeds[a2])) vis_tmp[:, :, ind] = vis[:, :, b_ind] except ValueError: b_ind = bls.index((feeds[a2], feeds[a1])) vis_tmp[:, :, ind] = vis[:, :, b_ind].conj() # average over redundancy vis_stream = np.zeros(vis.shape[:-1]+(len(redundancy),), dtype=vis_tmp.dtype) red_bin = np.cumsum(np.insert(redundancy, 0, 0)) # redundancy bin # average over redundancy for ind in xrange(len(redundancy)): vis_stream[:, :, ind] = np.sum(vis_tmp[:, :, red_bin[ind]:red_bin[ind+1]], axis=2) / redundancy[ind] del vis del vis_tmp # beamtransfer bt = beamtransfer.BeamTransfer(beam_dir, tel, noise_weight, True) bt.generate() if simulate: ndays = 733 print ndays tstream = timestream.simulate(bt, ts_dir, ts_name, input_maps, ndays, add_noise=add_noise) else: # timestream and map-making tstream = timestream.Timestream(ts_dir, ts_name, bt) for lfi, fi in freq.data.enumerate(axis=0): # Make directory if required if not os.path.exists(tstream._fdir(fi)): os.makedirs(tstream._fdir(fi)) # Write file contents with h5py.File(tstream._ffile(fi), 'w') as f: # Timestream data f.create_dataset('/timestream', data=vis_stream[:, lfi].T) f.create_dataset('/phi', data=phi) # Telescope layout data f.create_dataset('/feedmap', data=tel.feedmap) f.create_dataset('/feedconj', data=tel.feedconj) f.create_dataset('/feedmask', data=tel.feedmask) f.create_dataset('/uniquepairs', data=tel.uniquepairs) f.create_dataset('/baselines', data=tel.baselines) # Telescope frequencies f.create_dataset('/frequencies', data=freqs) # Write metadata f.attrs['beamtransfer_path'] = os.path.abspath(bt.directory) f.attrs['ntime'] = phi_size mpiutil.barrier() tstream.generate_mmodes() nside = hputil.nside_for_lmax(tel.lmax, accuracy_boost=tel.accuracy_boost) if dirty_map: tstream.mapmake_full(nside, 'map_full_dirty.hdf5', nbin, dirty=True, method=method, normalize=normalize, threshold=threshold) else: tstream.mapmake_full(nside, 'map_full.hdf5', nbin, dirty=False, method=method, normalize=normalize, threshold=threshold) # ts.add_history(self.history) return tstream
def process(self, ts): calibrator = self.params['calibrator'] catalog = self.params['catalog'] file_prefix = self.params['file_name'] plot_closure = self.params['plot_closure'] fig_prefix = self.params['fig_name'] bins = self.params['bins'] gauss_fit = self.params['gauss_fit'] tag_output_iter = self.params['tag_output_iter'] ts.redistribute('frequency') nfreq = len(ts.local_freq[:]) # local nfreq feedno = ts['feedno'][:].tolist() pol = ts['pol'][:].tolist() bl = ts.local_bl[:] # local bls bls = [ tuple(b) for b in bl ] # calibrator srclist, cutoff, catalogs = a.scripting.parse_srcs(calibrator, catalog) cat = a.src.get_catalog(srclist, cutoff, catalogs) assert(len(cat) == 1), 'Allow only one calibrator' s = cat.values()[0] if mpiutil.rank0: print 'Calibrating for source %s with' % calibrator, print 'strength', s._jys, 'Jy', print 'measured at', s.mfreq, 'GHz', print 'with index', s.index ra = ts['ra_dec'][:, 0] ra = np.unwrap(ra) ra = ra[np.logical_not(ts['ns_on'][:])] # get only ns_off values abs_diff = np.abs(np.diff(s._ra - ra)) ind1 = np.argmin(abs_diff) print 'ind1:', ind1 for pi in [ pol.index('xx'), pol.index('yy') ]: # xx and yy if nfreq > 0: # skip empty processes # find the ind that not be all masked for i in xrange(20): if not ts.local_vis_mask[ind1+i, :, pi].all(): ind = ind1 + i break if not ts.local_vis_mask[ind1-i, :, pi].all(): ind = ind1 - i break else: raise RuntimeError('vis is masked during this period for pol %s' % pol[pi]) print 'ind:', ind for fi in xrange(nfreq): gfi = fi + ts.freq.local_offset[0] # global freq index vis = ts.local_vis[ind, fi, pi, :] # only I vis_mask = ts.local_vis_mask[ind, fi, pi, :] # only I closure = [] cache = dict() for i, j, k in itertools.combinations(feedno, 3): bi, vij = mk_cache(cache, (i, j), bls, vis) if vis_mask[bi]: continue bi, vjk = mk_cache(cache, (j, k), bls, vis) if vis_mask[bi]: continue bi, vki = mk_cache(cache, (k, i), bls, vis) if vis_mask[bi]: continue # closure.append(np.angle(vij, True) + np.angle(vjk, True) + np.angle(vki, True)) # in degree, have 360 deg problem c = lambda x: np.complex128(x) # complex128 to avoid overflow in the product ang = np.angle(c(vij) * c(vjk) * c(vki), True) # in degree closure.append(ang) # in degree # save closure phase to file file_name = '%s_%d_%s.hdf5' % (file_prefix, gfi, pol[pi]) if tag_output_iter: file_name = output_path(file_name, iteration=self.iteration) else: file_name = output_path(file_name) with h5py.File(file_name, 'w') as f: f.create_dataset('closure_phase', data=np.array(closure)) if plot_closure: # plot all closure phase plt.figure() plt.plot(closure, 'o') fig_name = '%s_all_%d_%s.png' % (fig_prefix, gfi, pol[pi]) if tag_output_iter: fig_name = output_path(fig_name, iteration=self.iteration) else: fig_name = output_path(fig_name) plt.savefig(fig_name) plt.close() # plot histogram of closure phase # histogram plt.figure() data = plt.hist(closure, bins=bins) plt.xlabel('Closure phase / degree') if gauss_fit: # Generate data from bins as a set of points x = [0.5 * (data[1][i] + data[1][i+1]) for i in xrange(len(data[1])-1)] y = data[0] popt, pcov = optimize.curve_fit(f, x, y) A, b, c = popt xmax = max(abs(x[0]), abs(x[-1])) x_fit = np.linspace(-xmax, xmax, bins) y_fit = f(x_fit, *popt) lable = r'$a \, \exp{(- \frac{(x - \mu)^2} {2 \sigma^2})}$' + '\n\n' + r'$a = %f$' % A + '\n' + r'$\mu = %f$' % b + '\n' + r'$\sigma = %f$' % np.abs(c) plt.plot(x_fit, y_fit, lw=2, color="r", label=lable) plt.xlim(-xmax, xmax) plt.legend() fig_name = '%s_hist_%d_%s.png' % (fig_prefix, gfi, pol[pi]) if tag_output_iter: fig_name = output_path(fig_name, iteration=self.iteration) else: fig_name = output_path(fig_name) plt.savefig(fig_name) plt.close() mpiutil.barrier() ts.add_history(self.history) return ts
def process(self, ts): calibrator = self.params['calibrator'] catalog = self.params['catalog'] span = self.params['span'] save_gain = self.params['save_gain'] gain_file = self.params['gain_file'] ts.redistribute('frequency') lfreq = ts.local_freq[:] # local freq feedno = ts['feedno'][:].tolist() pol = ts['pol'][:].tolist() bl = ts.bl[:] bls = [ tuple(b) for b in bl ] # # antpointing = np.radians(ts['antpointing'][-1, :, :]) # radians # transitsource = ts['transitsource'][:] # transit_time = transitsource[-1, 0] # second, sec1970 # int_time = ts.attrs['inttime'] # second # calibrator srclist, cutoff, catalogs = a.scripting.parse_srcs(calibrator, catalog) cat = a.src.get_catalog(srclist, cutoff, catalogs) assert(len(cat) == 1), 'Allow only one calibrator' s = cat.values()[0] if mpiutil.rank0: print 'Calibrating for source %s with' % calibrator, print 'strength', s._jys, 'Jy', print 'measured at', s.mfreq, 'GHz', print 'with index', s.index # get transit time of calibrator # array aa = ts.array aa.set_jultime(ts['jul_date'][0]) # the first obs time point next_transit = aa.next_transit(s) transit_time = a.phs.ephem2juldate(next_transit) # Julian date if transit_time > ts['jul_date'][-1]: local_next_transit = ephem.Date(next_transit + 8.0 * ephem.hour) raise RuntimeError('Data does not contain local transit time %s of source %s' % (local_next_transit, calibrator)) # the first transit index transit_inds = [ np.searchsorted(ts['jul_date'][:], transit_time) ] # find all other transit indices aa.set_jultime(ts['jul_date'][0] + 1.0) transit_time = a.phs.ephem2juldate(aa.next_transit(s)) # Julian date cnt = 2 while(transit_time <= ts['jul_date'][-1]): transit_inds.append(np.searchsorted(ts['jul_date'][:], transit_time)) aa.set_jultime(ts['jul_date'][0] + 1.0*cnt) transit_time = a.phs.ephem2juldate(aa.next_transit(s)) # Julian date cnt += 1 print transit_inds ### now only use the first transit point to do the cal ### may need to improve in the future transit_ind = transit_inds[0] int_time = ts.attrs['inttime'] # second start_ind = transit_ind - np.int(span / int_time) end_ind = transit_ind + np.int(span / int_time) nt = end_ind - start_ind nfeed = len(feedno) eigval = np.empty((nt, nfeed, 2, len(lfreq)), dtype=np.float64) eigval[:] = np.nan gain = np.empty((nt, nfeed, 2, len(lfreq)), dtype=np.complex128) gain[:] = complex(np.nan, np.nan) # construct visibility matrix for a single time, pol, freq Vmat = np.zeros((nfeed, nfeed), dtype=ts.main_data.dtype) for ind, ti in enumerate(range(start_ind, end_ind)): # when noise on, just pass if ts['ns_on'][ti]: continue aa.set_jultime(ts['jul_date'][ti]) s.compute(aa) # get fluxes vs. freq of the calibrator Sc = s.get_jys() # get the topocentric coordinate of the calibrator at the current time s_top = s.get_crds('top', ncrd=3) aa.sim_cache(cat.get_crds('eq', ncrd=3)) # for compute bm_response and sim for pi in [pol.index('xx'), pol.index('yy')]: # xx, yy aa.set_active_pol(pol[pi]) for fi, freq in enumerate(lfreq): # mpi among freq for i, ai in enumerate(feedno): for j, aj in enumerate(feedno): # uij = aa.gen_uvw(i, j, src='z').squeeze() # (rj - ri)/lambda uij = aa.gen_uvw(i, j, src='z')[:, 0, :] # (rj - ri)/lambda # bmij = aa.bm_response(i, j).squeeze() # will get error for only one local freq # import pdb # pdb.set_trace() bmij = aa.bm_response(i, j).reshape(-1) try: bi = bls.index((ai, aj)) # Vmat[i, j] = ts.local_vis[ti, fi, pi, bi] / (Sc[fi] * bmij[fi] * np.exp(-2.0J * np.pi * np.dot(s_top, uij[:, fi]))) # xx, yy Vmat[i, j] = ts.local_vis[ti, fi, pi, bi] / (Sc[fi] * bmij[fi] * np.exp(2.0J * np.pi * np.dot(s_top, uij[:, fi]))) # xx, yy except ValueError: bi = bls.index((aj, ai)) # Vmat[i, j] = np.conj(ts.local_vis[ti, fi, pi, bi] / (Sc[fi] * bmij[fi] * np.exp(-2.0J * np.pi * np.dot(s_top, uij[:, fi])))) # xx, yy Vmat[i, j] = np.conj(ts.local_vis[ti, fi, pi, bi] / (Sc[fi] * bmij[fi] * np.exp(2.0J * np.pi * np.dot(s_top, uij[:, fi])))) # xx, yy # Eigen decomposition Vmat = np.where(np.isfinite(Vmat), Vmat, 0) e, U = eigh(Vmat) eigval[ind, :, pi, fi] = e[::-1] # descending order # max eigen-val lbd = e[-1] # lambda # the gain vector for this freq gvec = np.sqrt(lbd) * U[:, -1] # only eigen-vector corresponding to the maximum eigen-val gain[ind, :, pi, fi] = gvec # apply gain to vis # get the time mean gain tgain = np.ma.mean(np.ma.masked_invalid(gain), axis=0) # time mean tgain = mpiutil.gather_array(tgain, axis=-1, root=None) ts.redistribute('baseline') ts.pol_and_bl_data_operate(cal, tgain=tgain) # save gain if required: if save_gain: gain_file = output_path(gain_file) eigval = mpiarray.MPIArray.wrap(eigval, axis=3) gain = mpiarray.MPIArray.wrap(gain, axis=3) mem_gain = memh5.MemGroup(distributed=True) mem_gain.create_dataset('eigval', data=eigval) mem_gain.create_dataset('gain', data=gain) # add attris mem_gain.attrs['jul_data'] = ts['jul_date'][start_ind:end_ind] mem_gain.attrs['feed'] = np.array(feedno) mem_gain.attrs['pol'] = np.array(['xx', 'yy']) mem_gain.attrs['freq'] = ts.freq[:] # freq should be common # save to file mem_gain.to_hdf5(gain_file, hints=False) ts.add_history(self.history) return ts
def process(self, ts): mask_daytime = self.params["mask_daytime"] mask_time_range = self.params["mask_time_range"] beam_theta_range = self.params["beam_theta_range"] tsys = self.params["tsys"] accuracy_boost = self.params["accuracy_boost"] l_boost = self.params["l_boost"] bl_range = self.params["bl_range"] auto_correlations = self.params["auto_correlations"] pol = self.params["pol"] beam_dir = output_path(self.params["beam_dir"]) gen_inv = self.params["gen_invbeam"] noise_weight = self.params["noise_weight"] ts_dir = output_path(self.params["ts_dir"]) ts_name = self.params["ts_name"] simulate = self.params["simulate"] input_maps = self.params["input_maps"] add_noise = self.params["add_noise"] ts.redistribute("frequency") lat = ts.attrs["sitelat"] # lon = ts.attrs['sitelon'] lon = 0.0 # lon = np.degrees(ts['ra_dec'][0, 0]) # the first ra freqs = ts.freq.data.to_numpy_array(root=None) ndays = 1 feeds = ts["feedno"][:] az, alt = ts["az_alt"][0] az = np.degrees(az) alt = np.degrees(alt) pointing = [az, alt, 0.0] feedpos = ts["feedpos"][:] if ts.is_dish: dish_width = ts.attrs["dishdiam"] tel = tl_dish.TlUnpolarisedDishArray( lat, lon, freqs, beam_theta_range, tsys, ndays, accuracy_boost, l_boost, bl_range, auto_correlations, dish_width, feedpos, pointing, ) elif ts.is_cylinder: cyl_width = ts.attrs["cywid"] tel = tl_cylinder.TlUnpolarisedCylinder( lat, lon, freqs, beam_theta_range, tsys, ndays, accuracy_boost, l_boost, bl_range, auto_correlations, cyl_width, feedpos, ) else: raise RuntimeError("Unknown array type %s" % ts.attrs["telescope"]) # import matplotlib # matplotlib.use('Agg') # import matplotlib.pyplot as plt # plt.figure() # plt.plot(ts['ra_dec'][:]) # # plt.plot(ts['az_alt'][:]) # plt.savefig('ra_dec1.png') if not simulate: # mask daytime data if mask_daytime: day_inds = np.where( np.logical_and(ts["local_hour"][:] >= mask_time_range[0], ts["local_hour"][:] <= mask_time_range[1]) )[0] ts.local_vis_mask[day_inds] = True # do not change vis directly # average data nt = ts["sec1970"].shape[0] phi_size = tel.phi_size nt_m = float(nt) / phi_size # roll data to have phi=0 near the first roll_len = np.int(np.around(0.5 * nt_m)) ts.local_vis[:] = np.roll(ts.local_vis[:], roll_len, axis=0) ts.local_vis_mask[:] = np.roll(ts.local_vis_mask[:], roll_len, axis=0) ts["ra_dec"][:] = np.roll(ts["ra_dec"][:], roll_len, axis=0) repeat_inds = np.repeat(np.arange(nt), phi_size) num, start, end = mpiutil.split_m(nt * phi_size, phi_size) # phi = np.zeros((phi_size,), dtype=ts['ra_dec'].dtype) phi = np.linspace(0, 2 * np.pi, phi_size, endpoint=False) vis = np.zeros((phi_size,) + ts.local_vis.shape[1:], dtype=ts.vis.dtype) # average over time for idx in xrange(phi_size): inds, weight = unique(repeat_inds[start[idx] : end[idx]], return_counts=True) vis[idx] = average( np.ma.array(ts.local_vis[inds], mask=ts.local_vis_mask[inds]), axis=0, weights=weight ) # time mean # phi[idx] = np.average(ts['ra_dec'][:, 0][inds], axis=0, weights=weight) if pol == "xx": vis = vis[:, :, 0, :] elif pol == "yy": vis = vis[:, :, 1, :] elif pol == "I": vis = 0.5 * (vis[:, :, 0, :] + vis[:, :, 1, :]) elif pol == "all": vis = np.sum(vis, axis=2) # sum over all pol else: raise ValueError("Invalid pol: %s" % pol) allpairs = tel.allpairs redundancy = tel.redundancy # reorder bls according to allpairs vis_tmp = np.zeros_like(vis) bls = [tuple(bl) for bl in ts["blorder"][:]] for ind, (a1, a2) in enumerate(allpairs): try: b_ind = bls.index((feeds[a1], feeds[a2])) vis_tmp[:, :, ind] = vis[:, :, b_ind] except ValueError: b_ind = bls.index((feeds[a2], feeds[a1])) vis_tmp[:, :, ind] = vis[:, :, b_ind].conj() # average over redundancy vis_stream = np.zeros(vis.shape[:-1] + (len(redundancy),), dtype=vis_tmp.dtype) red_bin = np.cumsum(np.insert(redundancy, 0, 0)) # redundancy bin # average over redundancy for ind in xrange(len(redundancy)): vis_stream[:, :, ind] = np.sum(vis_tmp[:, :, red_bin[ind] : red_bin[ind + 1]], axis=2) / redundancy[ind] del vis del vis_tmp vis_stream = mpiarray.MPIArray.wrap(vis_stream, axis=1) vis_h5 = memh5.MemGroup(distributed=True) vis_h5.create_dataset("/timestream", data=vis_stream) vis_h5.create_dataset("/phi", data=phi) # Telescope layout data vis_h5.create_dataset("/feedmap", data=tel.feedmap) vis_h5.create_dataset("/feedconj", data=tel.feedconj) vis_h5.create_dataset("/feedmask", data=tel.feedmask) vis_h5.create_dataset("/uniquepairs", data=tel.uniquepairs) vis_h5.create_dataset("/baselines", data=tel.baselines) # Telescope frequencies vis_h5.create_dataset("/frequencies", data=freqs) # Write metadata # vis_h5.attrs['beamtransfer_path'] = os.path.abspath(bt.directory) vis_h5.attrs["ntime"] = phi_size # beamtransfer bt = beamtransfer.BeamTransfer(beam_dir, tel, gen_inv, noise_weight) bt.generate() if simulate: ndays = 733 print ndays ts = timestream.simulate(bt, ts_dir, ts_name, input_maps, ndays, add_noise=add_noise) else: # timestream and map-making ts = timestream.Timestream(ts_dir, ts_name, bt) # Make directory if required try: os.makedirs(ts._tsdir) except OSError: # directory exists pass vis_h5.to_hdf5(ts._tsfile) # ts.generate_mmodes(vis_stream.to_numpy_array(root=None)) ts.generate_mmodes() nside = hputil.nside_for_lmax(tel.lmax, accuracy_boost=tel.accuracy_boost) ts.mapmake_full(nside, "full") # ts.add_history(self.history) return ts
def _init_output_files(self): self.output_files = output_path(format_list( self.params['output_files']), mkdir=False)
def flag(self, vis, vis_mask, li, gi, bl, ts, **kwargs): """Function that does the actual flag.""" freq_window = self.params['freq_window'] time_window = self.params['time_window'] freq_sigma = self.params['freq_sigma'] time_sigma = self.params['time_sigma'] plot_fit = self.params['plot_fit'] freq_fig_prefix = self.params['freq_fig_name'] time_fig_prefix = self.params['time_fig_name'] tag_output_iter = self.params['tag_output_iter'] iteration = self.iteration freq_flag = kwargs.get('freq_flag') time_flag = kwargs.get('time_flag') time = ts.time[:] nt = len(time) freq = ts.freq[:] nfreq = len(freq) if isinstance(ts, Timestream): # for Timestream pol = bl[0] bl = tuple(bl[1]) elif isinstance(ts, RawTimestream): # for RawTimestream pol = None bl = tuple(bl) else: raise ValueError('Need either a RawTimestream or Timestream') if freq_flag: # time integration tm_vis = np.ma.mean(np.ma.array(vis, mask=vis_mask), axis=0) abs_vis = np.abs(tm_vis) if np.ma.count_masked(tm_vis) > 0: # has masked value abs_vis_valid = abs_vis[~abs_vis.mask] inds_valid = np.arange(nfreq)[~abs_vis.mask] itp = InterpolatedUnivariateSpline(inds_valid, abs_vis_valid) abs_vis_itp = itp(np.arange(nfreq)) abs_vis1 = abs_vis_itp.copy() else: abs_vis1 = abs_vis.copy() for cnt in xrange(10): if cnt != 0: abs_vis1[inds] = smooth[inds] smooth = savitzky_golay(abs_vis1, freq_window, 3) # flage RFI diff = abs_vis1 - smooth mean = np.mean(diff) std = np.std(diff) inds = np.where(np.abs(diff - mean) > freq_sigma * std)[0] if len(inds) == 0: break diff = abs_vis - smooth mean = np.mean(diff) std = np.std(diff) inds = np.where( np.abs(diff - mean) > freq_sigma * std)[0] # masked inds vis_mask[:, inds] = True # set mask if plot_fit: plt.figure() plt.plot(freq, abs_vis, label='data') plt.plot(freq[inds], abs_vis[inds], 'ro', label='flag') plt.plot(freq, smooth, label='smooth') plt.xlabel(r'$\nu$ / MHz') plt.legend(loc='best') if pol is None: fig_name = '%s_%d_%d.png' % (freq_fig_prefix, bl[0], bl[1]) else: fig_name = '%s_%d_%d_%s.png' % (freq_fig_prefix, bl[0], bl[1], pol) if tag_output_iter: fig_name = output_path(fig_name, iteration=iteration) else: fig_name = output_path(fig_name) plt.savefig(fig_name) plt.close() if time_flag: # freq integration fm_vis = np.ma.mean(np.ma.array(vis, mask=vis_mask), axis=1) abs_vis = np.abs(fm_vis) if np.ma.count_masked(fm_vis) > 0: # has masked value abs_vis_valid = abs_vis[~abs_vis.mask] inds_valid = np.arange(nt)[~abs_vis.mask] itp = InterpolatedUnivariateSpline(inds_valid, abs_vis_valid) abs_vis_itp = itp(np.arange(nt)) abs_vis1 = abs_vis_itp.copy() else: abs_vis1 = abs_vis.copy() for cnt in xrange(10): if cnt != 0: abs_vis1[inds] = smooth[inds] smooth = savitzky_golay(abs_vis1, time_window, 3) # flage RFI diff = abs_vis1 - smooth mean = np.mean(diff) std = np.std(diff) inds = np.where(np.abs(diff - mean) > time_sigma * std)[0] if len(inds) == 0: break diff = abs_vis - smooth mean = np.mean(diff) std = np.std(diff) inds = np.where( np.abs(diff - mean) > time_sigma * std)[0] # masked inds # Addtional threshold # inds1 = np.where(np.abs(diff[inds]) > 1.0e-2*np.abs(smooth[inds]))[0] # inds = inds[inds1] vis_mask[inds] = True # set mask if plot_fit: plt.figure() plt.plot(time, abs_vis, label='data') plt.plot(time[inds], abs_vis[inds], 'ro', label='flag') plt.plot(time, smooth, label='smooth') plt.xlabel(r'$t$ / Julian Date') plt.legend(loc='best') if pol is None: fig_name = '%s_%d_%d.png' % (time_fig_prefix, bl[0], bl[1]) else: fig_name = '%s_%d_%d_%s.png' % (time_fig_prefix, bl[0], bl[1], pol) if tag_output_iter: fig_name = output_path(fig_name, iteration=iteration) else: fig_name = output_path(fig_name) plt.savefig(fig_name) plt.close()
def plot(self, vis, vis_mask, li, gi, bl, ts, **kwargs): """Function that does the actual plot work.""" bl_incl = self.params['bl_incl'] bl_excl = self.params['bl_excl'] flag_mask = self.params['flag_mask'] flag_ns = self.params['flag_ns'] interpolate_ns = self.params['interpolate_ns'] y_axis = self.params['y_axis'] plot_abs = self.params['plot_abs'] fig_prefix = self.params['fig_name'] tag_output_iter = self.params['tag_output_iter'] iteration = self.iteration if isinstance(ts, Timestream): # for Timestream pol = bl[0] bl = tuple(bl[1]) elif isinstance(ts, RawTimestream): # for RawTimestream pol = None bl = tuple(bl) else: raise ValueError('Need either a RawTimestream or Timestream') if bl_incl != 'all': bl1 = set(bl) bl_incl = [ {f1, f2} for (f1, f2) in bl_incl ] bl_excl = [ {f1, f2} for (f1, f2) in bl_excl ] if (not bl1 in bl_incl) or (bl1 in bl_excl): return vis, vis_mask if flag_mask: vis1 = np.ma.array(vis, mask=vis_mask) elif flag_ns: vis1 = vis.copy() on = np.where(ts['ns_on'][:])[0] if not interpolate_ns: vis1[on] = complex(np.nan, np.nan) else: off = np.where(np.logical_not(ts['ns_on'][:]))[0] for fi in xrange(vis1.shape[1]): itp_real = InterpolatedUnivariateSpline(off, vis1[off, fi].real) itp_imag= InterpolatedUnivariateSpline(off, vis1[off, fi].imag) vis1[on, fi] = itp_real(on) + 1.0J * itp_imag(on) else: vis1 = vis freq = ts.freq[:] if y_axis == 'jul_date': y_aixs = ts.time[:] y_label = r'$t$ / Julian Date' elif y_axis == 'ra': y_aixs = ts['ra_dec'][:, 0] y_label = r'RA / radian' extent = [freq[0], freq[-1], y_aixs[0], y_aixs[-1]] plt.figure() if plot_abs: fig, axarr = plt.subplots(1, 3, sharey=True) else: fig, axarr = plt.subplots(1, 2, sharey=True) im = axarr[0].imshow(vis1.real, extent=extent, origin='lower', aspect='auto') axarr[0].set_xlabel(r'$\nu$ / MHz') axarr[0].set_ylabel(y_label) plt.colorbar(im, ax=axarr[0]) im = axarr[1].imshow(vis1.imag, extent=extent, origin='lower', aspect='auto') axarr[1].set_xlabel(r'$\nu$ / MHz') plt.colorbar(im, ax=axarr[1]) if plot_abs: im = axarr[2].imshow(np.abs(vis1), extent=extent, origin='lower', aspect='auto') axarr[2].set_xlabel(r'$\nu$ / MHz') plt.colorbar(im, ax=axarr[2]) if pol is None: fig_name = '%s_%d_%d.png' % (fig_prefix, bl[0], bl[1]) else: fig_name = '%s_%d_%d_%s.png' % (fig_prefix, bl[0], bl[1], pol) if tag_output_iter: fig_name = output_path(fig_name, iteration=iteration) else: fig_name = output_path(fig_name) plt.savefig(fig_name) plt.close() return vis, vis_mask
def plot(self, vis, vis_mask, li, gi, bl, ts, **kwargs): """Function that does the actual plot work.""" bl_incl = self.params['bl_incl'] bl_excl = self.params['bl_excl'] flag_mask = self.params['flag_mask'] flag_ns = self.params['flag_ns'] interpolate_ns = self.params['interpolate_ns'] y_axis = self.params['y_axis'] plot_abs = self.params['plot_abs'] abs_only = self.params['abs_only'] gray_color = self.params['gray_color'] color_flag = self.params['color_flag'] flag_color = self.params['flag_color'] transpose = self.params['transpose'] hist_equal = self.params['hist_equal'] fig_prefix = self.params['fig_name'] rotate_xdate = self.params['rotate_xdate'] feed_no = self.params['feed_no'] order_bl = self.params['order_bl'] tag_output_iter = self.params['tag_output_iter'] iteration = self.iteration if isinstance(ts, Timestream): # for Timestream pol = bl[0] bl = tuple(bl[1]) feed_no = True elif isinstance(ts, RawTimestream): # for RawTimestream pol = None bl = tuple(bl) if feed_no: pol = ts['bl_pol'].local_data[li] bl = tuple(ts['true_blorder'].local_data[li]) if order_bl and (bl[0] > bl[1]): bl = (bl[1], bl[0]) vis = vis.conj() else: raise ValueError('Need either a RawTimestream or Timestream') if bl_incl != 'all': bl1 = set(bl) bl_incl = [{f1, f2} for (f1, f2) in bl_incl] bl_excl = [{f1, f2} for (f1, f2) in bl_excl] if (not bl1 in bl_incl) or (bl1 in bl_excl): return if flag_mask: vis1 = np.ma.array(vis, mask=vis_mask) elif flag_ns: if 'ns_on' in ts.iterkeys(): vis1 = vis.copy() on = np.where(ts['ns_on'][:])[0] if not interpolate_ns: vis1[on] = complex(np.nan, np.nan) else: off = np.where(np.logical_not(ts['ns_on'][:]))[0] for fi in xrange(vis1.shape[1]): itp_real = InterpolatedUnivariateSpline( off, vis1[off, fi].real) itp_imag = InterpolatedUnivariateSpline( off, vis1[off, fi].imag) vis1[on, fi] = itp_real(on) + 1.0J * itp_imag(on) else: vis1 = vis else: vis1 = vis freq = ts.freq[:] x_label = r'$\nu$ / MHz' if y_axis == 'jul_date': y_aixs = ts.time[:] y_label = r'$t$ / Julian Date' elif y_axis == 'ra': y_aixs = ts['ra_dec'][:, 0] y_label = r'RA / radian' elif y_axis == 'time': y_aixs = [ datetime.fromtimestamp(s) for s in (ts['sec1970'][0], ts['sec1970'][-1]) ] y_label = '%s' % y_aixs[0].date() # convert datetime objects to the correct format for matplotlib to work with y_aixs = mdates.date2num(y_aixs) else: raise ValueError( 'Invalid y_axis %s, can only be "time", "jul_data" or "ra"' % y_axis) freq_extent = [freq[0], freq[-1]] time_extent = [y_aixs[0], y_aixs[-1]] extent = freq_extent + time_extent plt.figure() if gray_color: # cmap = 'gray' cmap = plt.cm.gray if color_flag: cmap.set_bad(flag_color) else: cmap = None if abs_only: if transpose: vis1 = vis1.T x_label, y_label = y_label, x_label extent = time_extent + freq_extent fig, ax = plt.subplots() vis_abs = np.abs(vis1) if hist_equal: if isinstance(vis_abs, np.ma.MaskedArray): vis_hist = hist_eq.hist_eq(vis_abs.filled(0)) vis_abs = np.ma.array(vis_hist, mask=np.ma.getmask(vis_abs)) else: vis_hist = hist_eq.hist_eq( np.where(np.isfinite(vis_abs), vis_abs, 0)) mask = np.where(np.isfinite(vis_abs), False, True) vis_abs = np.ma.array(vis_hist, mask=mask) im = ax.imshow(vis_abs, extent=extent, origin='lower', aspect='auto', cmap=cmap) # convert axis to datetime string if transpose: ax.xaxis_date() else: ax.yaxis_date() # format datetime string # date_format = mdates.DateFormatter('%y/%m/%d %H:%M') date_format = mdates.DateFormatter('%H:%M') if transpose: ax.xaxis.set_major_formatter(date_format) else: ax.yaxis.set_major_formatter(date_format) if transpose: if rotate_xdate: # set the x-axis tick labels to diagonal so it fits better fig.autofmt_xdate() else: # reduce the number of tick locators locator = MaxNLocator(nbins=6) ax.xaxis.set_major_locator(locator) ax.xaxis.set_minor_locator(AutoMinorLocator(2)) ax.set_xlabel(x_label) ax.set_ylabel(y_label) plt.colorbar(im) else: if plot_abs: fig, axarr = plt.subplots(1, 3, sharey=True) else: fig, axarr = plt.subplots(1, 2, sharey=True) im = axarr[0].imshow(vis1.real, extent=extent, origin='lower', aspect='auto', cmap=cmap) axarr[0].set_xlabel(x_label) axarr[0].yaxis_date() # format datetime string date_format = mdates.DateFormatter('%H:%M') axarr[0].yaxis.set_major_formatter(date_format) axarr[0].set_ylabel(y_label) plt.colorbar(im, ax=axarr[0]) im = axarr[1].imshow(vis1.imag, extent=extent, origin='lower', aspect='auto', cmap=cmap) axarr[1].set_xlabel(x_label) plt.colorbar(im, ax=axarr[1]) if plot_abs: im = axarr[2].imshow(np.abs(vis1), extent=extent, origin='lower', aspect='auto', cmap=cmap) axarr[2].set_xlabel(x_label) plt.colorbar(im, ax=axarr[2]) if feed_no: fig_name = '%s_%d_%d_%s.png' % (fig_prefix, bl[0], bl[1], ts.pol_dict[pol]) else: fig_name = '%s_%d_%d.png' % (fig_prefix, bl[0], bl[1]) if tag_output_iter: fig_name = output_path(fig_name, iteration=iteration) else: fig_name = output_path(fig_name) plt.savefig(fig_name) plt.close()
def cal(self, vis, vis_mask, li, gi, fbl, rt, **kwargs): """Function that does the actual cal.""" num_mean = self.params['num_mean'] plot_phs = self.params['plot_phs'] fig_prefix = self.params['fig_name'] rotate_xdate = self.params['rotate_xdate'] feed_no = self.params['feed_no'] tag_output_iter = self.params['tag_output_iter'] iteration = self.iteration bls_plt = kwargs['bls_plt'] freq_plt = kwargs['freq_plt'] if np.prod(vis.shape) == 0 : return fi = gi[0] # freq idx for this cal bl = tuple(fbl[1]) # bl for this cal nt = vis.shape[0] on_time = rt['ns_on'].attrs['on_time'] num_mean = min(num_mean, on_time-2) if num_mean <= 0: raise RuntimeError('Do not have enough noise on time samples to do the ns_cal') ns_on = rt['ns_on'][:] ns_on = np.where(ns_on, 1, 0) diff_ns = np.diff(ns_on) inds = np.where(diff_ns==1)[0] # if inds[0]-num_mean < 0: if inds[0]-1 < 0: # no off data in the beginning to use inds = inds[1:] # if inds[-1]+num_mean+1 > len(ns_on)-1: if inds[-1]+2 > nt-1: # no on data in the end to use inds = inds[:-1] valid_inds = [] phase = [] for ind in inds: if ind == inds[0]: # the first ind lower = max(0, ind-num_mean) else: lower = ind - num_mean off_sec = np.ma.array(vis[lower:ind], mask=vis_mask[lower:ind]) if off_sec.count() > 0: # not all data in this section are masked valid_inds.append(ind) if ind == inds[-1]: # the last ind upper = min(nt, ind+2+num_mean) else: upper = ind + 2 + num_mean phase.append( np.angle(np.mean(vis[ind+2:upper]) - np.ma.mean(off_sec)) ) # in radians # not enough valid data to do the ns_cal if len(phase) <= 3: vis_mask[:] = True # mask the vis as no ns_cal has done return phase = np.unwrap(phase) # unwrap 2pi discontinuity f = InterpolatedUnivariateSpline(valid_inds, phase) all_phase = f(np.arange(nt)) if plot_phs and (bl in bls_plt and fi in freq_plt): plt.figure() fig, ax = plt.subplots() ax_val = np.array([ datetime.fromtimestamp(sec) for sec in rt['sec1970'][:] ]) xlabel = '%s' % ax_val[0].date() ax_val = mdates.date2num(ax_val) ax.plot(ax_val, all_phase) ax.plot(ax_val[valid_inds], phase, 'ro') ax.xaxis_date() date_format = mdates.DateFormatter('%H:%M') ax.xaxis.set_major_formatter(date_format) if rotate_xdate: # set the x-axis tick labels to diagonal so it fits better fig.autofmt_xdate() else: # reduce the number of tick locators locator = MaxNLocator(nbins=6) ax.xaxis.set_major_locator(locator) ax.xaxis.set_minor_locator(AutoMinorLocator(2)) ax.set_xlabel(xlabel) ax.set_ylabel(r'$\Delta \phi$ / radian') if feed_no: pol = rt['bl_pol'].local_data[li[1]] bl = tuple(rt['true_blorder'].local_data[li[1]]) fig_name = '%s_%f_%d_%d_%s.png' % (fig_prefix, fbl[0], bl[0], bl[1], rt.pol_dict[pol]) else: fig_name = '%s_%f_%d_%d.png' % (fig_prefix, fbl[0], fbl[1][0], fbl[1][1]) if tag_output_iter: fig_name = output_path(fig_name, iteration=iteration) else: fig_name = output_path(fig_name) plt.savefig(fig_name) plt.close() vis[:] = vis * np.exp(-1.0J * all_phase)
def process(self, ts): assert isinstance( ts, Timestream ), '%s only works for Timestream object' % self.__class__.__name__ mask_daytime = self.params['mask_daytime'] mask_time_range = self.params['mask_time_range'] beam_theta_range = self.params['beam_theta_range'] tsys = self.params['tsys'] accuracy_boost = self.params['accuracy_boost'] l_boost = self.params['l_boost'] bl_range = self.params['bl_range'] auto_correlations = self.params['auto_correlations'] pol = self.params['pol'] beam_dir = output_path(self.params['beam_dir']) gen_inv = self.params['gen_invbeam'] noise_weight = self.params['noise_weight'] ts_dir = output_path(self.params['ts_dir']) ts_name = self.params['ts_name'] simulate = self.params['simulate'] input_maps = self.params['input_maps'] add_noise = self.params['add_noise'] dirty_map = self.params['dirty_map'] nbin = self.params['nbin'] method = self.params['method'] normalize = self.params['normalize'] threshold = self.params['threshold'] ts.redistribute('frequency') lat = ts.attrs['sitelat'] # lon = ts.attrs['sitelon'] lon = 0.0 # lon = np.degrees(ts['ra_dec'][0, 0]) # the first ra local_origin = False freq = ts.freq freqs = ts.freq.data.to_numpy_array(root=None) # MHz band_width = ts.attrs['freqstep'] # MHz ndays = ts.attrs['ndays'] feeds = ts['feedno'][:] az, alt = ts['az_alt'][0] az = np.degrees(az) alt = np.degrees(alt) pointing = [az, alt, 0.0] feedpos = ts['feedpos'][:] if ts.is_dish: dish_width = ts.attrs['dishdiam'] tel = tl_dish.TlUnpolarisedDishArray(lat, lon, freqs, band_width, tsys, ndays, accuracy_boost, l_boost, bl_range, auto_correlations, local_origin, dish_width, feedpos, pointing) elif ts.is_cylinder: # factor = 1.2 # suppose an illumination efficiency, keep same with that in timestream_common factor = 0.79 # for xx # factor = 0.88 # for yy cyl_width = factor * ts.attrs['cywid'] tel = tl_cylinder.TlUnpolarisedCylinder( lat, lon, freqs, band_width, tsys, ndays, accuracy_boost, l_boost, bl_range, auto_correlations, local_origin, cyl_width, feedpos) else: raise RuntimeError('Unknown array type %s' % ts.attrs['telescope']) if not simulate: # mask daytime data if mask_daytime: day_inds = np.where( np.logical_and( ts['local_hour'][:] >= mask_time_range[0], ts['local_hour'][:] <= mask_time_range[1]))[0] ts.local_vis_mask[ day_inds] = True # do not change vis directly # average data nt = ts['sec1970'].shape[0] phi_size = 2 * tel.mmax + 1 nt_m = float(nt) / phi_size # roll data to have phi=0 near the first roll_len = np.int(np.around(0.5 * nt_m)) ts.local_vis[:] = np.roll(ts.local_vis[:], roll_len, axis=0) ts.local_vis_mask[:] = np.roll(ts.local_vis_mask[:], roll_len, axis=0) ts['ra_dec'][:] = np.roll(ts['ra_dec'][:], roll_len, axis=0) repeat_inds = np.repeat(np.arange(nt), phi_size) num, start, end = mpiutil.split_m(nt * phi_size, phi_size) # phi = np.zeros((phi_size,), dtype=ts['ra_dec'].dtype) phi = np.linspace(0, 2 * np.pi, phi_size, endpoint=False) vis = np.zeros((phi_size, ) + ts.local_vis.shape[1:], dtype=ts.vis.dtype) # average over time for idx in xrange(phi_size): inds, weight = unique(repeat_inds[start[idx]:end[idx]], return_counts=True) vis[idx] = average(np.ma.array(ts.local_vis[inds], mask=ts.local_vis_mask[inds]), axis=0, weights=weight) # time mean # phi[idx] = np.average(ts['ra_dec'][:, 0][inds], axis=0, weights=weight) if pol == 'xx': vis = vis[:, :, 0, :] elif pol == 'yy': vis = vis[:, :, 1, :] elif pol == 'I': vis = 0.5 * (vis[:, :, 0, :] + vis[:, :, 1, :]) elif pol == 'all': vis = np.sum(vis, axis=2) # sum over all pol else: raise ValueError('Invalid pol: %s' % pol) allpairs = tel.allpairs redundancy = tel.redundancy # reorder bls according to allpairs vis_tmp = np.zeros_like(vis) bls = [tuple(bl) for bl in ts['blorder'][:]] for ind, (a1, a2) in enumerate(allpairs): try: b_ind = bls.index((feeds[a1], feeds[a2])) vis_tmp[:, :, ind] = vis[:, :, b_ind] except ValueError: b_ind = bls.index((feeds[a2], feeds[a1])) vis_tmp[:, :, ind] = vis[:, :, b_ind].conj() # average over redundancy vis_stream = np.zeros(vis.shape[:-1] + (len(redundancy), ), dtype=vis_tmp.dtype) red_bin = np.cumsum(np.insert(redundancy, 0, 0)) # redundancy bin # average over redundancy for ind in xrange(len(redundancy)): vis_stream[:, :, ind] = np.sum( vis_tmp[:, :, red_bin[ind]:red_bin[ind + 1]], axis=2) / redundancy[ind] del vis del vis_tmp # beamtransfer bt = beamtransfer.BeamTransfer(beam_dir, tel, noise_weight, True) bt.generate() if simulate: ndays = 733 print ndays tstream = timestream.simulate(bt, ts_dir, ts_name, input_maps, ndays, add_noise=add_noise) else: # timestream and map-making tstream = timestream.Timestream(ts_dir, ts_name, bt) for lfi, fi in freq.data.enumerate(axis=0): # Make directory if required if not os.path.exists(tstream._fdir(fi)): os.makedirs(tstream._fdir(fi)) # Write file contents with h5py.File(tstream._ffile(fi), 'w') as f: # Timestream data f.create_dataset('/timestream', data=vis_stream[:, lfi].T) f.create_dataset('/phi', data=phi) # Telescope layout data f.create_dataset('/feedmap', data=tel.feedmap) f.create_dataset('/feedconj', data=tel.feedconj) f.create_dataset('/feedmask', data=tel.feedmask) f.create_dataset('/uniquepairs', data=tel.uniquepairs) f.create_dataset('/baselines', data=tel.baselines) # Telescope frequencies f.create_dataset('/frequencies', data=freqs) # Write metadata f.attrs['beamtransfer_path'] = os.path.abspath( bt.directory) f.attrs['ntime'] = phi_size mpiutil.barrier() tstream.generate_mmodes() nside = hputil.nside_for_lmax(tel.lmax, accuracy_boost=tel.accuracy_boost) if dirty_map: tstream.mapmake_full(nside, 'map_full_dirty.hdf5', nbin, dirty=True, method=method, normalize=normalize, threshold=threshold) else: tstream.mapmake_full(nside, 'map_full.hdf5', nbin, dirty=False, method=method, normalize=normalize, threshold=threshold) # ts.add_history(self.history) return tstream
def fit(vis_obs, vis_mask, vis_sim, start_ind, end_ind, num_shift, idx, plot_fit, fig_prefix, iteration, tag_output_iter, bls_plt, freq_plt): vis_obs = np.ma.array(vis_obs, mask=vis_mask) num_nomask = vis_obs.count() if num_nomask == 0: # no valid vis data return 1.0, 0 fi, pi, (i, j) = idx gains = [] chi2s = [] shifts = xrange(-num_shift / 2, num_shift / 2 + 1) for si in shifts: # vis = vis_obs[start_ind+si:end_ind+si].astype(np.complex128) # improve precision vis = vis_obs[start_ind + si:end_ind + si] num_nomask = vis.count() if num_nomask == 0: # no valid vis data continue else: xx = np.ma.dot(vis_sim.conj(), vis_sim) xy = np.ma.dot(vis_sim.conj(), vis) gain = xy / xx vis_cal = gain * vis_sim err = vis - vis_cal chi2 = np.ma.dot(err.conj(), err).real gains.append(gain) chi2s.append(chi2 / num_nomask) if len(gains) == 0: # no valid vis data return 1.0, 0 chi2s = np.array(chi2s) if np.allclose(chi2s, np.sort(chi2s)): if mpiutil.rank0: print 'Warn: chi2 increasing for %s...' % (idx, ) if np.allclose(chi2s, np.sort(chi2s)[::-1]): if mpiutil.rank0: print 'Warn: chi2 decreasing for %s...' % (idx, ) ind = np.argmin(chi2s) gain = gains[ind] # chi2 = chi2s[ind] si = shifts[ind] obs_data = np.ma.array(vis_obs[start_ind:end_ind], mask=vis_mask[start_ind:end_ind]) factor = np.ma.max(np.ma.abs(obs_data)) / np.max(np.abs(vis_sim)) obs_data = obs_data / factor # make amp close to each other vis_cal = np.ma.array(vis_obs[start_ind + si:end_ind + si], mask=vis_mask[start_ind + si:end_ind + si]) / gain if si != 0 and mpiutil.rank0: print 'shift %d for %s...' % (si, idx) if plot_fit and (fi in freq_plt and (i, j) in bls_plt): # plot the fit plt.figure() plt.subplot(311) plt.plot(obs_data.real, label='obs, real') if not vis_cal is np.ma.masked: # in case gain is -- plt.plot(vis_cal.real, label='cal, real') plt.plot(vis_sim.real, label='sim, real') plt.legend(loc='best') plt.subplot(312) plt.plot(obs_data.imag, label='obs, imag') if not vis_cal is np.ma.masked: # in case gain is -- plt.plot(vis_cal.imag, label='cal, imag') plt.plot(vis_sim.imag, label='sim, imag') plt.legend(loc='best') plt.subplot(313) plt.plot(np.abs(obs_data), label='obs, abs') if not vis_cal is np.ma.masked: # in case gain is -- plt.plot(np.abs(vis_cal), label='cal, abs') plt.plot(np.abs(vis_sim), label='sim, abs') plt.legend(loc='best') fig_name = '%s_%d_%d_%d_%d.png' % (fig_prefix, fi, pi, i, j) if tag_output_iter: fig_name = output_path(fig_name, iteration=iteration) else: fig_name = output_path(fig_name) plt.savefig(fig_name) plt.close() return gain, si
def plot(self, vis, vis_mask, li, gi, bl, ts, **kwargs): """Function that does the actual plot work.""" integral = self.params['integral'] bl_incl = self.params['bl_incl'] bl_excl = self.params['bl_excl'] flag_mask = self.params['flag_mask'] flag_ns = self.params['flag_ns'] fig_prefix = self.params['fig_name'] tag_output_iter = self.params['tag_output_iter'] iteration= self.iteration if isinstance(ts, Timestream): # for Timestream pol = bl[0] bl = tuple(bl[1]) elif isinstance(ts, RawTimestream): # for RawTimestream pol = None bl = tuple(bl) else: raise ValueError('Need either a RawTimestream or Timestream') if bl_incl != 'all': bl1 = set(bl) bl_incl = [ {f1, f2} for (f1, f2) in bl_incl ] bl_excl = [ {f1, f2} for (f1, f2) in bl_excl ] if (not bl1 in bl_incl) or (bl1 in bl_excl): return vis, vis_mask if flag_mask: vis1 = np.ma.array(vis, mask=vis_mask) elif flag_ns: vis1 = vis.copy() on = np.where(ts['ns_on'][:])[0] vis1[on] = complex(np.nan, np.nan) else: vis1 = vis if integral == 'time': ax_val = ts.freq[:] vis1 = np.ma.mean(np.ma.masked_invalid(vis1), axis=0) xlabel = r'$\nu$ / MHz' elif integral == 'freq': ax_val = ts.time[:] vis1 = np.ma.mean(np.ma.masked_invalid(vis1), axis=1) xlabel = r'$t$ / Julian Date' else: raise ValueError('Unknown integral type %s' % integral) plt.figure() f, axarr = plt.subplots(3, sharex=True) axarr[0].plot(ax_val, vis1.real, label='real') axarr[0].legend() axarr[1].plot(ax_val, vis1.imag, label='imag') axarr[1].legend() axarr[2].plot(ax_val, np.abs(vis1), label='abs') axarr[2].legend() axarr[2].set_xlabel(xlabel) if pol is None: fig_name = '%s_%s_%d_%d.png' % (fig_prefix, integral, bl[0], bl[1]) else: fig_name = '%s_%s_%d_%d_%s.png' % (fig_prefix, integral, bl[0], bl[1], pol) if tag_output_iter: fig_name = output_path(fig_name, iteration=iteration) else: fig_name = output_path(fig_name) plt.savefig(fig_name) plt.close() return vis, vis_mask
def process(self, ts): assert isinstance( ts, Timestream ), '%s only works for Timestream object' % self.__class__.__name__ calibrator = self.params['calibrator'] catalog = self.params['catalog'] vis_conj = self.params['vis_conj'] zero_diag = self.params['zero_diag'] span = self.params['span'] reserve_high_gain = self.params['reserve_high_gain'] plot_figs = self.params['plot_figs'] fig_prefix = self.params['fig_name'] tag_output_iter = self.params['tag_output_iter'] save_src_vis = self.params['save_src_vis'] src_vis_file = self.params['src_vis_file'] subtract_src = self.params['subtract_src'] replace_with_src = self.params['replace_with_src'] apply_gain = self.params['apply_gain'] save_gain = self.params['save_gain'] save_phs_change = self.params['save_phs_change'] gain_file = self.params['gain_file'] # temperature_convert = self.params['temperature_convert'] show_progress = self.params['show_progress'] progress_step = self.params['progress_step'] if save_src_vis or subtract_src or apply_gain or save_gain: pol_type = ts['pol'].attrs['pol_type'] if pol_type != 'linear': raise RuntimeError('Can not do ps_cal for pol_type: %s' % pol_type) ts.redistribute('baseline') feedno = ts['feedno'][:].tolist() pol = [ts.pol_dict[p] for p in ts['pol'][:]] # as string gain_pd = { 'xx': 0, 'yy': 1, 0: 'xx', 1: 'yy' } # for gain related op bls = mpiutil.gather_array(ts.local_bl[:], root=None, comm=ts.comm) # # antpointing = np.radians(ts['antpointing'][-1, :, :]) # radians # transitsource = ts['transitsource'][:] # transit_time = transitsource[-1, 0] # second, sec1970 # int_time = ts.attrs['inttime'] # second # get the calibrator try: s = calibrators.get_src(calibrator) except KeyError: if mpiutil.rank0: print 'Calibrator %s is unavailable, available calibrators are:' for key, d in calibrators.src_data.items(): print '%8s -> %12s' % (key, d[0]) raise RuntimeError('Calibrator %s is unavailable') if mpiutil.rank0: print 'Try to calibrate with %s...' % s.src_name # get transit time of calibrator # array aa = ts.array aa.set_jultime(ts['jul_date'][0]) # the first obs time point next_transit = aa.next_transit(s) transit_time = a.phs.ephem2juldate(next_transit) # Julian date # get time zone pattern = '[-+]?\d+' tz = re.search(pattern, ts.attrs['timezone']).group() tz = int(tz) local_next_transit = ephem.Date( next_transit + tz * ephem.hour) # plus 8h to get Beijing time # if transit_time > ts['jul_date'][-1]: if transit_time > max(ts['jul_date'][-1], ts['jul_date'][:].max()): raise RuntimeError( 'Data does not contain local transit time %s of source %s' % (local_next_transit, calibrator)) # the first transit index transit_inds = [np.searchsorted(ts['jul_date'][:], transit_time)] # find all other transit indices aa.set_jultime(ts['jul_date'][0] + 1.0) transit_time = a.phs.ephem2juldate( aa.next_transit(s)) # Julian date cnt = 2 while (transit_time <= ts['jul_date'][-1]): transit_inds.append( np.searchsorted(ts['jul_date'][:], transit_time)) aa.set_jultime(ts['jul_date'][0] + 1.0 * cnt) transit_time = a.phs.ephem2juldate( aa.next_transit(s)) # Julian date cnt += 1 if mpiutil.rank0: print 'transit ind of %s: %s, time: %s' % ( s.src_name, transit_inds, local_next_transit) ### now only use the first transit point to do the cal ### may need to improve in the future transit_ind = transit_inds[0] int_time = ts.attrs['inttime'] # second start_ind = transit_ind - np.int(span / int_time) end_ind = transit_ind + np.int( span / int_time) + 1 # plus 1 to make transit_ind is at the center start_ind = max(0, start_ind) end_ind = min(end_ind, ts.vis.shape[0]) if vis_conj: ts.local_vis[:] = ts.local_vis.conj() nt = end_ind - start_ind t_inds = range(start_ind, end_ind) freq = ts.freq[:] # MHz nf = len(freq) nlb = len(ts.local_bl[:]) nfeed = len(feedno) tfp_inds = list( itertools.product( t_inds, range(nf), [pol.index('xx'), pol.index('yy')])) # only for xx and yy ns, ss, es = mpiutil.split_all(len(tfp_inds), comm=ts.comm) # gather data to make each process to have its own data which has all bls for ri, (ni, si, ei) in enumerate(zip(ns, ss, es)): lvis = np.zeros((ni, nlb), dtype=ts.vis.dtype) lvis_mask = np.zeros((ni, nlb), dtype=ts.vis_mask.dtype) for ii, (ti, fi, pi) in enumerate(tfp_inds[si:ei]): lvis[ii] = ts.local_vis[ti, fi, pi] lvis_mask[ii] = ts.local_vis_mask[ti, fi, pi] # gather vis from all process for separate bls gvis = mpiutil.gather_array(lvis, axis=1, root=ri, comm=ts.comm) gvis_mask = mpiutil.gather_array(lvis_mask, axis=1, root=ri, comm=ts.comm) if ri == mpiutil.rank: tfp_linds = tfp_inds[si:ei] # inds for this process this_vis = gvis this_vis_mask = gvis_mask del tfp_inds del lvis del lvis_mask tfp_len = len(tfp_linds) # lotl_mask = np.zeros((tfp_len, nfeed, nfeed), dtype=bool) cnan = complex(np.nan, np.nan) # complex nan if save_src_vis or subtract_src: # save calibrator src vis lsrc_vis = np.full((tfp_len, nfeed, nfeed), cnan, dtype=ts.vis.dtype) if save_src_vis: # save sky vis lsky_vis = np.full((tfp_len, nfeed, nfeed), cnan, dtype=ts.vis.dtype) # save outlier vis lotl_vis = np.full((tfp_len, nfeed, nfeed), cnan, dtype=ts.vis.dtype) if apply_gain or save_gain: lGain = np.full((tfp_len, nfeed), cnan, dtype=ts.vis.dtype) # find indices mapping between Vmat and vis # bis = range(nbl) bis_conj = [] # indices that shold be conj mis = [ ] # indices in the nfeed x nfeed matrix by flatten it to a vector mis_conj = [ ] # indices (of conj vis) in the nfeed x nfeed matrix by flatten it to a vector for bi, (fdi, fdj) in enumerate(bls): ai, aj = feedno.index(fdi), feedno.index(fdj) mis.append(ai * nfeed + aj) if ai != aj: bis_conj.append(bi) mis_conj.append(aj * nfeed + ai) # construct visibility matrix for a single time, freq, pol Vmat = np.full((nfeed, nfeed), cnan, dtype=ts.vis.dtype) # get flus of the calibrator in the observing frequencies if show_progress and mpiutil.rank0: pg = progress.Progress(tfp_len, step=progress_step) for ii, (ti, fi, pi) in enumerate(tfp_linds): if show_progress and mpiutil.rank0: pg.show(ii) # when noise on, just pass if 'ns_on' in ts.iterkeys() and ts['ns_on'][ti]: continue # aa.set_jultime(ts['jul_date'][ti]) # s.compute(aa) # get the topocentric coordinate of the calibrator at the current time # s_top = s.get_crds('top', ncrd=3) # aa.sim_cache(cat.get_crds('eq', ncrd=3)) # for compute bm_response and sim Vmat.flat[mis] = np.ma.array( this_vis[ii], mask=this_vis_mask[ii]).filled(cnan) Vmat.flat[mis_conj] = np.ma.array( this_vis[ii, bis_conj], mask=this_vis_mask[ii, bis_conj]).conj().filled(cnan) if save_src_vis: lsky_vis[ii] = Vmat # set invalid val to 0 invalid = ~np.isfinite(Vmat) # a bool array # if too many masks if np.where(invalid)[0].shape[0] > 0.3 * nfeed**2: continue Vmat[invalid] = 0 # if all are zeros if np.allclose(Vmat, 0.0): continue # fill diagonal of Vmat to 0 if zero_diag: np.fill_diagonal(Vmat, 0) # initialize the outliers med = np.median(Vmat.real) + 1.0J * np.median(Vmat.imag) diff = Vmat - med S0 = np.where( np.abs(diff) > 3.0 * rpca_decomp.MAD(Vmat), diff, 0) # stable PCA decomposition V0, S = rpca_decomp.decompose(Vmat, rank=1, S=S0, max_iter=200, threshold='hard', tol=1.0e-6, debug=False) # # find abnormal values in S # # first check diagonal elements # import pdb; pdb.set_trace() # svals = np.diag(S) # smed = np.median(svals.real) + 1.0J * np.median(svals.imag) # smad = rpca_decomp.MAD(svals) # # abnormal indices # abis = np.where(np.abs(svals - smed) > 3.0 * smad)[0] # for abi in abis: # lotl_mask[ii, abi, abi] = True # # then check non-diagonal elements # for rii in range(nfeed): # for cii in range(nfeed): # if rii == cii: # continue # rli = max(0, rii-2) # rhi = min(nfeed, rii+3) # cli = max(0, cii-2) # chi = min(nfeed, cii+3) # svals = np.array([ S[xi, yi] for xi in range(rli, rhi) for yi in range(cli, chi) if xi != yi ]) # smed = np.median(svals.real) + 1.0J * np.median(svals.imag) # smad = rpca_decomp.MAD(svals) # if np.abs(S[rii, cii] - smed) > 3.0 * smad: # lotl_mask[ii, rii, cii] = True if save_src_vis or subtract_src: lsrc_vis[ii] = V0 if save_src_vis: lotl_vis[ii] = S # plot if plot_figs: ind = ti - start_ind # plot Vmat plt.figure(figsize=(13, 5)) plt.subplot(121) plt.imshow(Vmat.real, aspect='equal', origin='lower', interpolation='nearest') plt.colorbar(shrink=1.0) plt.subplot(122) plt.imshow(Vmat.imag, aspect='equal', origin='lower', interpolation='nearest') plt.colorbar(shrink=1.0) fig_name = '%s_V_%d_%d_%s.png' % (fig_prefix, ind, fi, pol[pi]) if tag_output_iter: fig_name = output_path(fig_name, iteration=self.iteration) else: fig_name = output_path(fig_name) plt.savefig(fig_name) plt.close() # plot V0 plt.figure(figsize=(13, 5)) plt.subplot(121) plt.imshow(V0.real, aspect='equal', origin='lower', interpolation='nearest') plt.colorbar(shrink=1.0) plt.subplot(122) plt.imshow(V0.imag, aspect='equal', origin='lower', interpolation='nearest') plt.colorbar(shrink=1.0) fig_name = '%s_V0_%d_%d_%s.png' % (fig_prefix, ind, fi, pol[pi]) if tag_output_iter: fig_name = output_path(fig_name, iteration=self.iteration) else: fig_name = output_path(fig_name) plt.savefig(fig_name) plt.close() # plot S plt.figure(figsize=(13, 5)) plt.subplot(121) plt.imshow(S.real, aspect='equal', origin='lower', interpolation='nearest') plt.colorbar(shrink=1.0) plt.subplot(122) plt.imshow(S.imag, aspect='equal', origin='lower', interpolation='nearest') plt.colorbar(shrink=1.0) fig_name = '%s_S_%d_%d_%s.png' % (fig_prefix, ind, fi, pol[pi]) if tag_output_iter: fig_name = output_path(fig_name, iteration=self.iteration) else: fig_name = output_path(fig_name) plt.savefig(fig_name) plt.close() # plot N N = Vmat - V0 - S plt.figure(figsize=(13, 5)) plt.subplot(121) plt.imshow(N.real, aspect='equal', origin='lower', interpolation='nearest') plt.colorbar(shrink=1.0) plt.subplot(122) plt.imshow(N.imag, aspect='equal', origin='lower', interpolation='nearest') plt.colorbar(shrink=1.0) fig_name = '%s_N_%d_%d_%s.png' % (fig_prefix, ind, fi, pol[pi]) if tag_output_iter: fig_name = output_path(fig_name, iteration=self.iteration) else: fig_name = output_path(fig_name) plt.savefig(fig_name) plt.close() if apply_gain or save_gain: # use v_ij = gi gj^* \int Ai Aj^* e^(2\pi i n \cdot uij) T(x) d^2n # precisely, we shold have # V0 = (lambda^2 * Sc / (2 k_B)) * gi gj^* Ai Aj^* e^(2\pi i n0 \cdot uij) e, U = la.eigh(V0, eigvals=(nfeed - 1, nfeed - 1)) g = U[:, -1] * e[ -1]**0.5 # = \sqrt(lambda^2 * Sc / (2 k_B)) * gi Ai * e^(2\pi i n0 \cdot ui) if g[0].real < 0: g *= -1.0 # make all g[0] phase 0, instead of pi lGain[ii] = g # plot Gain if plot_figs: plt.figure() plt.plot(feedno, g.real, 'b-', label='real') plt.plot(feedno, g.real, 'bo') plt.plot(feedno, g.imag, 'g-', label='imag') plt.plot(feedno, g.imag, 'go') plt.plot(feedno, np.abs(g), 'r-', label='abs') plt.plot(feedno, np.abs(g), 'ro') plt.xlim(feedno[0] - 1, feedno[-1] + 1) yl, yh = plt.ylim() plt.ylim(yl, yh + (yh - yl) / 5) plt.xlabel('Feed number') plt.legend() fig_name = '%s_ants_%d_%d_%s.png' % (fig_prefix, ind, fi, pol[pi]) if tag_output_iter: fig_name = output_path(fig_name, iteration=self.iteration) else: fig_name = output_path(fig_name) plt.savefig(fig_name) plt.close() # # apply outlier mask # nbl = len(bls) # lom = np.zeros((lotl_mask.shape[0], nbl), dtype=lotl_mask.dtype) # for bi, (fd1, fd2) in enumerate(bls): # b1, b2 = feedno.index(fd1), feedno.index(fd2) # lom[:, bi] = lotl_mask[:, b1, b2] # lom = mpiarray.MPIArray.wrap(lom, axis=0, comm=ts.comm) # lom = lom.redistribute(axis=1).local_array.reshape(nt, nf, 2, -1) # ts.local_vis_mask[start_ind:end_ind, :, pol.index('xx')] |= lom[:, :, 0] # ts.local_vis_mask[start_ind:end_ind, :, pol.index('yy')] |= lom[:, :, 1] # subtract the vis of calibrator from self.vis if subtract_src: nbl = len(bls) lv = np.zeros((lsrc_vis.shape[0], nbl), dtype=lsrc_vis.dtype) for bi, (fd1, fd2) in enumerate(bls): b1, b2 = feedno.index(fd1), feedno.index(fd2) lv[:, bi] = lsrc_vis[:, b1, b2] lv = mpiarray.MPIArray.wrap(lv, axis=0, comm=ts.comm) lv = lv.redistribute(axis=1).local_array.reshape(nt, nf, 2, -1) if replace_with_src: ts.local_vis[start_ind:end_ind, :, pol.index('xx')] = lv[:, :, 0] ts.local_vis[start_ind:end_ind, :, pol.index('yy')] = lv[:, :, 1] else: if 'ns_on' in ts.iterkeys(): lv[ts['ns_on'] [start_ind: end_ind]] = 0 # avoid ns_on signal to become nan ts.local_vis[start_ind:end_ind, :, pol.index('xx')] -= lv[:, :, 0] ts.local_vis[start_ind:end_ind, :, pol.index('yy')] -= lv[:, :, 1] del lv if not save_src_vis: if subtract_src: del lsrc_vis else: if tag_output_iter: src_vis_file = output_path(src_vis_file, iteration=self.iteration) else: src_vis_file = output_path(src_vis_file) # create file and allocate space first by rank0 if mpiutil.rank0: with h5py.File(src_vis_file, 'w') as f: # allocate space shp = (nt, nf, 2, nfeed, nfeed) f.create_dataset('sky_vis', shp, dtype=lsky_vis.dtype) f.create_dataset('src_vis', shp, dtype=lsrc_vis.dtype) f.create_dataset('outlier_vis', shp, dtype=lotl_vis.dtype) # f.create_dataset('outlier_mask', shp, dtype=lotl_mask.dtype) f.attrs['calibrator'] = calibrator f.attrs['dim'] = 'time, freq, pol, feed, feed' try: f.attrs['time'] = ts.time[start_ind:end_ind] except RuntimeError: f.create_dataset('time', data=ts.time[start_ind:end_ind]) f.attrs['time'] = '/time' f.attrs['freq'] = freq f.attrs['pol'] = np.array(['xx', 'yy']) f.attrs['feed'] = np.array(feedno) mpiutil.barrier() # write data to file for i in range(10): try: # NOTE: if write simultaneously, will loss data with processes distributed in several nodes for ri in xrange(mpiutil.size): if ri == mpiutil.rank: with h5py.File(src_vis_file, 'r+') as f: for ii, (ti, fi, pi) in enumerate(tfp_linds): ti_ = ti - start_ind pi_ = gain_pd[pol[pi]] f['sky_vis'][ti_, fi, pi_] = lsky_vis[ii] f['src_vis'][ti_, fi, pi_] = lsrc_vis[ii] f['outlier_vis'][ti_, fi, pi_] = lotl_vis[ii] # f['outlier_mask'][ti_, fi, pi_] = lotl_mask[ii] mpiutil.barrier() break except IOError: time.sleep(0.5) continue else: raise RuntimeError('Could not open file: %s...' % src_vis_file) del lsrc_vis del lsky_vis del lotl_vis # del lotl_mask mpiutil.barrier() if apply_gain or save_gain: # flag outliers in lGain along each feed lG_abs = np.full_like(lGain, np.nan, dtype=lGain.real.dtype) for i in range(lGain.shape[0]): valid_inds = np.where(np.isfinite(lGain[i]))[0] if len(valid_inds) > 3: vabs = np.abs(lGain[i, valid_inds]) vmed = np.median(vabs) vabs_diff = np.abs(vabs - vmed) vmad = np.median(vabs_diff) / 0.6745 if reserve_high_gain: # reserve significantly higher ones, flag only significantly lower ones lG_abs[i, valid_inds] = np.where( vmed - vabs > 3.0 * vmad, np.nan, vabs) else: # flag both significantly higher and lower ones lG_abs[i, valid_inds] = np.where( vabs_diff > 3.0 * vmad, np.nan, vabs) # choose data slice near the transit time li = max(start_ind, transit_ind - 10) - start_ind hi = min(end_ind, transit_ind + 10 + 1) - start_ind ci = transit_ind - start_ind # center index for transit_ind # compute s_top for this time range n0 = np.zeros(((hi - li), 3)) for ti, jt in enumerate(ts.time[start_ind:end_ind][li:hi]): aa.set_jultime(jt) s.compute(aa) n0[ti] = s.get_crds('top', ncrd=3) if save_phs_change: n0t = np.zeros((nt, 3)) for ti, jt in enumerate(ts.time[start_ind:end_ind]): aa.set_jultime(jt) s.compute(aa) n0t[ti] = s.get_crds('top', ncrd=3) # get the positions of feeds feedpos = ts['feedpos'][:] # wrap and redistribute Gain and flagged G_abs Gain = mpiarray.MPIArray.wrap(lGain, axis=0, comm=ts.comm) Gain = Gain.redistribute(axis=1).reshape( nt, nf, 2, None).redistribute(axis=0).reshape( None, nf * 2 * nfeed).redistribute(axis=1) G_abs = mpiarray.MPIArray.wrap(lG_abs, axis=0, comm=ts.comm) G_abs = G_abs.redistribute(axis=1).reshape( nt, nf, 2, None).redistribute(axis=0).reshape( None, nf * 2 * nfeed).redistribute(axis=1) fpd_inds = list( itertools.product(range(nf), range(2), range(nfeed))) # only for xx and yy fpd_linds = mpiutil.mpilist(fpd_inds, method='con', comm=ts.comm) del fpd_inds # create data to save the solved gain for each feed lgain = np.full((len(fpd_linds), ), cnan, dtype=Gain.dtype) # gain for each feed if save_phs_change: lphs = np.full((nt, len(fpd_linds)), np.nan, dtype=Gain.real.dtype ) # phase change with time for each feed # check for conj num_conj = 0 for ii, (fi, pi, di) in enumerate(fpd_linds): y = G_abs.local_array[li:hi, ii] inds = np.where(np.isfinite(y))[0] if len(inds) >= max(4, 0.5 * len(y)): # get the approximate magnitude by averaging the central G_abs # solve phase by least square fit ui = (feedpos[di] - feedpos[0]) * ( 1.0e6 * freq[fi] ) / const.c # position of this feed (relative to the first feed) in unit of wavelength exp_factor = np.exp(2.0J * np.pi * np.dot(n0, ui)) ef = exp_factor Gi = Gain.local_array[li:hi, ii] e_phs = np.dot(ef[inds].conj(), Gi[inds] / y[inds]) / len(inds) ea = np.abs(e_phs) e_phs_conj = np.dot(ef[inds], Gi[inds] / y[inds]) / len(inds) eac = np.abs(e_phs_conj) if eac > ea: num_conj += 1 # reduce num_conj from all processes num_conj = mpiutil.allreduce(num_conj, comm=ts.comm) if num_conj > 0.5 * (nf * 2 * nfeed): # 2 for 2 pols if mpiutil.rank0: print '!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!' print '!!! Detect data should be their conjugate... !!!' print '!!! Correct it automatically... !!!' print '!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!' mpiutil.barrier() # correct vis ts.local_vis[:] = ts.local_vis.conj() # correct G Gain.local_array[:] = Gain.local_array.conj() # solve for gain for ii, (fi, pi, di) in enumerate(fpd_linds): y = G_abs.local_array[li:hi, ii] inds = np.where(np.isfinite(y))[0] if len(inds) >= max(4, 0.5 * len(y)): # get the approximate magnitude by averaging the central G_abs mag = np.mean( y[inds] ) # = \sqrt(lambda^2 * Sc / (2 k_B)) * |gi| Ai # solve phase by least square fit ui = (feedpos[di] - feedpos[0]) * ( 1.0e6 * freq[fi] ) / const.c # position of this feed (relative to the first feed) in unit of wavelength exp_factor = np.exp(2.0J * np.pi * np.dot(n0, ui)) ef = exp_factor Gi = Gain.local_array[li:hi, ii] e_phs = np.dot(ef[inds].conj(), Gi[inds] / y[inds]) / len(inds) # the phase of gi ea = np.abs(e_phs) if np.abs(ea - 1.0) < 0.1: # compute gain for this feed lgain[ ii] = mag * e_phs # \sqrt(lambda^2 * Sc / (2 k_B)) * gi Ai if save_phs_change: lphs[:, ii] = np.angle( np.exp(-2.0J * np.pi * np.dot(n0t, ui)) * Gain.local_array[:, ii]) else: e_phs_conj = np.dot(ef[inds], Gi[inds] / y[inds]) / len(inds) eac = np.abs(e_phs_conj) if eac > ea: if np.abs(eac - 1.0) < 0.01: print 'feedno = %d, fi = %d, pol = %s: may need to be conjugated' % ( feedno[di], fi, gain_pd[pi]) else: print 'feedno = %d, fi = %d, pol = %s: maybe wrong abs(e_phs): %s' % ( feedno[di], fi, gain_pd[pi], ea) # gather local gain gain = mpiutil.gather_array(lgain, axis=0, root=None, comm=ts.comm) del lgain gain = gain.reshape(nf, 2, nfeed) if save_phs_change: phs = mpiutil.gather_array(lphs, axis=1, root=0, comm=ts.comm) del lphs if mpiutil.rank0: phs = phs.reshape(nt, nf, 2, nfeed) # normalize to get the exact gain Sc = s.get_jys(1.0e-3 * freq) # Omega = aa.ants[0].beam.Omega ### TODO: implement Omega for dish Ai = aa.ants[0].beam.response(n0[ci - li]) lmd = const.c / (1.0e6 * freq) factor = np.sqrt( (lmd**2 * 1.0e-26 * Sc) / (2 * const.k_B)) * Ai # NOTE: 1Jy = 1.0e-26 W m^-2 Hz^-1 gain /= factor[:, np.newaxis, np.newaxis] # apply gain to vis if apply_gain: for fi in range(nf): for pi in [pol.index('xx'), pol.index('yy')]: pi_ = gain_pd[pol[pi]] for bi, (fd1, fd2) in enumerate( ts['blorder'].local_data): g1 = gain[fi, pi_, feedno.index(fd1)] g2 = gain[fi, pi_, feedno.index(fd2)] if np.isfinite(g1) and np.isfinite(g2): if fd1 == fd2: # auto-correlation should be real ts.local_vis[:, fi, pi, bi] /= (g1 * np.conj(g2)).real else: ts.local_vis[:, fi, pi, bi] /= (g1 * np.conj(g2)) else: # mask the un-calibrated vis ts.local_vis_mask[:, fi, pi, bi] = True # in unit K after the calibration ts.vis.attrs['unit'] = 'K' # save gain to file if save_gain: if tag_output_iter: gain_file = output_path(gain_file, iteration=self.iteration) else: gain_file = output_path(gain_file) if mpiutil.rank0: with h5py.File(gain_file, 'w') as f: # allocate space for Gain dset = f.create_dataset('Gain', (nt, nf, 2, nfeed), dtype=Gain.dtype) dset.attrs['calibrator'] = calibrator dset.attrs['dim'] = 'time, freq, pol, feed' try: dset.attrs['time'] = ts.time[start_ind:end_ind] except RuntimeError: f.create_dataset( 'time', data=ts.time[start_ind:end_ind]) dset.attrs['time'] = '/time' dset.attrs['freq'] = freq dset.attrs['pol'] = np.array(['xx', 'yy']) dset.attrs['feed'] = np.array(feedno) dset.attrs['transit_ind'] = transit_ind # save gain dset = f.create_dataset('gain', data=gain) dset.attrs['calibrator'] = calibrator dset.attrs['dim'] = 'freq, pol, feed' dset.attrs['freq'] = freq dset.attrs['pol'] = np.array(['xx', 'yy']) dset.attrs['feed'] = np.array(feedno) # save phs if save_phs_change: f.create_dataset('phs', data=phs) mpiutil.barrier() # save Gain for i in range(10): try: # NOTE: if write simultaneously, will loss data with processes distributed in several nodes for ri in xrange(mpiutil.size): if ri == mpiutil.rank: with h5py.File(gain_file, 'r+') as f: for ii, (ti, fi, pi) in enumerate(tfp_linds): ti_ = ti - start_ind pi_ = gain_pd[pol[pi]] f['Gain'][ti_, fi, pi_] = lGain[ii] mpiutil.barrier() break except IOError: time.sleep(0.5) continue else: raise RuntimeError('Could not open file: %s...' % gain_file) mpiutil.barrier() return super(PsCal, self).process(ts)
def plot(self, vis, vis_mask, li, gi, bl, ts, **kwargs): """Function that does the actual plot work.""" integral = self.params['integral'] bl_incl = self.params['bl_incl'] bl_excl = self.params['bl_excl'] flag_mask = self.params['flag_mask'] flag_ns = self.params['flag_ns'] fig_prefix = self.params['fig_name'] rotate_xdate = self.params['rotate_xdate'] feed_no = self.params['feed_no'] tag_output_iter = self.params['tag_output_iter'] iteration = self.iteration if isinstance(ts, Timestream): # for Timestream pol = bl[0] bl = tuple(bl[1]) feed_no = True elif isinstance(ts, RawTimestream): # for RawTimestream pol = None bl = tuple(bl) if feed_no: pol = ts['bl_pol'].local_data[li] bl = tuple(ts['true_blorder'].local_data[li]) else: raise ValueError('Need either a RawTimestream or Timestream') if bl_incl != 'all': bl1 = set(bl) bl_incl = [{f1, f2} for (f1, f2) in bl_incl] bl_excl = [{f1, f2} for (f1, f2) in bl_excl] if (not bl1 in bl_incl) or (bl1 in bl_excl): return if flag_mask: vis1 = np.ma.array(vis, mask=vis_mask) elif flag_ns: if 'ns_on' in ts.iterkeys(): vis1 = vis.copy() on = np.where(ts['ns_on'][:])[0] vis1[on] = complex(np.nan, np.nan) else: vis1 = vis else: vis1 = vis if integral == 'time': vis1 = np.ma.mean(np.ma.masked_invalid(vis1), axis=0) ax_val = ts.freq[:] xlabel = r'$\nu$ / MHz' elif integral == 'freq': vis1 = np.ma.mean(np.ma.masked_invalid(vis1), axis=1) # ax_val = ts.time[:] # xlabel = r'$t$ / Julian Date' ax_val = np.array( [datetime.fromtimestamp(sec) for sec in ts['sec1970'][:]]) xlabel = '%s' % ax_val[0].date() ax_val = mdates.date2num(ax_val) else: raise ValueError('Unknown integral type %s' % integral) plt.figure() f, axarr = plt.subplots(3, sharex=True) axarr[0].plot(ax_val, vis1.real, label='real') axarr[0].legend() axarr[1].plot(ax_val, vis1.imag, label='imag') axarr[1].legend() axarr[2].plot(ax_val, np.abs(vis1), label='abs') axarr[2].legend() axarr[2].xaxis_date() date_format = mdates.DateFormatter('%H:%M') axarr[2].xaxis.set_major_formatter(date_format) if rotate_xdate: # set the x-axis tick labels to diagonal so it fits better f.autofmt_xdate() else: # reduce the number of tick locators locator = MaxNLocator(nbins=6) axarr[2].xaxis.set_major_locator(locator) axarr[2].xaxis.set_minor_locator(AutoMinorLocator(2)) axarr[2].set_xlabel(xlabel) if feed_no: fig_name = '%s_%s_%d_%d_%s.png' % (fig_prefix, integral, bl[0], bl[1], ts.pol_dict[pol]) else: fig_name = '%s_%s_%d_%d.png' % (fig_prefix, integral, bl[0], bl[1]) if tag_output_iter: fig_name = output_path(fig_name, iteration=iteration) else: fig_name = output_path(fig_name) plt.savefig(fig_name) plt.close()
def process(self, ts): calibrator = self.params['calibrator'] catalog = self.params['catalog'] span = self.params['span'] save_gain = self.params['save_gain'] gain_file = self.params['gain_file'] ts.redistribute('frequency') lfreq = ts.local_freq[:] # local freq feedno = ts['feedno'][:].tolist() pol = ts['pol'][:].tolist() bl = ts.bl[:] bls = [tuple(b) for b in bl] # # antpointing = np.radians(ts['antpointing'][-1, :, :]) # radians # transitsource = ts['transitsource'][:] # transit_time = transitsource[-1, 0] # second, sec1970 # int_time = ts.attrs['inttime'] # second # calibrator srclist, cutoff, catalogs = a.scripting.parse_srcs(calibrator, catalog) cat = a.src.get_catalog(srclist, cutoff, catalogs) assert (len(cat) == 1), 'Allow only one calibrator' s = cat.values()[0] if mpiutil.rank0: print 'Calibrating for source %s with' % calibrator, print 'strength', s._jys, 'Jy', print 'measured at', s.mfreq, 'GHz', print 'with index', s.index # get transit time of calibrator # array aa = ts.array aa.set_jultime(ts['jul_date'][0]) # the first obs time point next_transit = aa.next_transit(s) transit_time = a.phs.ephem2juldate(next_transit) # Julian date if transit_time > ts['jul_date'][-1]: local_next_transit = ephem.Date(next_transit + 8.0 * ephem.hour) raise RuntimeError( 'Data does not contain local transit time %s of source %s' % (local_next_transit, calibrator)) # the first transit index transit_inds = [np.searchsorted(ts['jul_date'][:], transit_time)] # find all other transit indices aa.set_jultime(ts['jul_date'][0] + 1.0) transit_time = a.phs.ephem2juldate(aa.next_transit(s)) # Julian date cnt = 2 while (transit_time <= ts['jul_date'][-1]): transit_inds.append( np.searchsorted(ts['jul_date'][:], transit_time)) aa.set_jultime(ts['jul_date'][0] + 1.0 * cnt) transit_time = a.phs.ephem2juldate( aa.next_transit(s)) # Julian date cnt += 1 print transit_inds ### now only use the first transit point to do the cal ### may need to improve in the future transit_ind = transit_inds[0] int_time = ts.attrs['inttime'] # second start_ind = transit_ind - np.int(span / int_time) end_ind = transit_ind + np.int(span / int_time) nt = end_ind - start_ind nfeed = len(feedno) eigval = np.empty((nt, nfeed, 2, len(lfreq)), dtype=np.float64) eigval[:] = np.nan gain = np.empty((nt, nfeed, 2, len(lfreq)), dtype=np.complex128) gain[:] = complex(np.nan, np.nan) # construct visibility matrix for a single time, pol, freq Vmat = np.zeros((nfeed, nfeed), dtype=ts.main_data.dtype) for ind, ti in enumerate(range(start_ind, end_ind)): # when noise on, just pass if 'ns_on' in ts.iterkeys() and ts['ns_on'][ti]: continue aa.set_jultime(ts['jul_date'][ti]) s.compute(aa) # get fluxes vs. freq of the calibrator Sc = s.get_jys() # get the topocentric coordinate of the calibrator at the current time s_top = s.get_crds('top', ncrd=3) aa.sim_cache(cat.get_crds( 'eq', ncrd=3)) # for compute bm_response and sim for pi in [pol.index('xx'), pol.index('yy')]: # xx, yy aa.set_active_pol(pol[pi]) for fi, freq in enumerate(lfreq): # mpi among freq for i, ai in enumerate(feedno): for j, aj in enumerate(feedno): # uij = aa.gen_uvw(i, j, src='z').squeeze() # (rj - ri)/lambda uij = aa.gen_uvw(i, j, src='z')[:, 0, :] # (rj - ri)/lambda # bmij = aa.bm_response(i, j).squeeze() # will get error for only one local freq # import pdb # pdb.set_trace() bmij = aa.bm_response(i, j).reshape(-1) try: bi = bls.index((ai, aj)) # Vmat[i, j] = ts.local_vis[ti, fi, pi, bi] / (Sc[fi] * bmij[fi] * np.exp(-2.0J * np.pi * np.dot(s_top, uij[:, fi]))) # xx, yy Vmat[i, j] = ts.local_vis[ti, fi, pi, bi] / ( Sc[fi] * bmij[fi] * np.exp(2.0J * np.pi * np.dot( s_top, uij[:, fi]))) # xx, yy except ValueError: bi = bls.index((aj, ai)) # Vmat[i, j] = np.conj(ts.local_vis[ti, fi, pi, bi] / (Sc[fi] * bmij[fi] * np.exp(-2.0J * np.pi * np.dot(s_top, uij[:, fi])))) # xx, yy Vmat[i, j] = np.conj( ts.local_vis[ti, fi, pi, bi] / (Sc[fi] * bmij[fi] * np.exp(2.0J * np.pi * np.dot( s_top, uij[:, fi])))) # xx, yy # Eigen decomposition Vmat = np.where(np.isfinite(Vmat), Vmat, 0) e, U = eigh(Vmat) eigval[ind, :, pi, fi] = e[::-1] # descending order # max eigen-val lbd = e[-1] # lambda # the gain vector for this freq gvec = np.sqrt( lbd ) * U[:, -1] # only eigen-vector corresponding to the maximum eigen-val gain[ind, :, pi, fi] = gvec # apply gain to vis # get the time mean gain tgain = np.ma.mean(np.ma.masked_invalid(gain), axis=0) # time mean tgain = mpiutil.gather_array(tgain, axis=-1, root=None) ts.redistribute('baseline') ts.pol_and_bl_data_operate(cal, tgain=tgain) # save gain if required: if save_gain: gain_file = output_path(gain_file) eigval = mpiarray.MPIArray.wrap(eigval, axis=3) gain = mpiarray.MPIArray.wrap(gain, axis=3) mem_gain = memh5.MemGroup(distributed=True) mem_gain.create_dataset('eigval', data=eigval) mem_gain.create_dataset('gain', data=gain) # add attris mem_gain.attrs['jul_data'] = ts['jul_date'][start_ind:end_ind] mem_gain.attrs['feed'] = np.array(feedno) mem_gain.attrs['pol'] = np.array(['xx', 'yy']) mem_gain.attrs['freq'] = ts.freq[:] # freq should be common # save to file mem_gain.to_hdf5(gain_file, hints=False) return super(PsCal, self).process(ts)
def process(self, ts): excl_auto = self.params['excl_auto'] plot_stats = self.params['plot_stats'] fig_prefix = self.params['fig_name'] rotate_xdate = self.params['rotate_xdate'] tag_output_iter = self.params['tag_output_iter'] ts.redistribute('baseline') if ts.local_vis_mask.ndim == 3: # RawTimestream if excl_auto: bl = ts.local_bl vis_mask = ts.local_vis_mask[:, :, bl[:, 0] != bl[:, 1]].copy() else: vis_mask = ts.local_vis_mask.copy() nt, nf, lnb = vis_mask.shape elif ts.local_vis_mask.ndim == 4: # Timestream # suppose masks are the same for all 4 pols if excl_auto: bl = ts.local_bl vis_mask = ts.local_vis_mask[:, :, 0, bl[:, 0] != bl[:, 1]].copy() else: vis_mask = ts.local_vis_mask[:, :, 0].copy() nt, nf, lnb = vis_mask.shape else: raise RuntimeError('Incorrect vis_mask shape %s' % ts.local_vis_mask.shape) # total number of bl nb = mpiutil.allreduce(lnb, comm=ts.comm) # un-mask ns-on positions if 'ns_on' in ts.iterkeys(): vis_mask[ts['ns_on'][:]] = False # statistics along time axis time_mask = np.sum(vis_mask, axis=(1, 2)).reshape(-1, 1) # gather local array to rank0 time_mask = mpiutil.gather_array(time_mask, axis=1, root=0, comm=ts.comm) if mpiutil.rank0: time_mask = np.sum(time_mask, axis=1) # statistics along time axis freq_mask = np.sum(vis_mask, axis=(0, 2)).reshape(-1, 1) # gather local array to rank0 freq_mask = mpiutil.gather_array(freq_mask, axis=1, root=0, comm=ts.comm) if mpiutil.rank0: freq_mask = np.sum(freq_mask, axis=1) if plot_stats and mpiutil.rank0: time_fig_name = '%s_%s.png' % (fig_prefix, 'time') if tag_output_iter: time_fig_name = output_path(time_fig_name, iteration=self.iteration) else: time_fig_name = output_path(time_fig_name) # plot time_mask plt.figure() fig, ax = plt.subplots() x_vals = np.array([ (datetime.utcfromtimestamp(s) + timedelta(hours=8)) for s in ts['sec1970'][:] ]) xlabel = '%s' % x_vals[0].date() x_vals = mdates.date2num(x_vals) ax.plot(x_vals, 100 * time_mask / np.float(nf * nb)) ax.xaxis_date() date_format = mdates.DateFormatter('%H:%M') ax.xaxis.set_major_formatter(date_format) if rotate_xdate: # set the x-axis tick labels to diagonal so it fits better fig.autofmt_xdate() else: # reduce the number of tick locators locator = MaxNLocator(nbins=6) ax.xaxis.set_major_locator(locator) ax.xaxis.set_minor_locator(AutoMinorLocator(2)) ax.set_xlabel(xlabel) ax.set_ylabel(r'RFI (%)') plt.savefig(time_fig_name) plt.close() freq_fig_name = '%s_%s.png' % (fig_prefix, 'freq') if tag_output_iter: freq_fig_name = output_path(freq_fig_name, iteration=self.iteration) else: freq_fig_name = output_path(freq_fig_name) # plot freq_mask plt.figure() plt.plot(ts.freq[:], 100 * freq_mask / np.float(nt * nb)) plt.grid(True) plt.xlabel(r'$\nu$ / MHz') plt.ylabel(r'RFI (%)') plt.savefig(freq_fig_name) plt.close() return super(Stats, self).process(ts)