compr_flags[p, ..., bl] = False for t in range(calibrators[pol].nTime): print '.', sys.stdout.flush() calpar[:, calibrators[pol].subsetant] = (10**(calibrators[pol].rawCalpar[t, :, 3: (3 + calibrators[pol].nAntenna)])) * np.exp(1.j * calibrators[pol].rawCalpar[t, :, (3 + calibrators[pol].nAntenna): (3 + 2 * calibrators[pol].nAntenna)]) with warnings.catch_warnings(): warnings.filterwarnings("ignore",category=RuntimeWarning) if not removeadditive: d = omni.apply_calpar(data[p,t], calpar, calibrators[pol].totalVisibilityId) else: d = omni.apply_calpar(data[p,t] - adds[pol][t], calpar, calibrators[pol].totalVisibilityId) #for t, d in enumerate(calibrators[pol].get_calibrated_data(data[p])): d[flags[pol][t]] = 0 compr_result = omni.deconvolve_spectra2(d, ~flags[pol][t], (delay_compression+1)/2, correction_weight=1e-6) compressed_data[p, t] = float(delay_compression)/nfreq * compr_result[0] compr_error[p,t] = compr_result[1] compr_error_bar = float(delay_compression)/nfreq * np.array([compr_result[3][i,i] for i in range(delay_compression)])**.5 compr_flags[p,t,compr_error_bar > min(compr_error_bar) * 1.05] = True print "Done. %fmin"%(float(time.time()-timer)/60.) sys.stdout.flush() for p, pol in enumerate(crosspols.keys()): print FILENAME + " MSG: starting compression on %s %s"%(dataano, pol), sys.stdout.flush() timer = time.time() for bl in c.subsetbl: compr_flags[p, ..., bl] = False for t in range(c.nTime): print '.',
####start compression compressed_data = np.zeros((4, len(omnifits['xx']), delay_compression, info['nUBL']), dtype='complex64') compr_flag = np.zeros((len(omnifits['xx']), delay_compression), dtype='bool') compr_var = np.zeros((4, len(omnifits['xx']), delay_compression, info['nUBL']), dtype='float32') if compress_method == 'deconvolve': print "First Round Compressing: ",#just to flag those with large compression errors (3 sigma) compr_error = np.zeros((4, len(omnifits['xx']), nfreq, info['nUBL']), dtype='complex64') for p, pol in enumerate(['xx', 'xy', 'yx', 'yy']): print pol, sys.stdout.flush() for t in range(len(omnifits['xx'])): d = np.copy(omnifits[pol][t]) d[flag[t]] = 0 compr_result = omni.deconvolve_spectra2(d, ~flag[t], (delay_compression+1)/2, var = raw_vars[pol][t,:,0], correction_weight=1e-6) compressed_data[p, t] = float(delay_compression)/nfreq * compr_result[0] compr_error[p,t] = compr_result[2] compr_error_bar = float(delay_compression)/nfreq * np.array([np.abs(compr_result[3][i,i]) for i in range(delay_compression)])**.5 compr_var[p, t] = np.outer(compr_error_bar**2 * info['ublcount'][0], 1./info['ublcount']) with warnings.catch_warnings(): warnings.filterwarnings("ignore",category=RuntimeWarning) flag = flag | (np.linalg.norm(compr_error[0]*(nfreq/(nfreq-delay_compression))**.5, axis=-1)/np.linalg.norm(raw_vars['xx']**.5,axis=-1) > 3) print "" print "Second Round Compressing: ", compressed_data = np.zeros((4, len(omnifits['xx']), delay_compression, info['nUBL']), dtype='complex64') compr_error = np.zeros((4, len(omnifits['xx']), nfreq, info['nUBL']), dtype='complex64') compr_flag = np.zeros((len(omnifits['xx']), delay_compression), dtype='bool') compr_var = np.zeros((4, len(omnifits['xx']), delay_compression, info['nUBL']), dtype='float32')
compressed_data = np.zeros(compr_shape, dtype=data.dtype) compr_flags = np.zeros(compr_shape, dtype='bool') compr_error = np.zeros((compr_shape[0], compr_shape[2], compr_shape[-1]), dtype='float32') for p, pol in enumerate(wantpols.keys()): print FILENAME + " MSG: starting compression on %s %s" % (dataano, pol), sys.stdout.flush() timer = time.time() for f in range(data.shape[2]): print '.', sys.stdout.flush() flag = rawflag[p, :, f].all(axis=-1) badbl = np.sum(rawflag[p, :, f], axis=0) > np.sum(flag) compr_result = omni.deconvolve_spectra2(data[p, :, f], ~flag, (compr_shape[1] + 1) / 2, correction_weight=1e-6) compressed_data[p, :, f] = float(compr_shape[1]) / nfreq * compr_result[0] compr_error[p, f] = compr_result[1] compr_error_bar = float(compr_shape[1]) / nfreq * np.array( [compr_result[3][i, i] for i in range(compr_shape[1])])**.5 compr_flags[p, compr_error_bar > min(compr_error_bar) * 1.05, f] = True compr_flags[p, :, f, badbl] = True print "Done. %fmin" % (float(time.time() - timer) / 60.) sys.stdout.flush() #######save uvs print FILENAME + " MSG: saving compressed uv files", sys.stdout.flush() if len(wantpols.keys()) == 1:
compr_shape = list(data.shape) compr_shape[1] = len(uvfiles) * delay_compression compressed_data = np.zeros(compr_shape, dtype=data.dtype) compr_flags = np.zeros(compr_shape, dtype='bool') compr_error = np.zeros((compr_shape[0], compr_shape[2], compr_shape[-1]), dtype='float32') for p, pol in enumerate(wantpols.keys()): print FILENAME + " MSG: starting compression on %s %s"%(dataano, pol), sys.stdout.flush() timer = time.time() for f in range(data.shape[2]): print '.', sys.stdout.flush() flag = rawflag[p, :, f].all(axis = -1) badbl = np.sum(rawflag[p, :, f], axis = 0) > np.sum(flag) compr_result = omni.deconvolve_spectra2(data[p, :, f], ~flag, (compr_shape[1]+1)/2, correction_weight=1e-6) compressed_data[p, :, f] = float(compr_shape[1])/nfreq * compr_result[0] compr_error[p,f] = compr_result[1] compr_error_bar = float(compr_shape[1])/nfreq * np.array([compr_result[3][i,i] for i in range(compr_shape[1])])**.5 compr_flags[p, compr_error_bar > min(compr_error_bar) * 1.05, f] = True compr_flags[p, :, f, badbl] = True print "Done. %fmin"%(float(time.time()-timer)/60.) sys.stdout.flush() #######save uvs print FILENAME + " MSG: saving compressed uv files", sys.stdout.flush() if len(wantpols.keys()) == 1: compr_uvfile = oppath + '/' + dataano + "_%s.uvOEE"%pol else: compr_uvfile = oppath + '/' + dataano + ".uvOEE"
t, :, (3 + calibrators[pol].nAntenna): (3 + 2 * calibrators[pol].nAntenna)]) with warnings.catch_warnings(): warnings.filterwarnings("ignore", category=RuntimeWarning) if not removeadditive: d = omni.apply_calpar(data[p, t], calpar, calibrators[pol].totalVisibilityId) else: d = omni.apply_calpar(data[p, t] - adds[pol][t], calpar, calibrators[pol].totalVisibilityId) #for t, d in enumerate(calibrators[pol].get_calibrated_data(data[p])): d[flags[pol][t]] = 0 compr_result = omni.deconvolve_spectra2( d, ~flags[pol][t], (delay_compression + 1) / 2, correction_weight=1e-6) compressed_data[ p, t] = float(delay_compression) / nfreq * compr_result[0] compr_error[p, t] = compr_result[1] compr_error_bar = float(delay_compression) / nfreq * np.array( [compr_result[3][i, i] for i in range(delay_compression)])**.5 compr_flags[p, t, compr_error_bar > min(compr_error_bar) * 1.05] = True print "Done. %fmin" % (float(time.time() - timer) / 60.) sys.stdout.flush() #######save uvs print FILENAME + " MSG: saving compressed uv files", sys.stdout.flush() if len(wantpols.keys()) == 1: compr_uvfile = oppath + '/' + dataano + "_%s.uvOE" % pol