def chisq4tau(filename, tau_lim=30., delta_tau=0.01, save_npz=True): """ define array of possible tau values [always symmetric about zero] and impose freqency coupling. Use these to calculate the chi squared statistic: X^2(nu,t) = sum_{baselines b} V^xy_b - V^{yx}_b exp(-2 pi i nu tau) for a sweep of taus; i.e. the x-to-y delay/angle that minimzes Stokes V across the array. Arguments: -- tau_lim: sweep of taus runs (-tau_lim, taulim) -- delta_tau: spacing of the sweep -- save_npz: saves an npz of the chi square for each tau, time and frequency Returns: an array of chi squared values for every tau, time and frequency. The minimum of the tau axis will give the required minimization for V. """ print '==== '+filename+' ====' dataxy,txy,timingxy,lstxy,flagsxy = omni.importuvs([filename],{'xy':-7}, timingTolerance=1e9, init_mem=.5e9) datayx,tyx,timingyx,lstyx,flagsyx = omni.importuvs([filename],{'yx':-8}, timingTolerance=1e9, init_mem=.5e9) global tau_arr global angle_arr tau_arr = np.arange(-1*tau_lim, tau_lim, delta_tau) #nanseconds angle_arr = np.exp(-2*np.pi*1.j*np.outer(nu_arr,tau_arr)) #calculate the chi for each baseline; sum over the array chisquared = np.zeros( (tau_arr.shape[0], dataxy.shape[1], nu_arr.shape[0]) ) #tau x time x freq ###XXX TODO: VECTORIZE THIS BABY ###XXX update: I get memory errors when I try to create a baseline axis by ###### outer-producting with angle_arr. How to fix? print 'Begin stupid tau/bl loop (vectorizing is giving Mem Errs)' for i in range(angle_arr.shape[1]): # for each tau calc = 0 for bl in range(dataxy.shape[3]): #for each baseline XY = dataxy[0,:,:,bl] YX = datayx[0,:,:,bl] calc += pow(np.absolute(XY - YX*angle_arr[:,i]),2.) chisquared[i,:,:]=calc a=filename.split('.') a[0]='tau' a='.'.join(a)+'.npz' if save_npz: np.savez(a, chisquared=chisquared, angle=angle_arr, tau=tau_arr) return chisquared
import numpy as np import omnical.calibration_omni as omni import matplotlib.pyplot as plt import aipy as ap paper_uvfiles = ['/data4/paper/2012EoR/psa_live/psa6266/zen.2456266.64714.uvcRREcAC'] omni_uvfiles = ["/data4/paper/2012EoR/psa_live/forlstbinning_omnical_2/zen.2456266.64714.uvcRREcACO"] info=omni.read_redundantinfo('/data2/home/hz2ug/omnical/doc/redundantinfo_PSA64_ba19_37_50.bin') nant = 64 wantpols = {} wantpols['xx'] = ap.miriad.str2pol['xx'] papercal_data, _, _, _ = omni.importuvs(paper_uvfiles, np.concatenate([[[i,j] for i in range(j + 1)] for j in range(nant)]), wantpols, timingTolerance=100) omnical_data, _, _, _ = omni.importuvs(omni_uvfiles, np.concatenate([[[i,j] for i in range(j + 1)] for j in range(nant)]), wantpols, timingTolerance=100) omni.omniview(np.array([papercal_data[0,0,120], omnical_data[0,0,120]]), info)
elif 'yy' == key: pol_select = pol_select + [3] rawdata = np.zeros((len(pol_select), np.sum(nTimes), nfreq, nant * (nant + 1) / 2), dtype='complex64') timing = [] lst = [] for i, uvfile in enumerate(uvfiles): rawdata[:, len(timing):len(timing)+nTimes[i]] = np.fromfile(uvfile+'/visibilities', dtype='complex64').reshape((4, nTimes[i], nfreq, nant * (nant + 1) / 2))[pol_select] with open(uvfile.replace('.odf', '.odfa') + '/timing.dat') as f: for l in f.readlines(): sa.date = l.replace('\n','') sa.date += ephem.second * 3600 * 4 lst += [sa.sidereal_time()] timing += [sa.date.__str__()] else: if len(uvfiles) == 1: rawdata, t, timing, lst, rawflag = omni.importuvs(uvfiles, wantpols, totalVisibilityId = np.concatenate([[[i,j] for i in range(j + 1)] for j in range(len(aa))]), timingTolerance=100)#, nTotalAntenna = len(aa)) else: for p, pol in enumerate(wantpols.keys()): if p == 0: rawdata, t, timing, lst, rawflag = omni.importuvs([uvfiles_dic[pol]], {pol:wantpols[pol]}, totalVisibilityId = np.concatenate([[[i,j] for i in range(j + 1)] for j in range(len(aa))]), timingTolerance=100) else: tmpdata, t, timing, lst, tmpflag = omni.importuvs([uvfiles_dic[pol]], {pol:wantpols[pol]}, totalVisibilityId = np.concatenate([[[i,j] for i in range(j + 1)] for j in range(len(aa))]), timingTolerance=100) rawdata = np.concatenate((rawdata, tmpdata)) print FILENAME + " MSG:", len(t), "slices read." sys.stdout.flush() sun = ephem.Sun() sunpos = np.zeros((len(timing), 2)) southern_points = {'hyd':{'ra': '09:18:05.7', 'dec': '-12:05:44'}, 'cen':{'ra': '13:25:27.6', 'dec': '-43:01:09'},
uv=ap.miriad.UV(uvfiles[0]) nfreq = uv.nchan; nant = uv['nants'] / 2 # 'nants' counting ant-pols, so divide 2 startfreq = uv['sfreq'] dfreq = uv['sdf'] del(uv) ####create redundant calibrators################ #calibrators = [omni.RedundantCalibrator(nant, info = infopaths[key]) for key in wantpols.keys()] calibrators = [omni.RedundantCalibrator(nant) for key in wantpols.keys()] for calibrator, key in zip(calibrators, wantpols.keys()): calibrator.compute_redundantinfo(arrayinfoPath = arrayinfos[key]) #calibrator.write_redundantinfo(infoPath = './redundantinfo_test_' + key + '.txt', overwrite = True) ###start reading miriads################ ##print FILENAME + " MSG:", len(uvfiles), "uv files to be processed for " + ano data, t, timing, lst = omni.importuvs(uvfiles, calibrators[0].totalVisibilityId, wantpols, nTotalAntenna = 32,timingTolerance = 2*math.pi, init_mem = 5.e7) ##print FILENAME + " MSG:", len(t), "slices read." ###raw calibration################ if needrawcal: for p, key in zip(range(len(wantpols)), wantpols.keys()): rawcalpar = np.fromfile(rawpaths[key], dtype="complex64").reshape(nfreq, nant) data[p] = omni.apply_calpar(data[p], rawcalpar, calibrators[p].totalVisibilityId) #####Save various files read################ ###np.savetxt('miriadextract_' + ano + "_sunpos.dat", sunpos[:len(t)], fmt='%8.5f') ##f = open(oppath + 'miriadextract_' + ano + "_localtime.dat",'w') ##for time in timing: ##f.write("%s\n"%time) ##f.close() ##f = open(oppath + 'miriadextract_' + ano + "_lsthour.dat",'w')
badUBLpair.append([aa.ant_layout[0, 0], aa.ant_layout[r, c]]) badUBLpair.append([aa.ant_layout[-1, 0], aa.ant_layout[r, c]]) badAntenna_x = [8, 14, 15, 16, 26, 27, 28, 34, 37, 38, 42, 46, 50, 53, 72, 74, 82, 84, 85, 110] badAntenna_y = [3, 7, 15, 16, 23, 26, 27, 34, 38, 46, 50, 56, 57, 72, 100] badAntenna = sorted(np.unique(badAntenna_x + badAntenna_y).tolist()) rc.compute_redundantinfo(badAntenna=badAntenna, badUBLpair=badUBLpair, antennaLocationTolerance=.3) ##start reading data rawdata = {} rawflag = {} for p, pol in enumerate(['xx', 'yy']): data_list = glob.glob(datadir+'*.%s.uv'%pol) print len(data_list) rawdata[pol], t, timing, raw_lst, rawflag[pol] = omni.importuvs(data_list, {pol: ap.miriad.str2pol[pol]}, totalVisibilityId=rc.Info.totalVisibilityId[rc.Info.subsetbl[rc.Info.crossindex]], timingTolerance=0.001) #reorder and sort lst rawdata[pol] = rawdata[pol][:, np.argsort(raw_lst)] rawflag[pol] = rawflag[pol][:, np.argsort(raw_lst)] raw_lst = sorted(raw_lst) plt.plot(raw_lst) plt.show() nf = rawdata['xx'].shape[2] nbl = rawdata['xx'].shape[3] uv = ap.miriad.UV(data_list[0]) freqs = (uv['freq'] + np.arange(nf) * uv['sdf']) * 1.e3 t_chunks = len(raw_lst) / t_avg lst = np.mean(np.array(raw_lst)[:t_chunks * t_avg].reshape((t_chunks, t_avg)), axis=-1)
sa = ephem.Observer() sa.lon = uv['longitu'] sa.lat = uv['latitud'] sa.pressure = 0 startfreq = uv['sfreq'] dfreq = uv['sdf'] del (uv) print "Done." sys.stdout.flush() ###start reading miriads################ print FILENAME + " MSG:", len(uvfiles), "uv files to be processed" sys.stdout.flush() rawdata, t, timing, lst = omni.importuvs( uvfiles, np.concatenate([[[i, j] for i in range(j + 1)] for j in range(len(aa))]), wantpols, timingTolerance=100) #, nTotalAntenna = len(aa)) print FILENAME + " MSG:", len(t), "slices read." sys.stdout.flush() sun = ephem.Sun() sunpos = np.zeros((len(timing), 2)) southern_points = { 'hyd': { 'ra': '09:18:05.7', 'dec': '-12:05:44' }, 'cen': { 'ra': '13:25:27.6', 'dec': '-43:01:09'
import numpy as np import omnical.calibration_omni as omni import matplotlib.pyplot as plt import aipy as ap paper_uvfiles = [ '/data4/paper/2012EoR/psa_live/psa6266/zen.2456266.64714.uvcRREcAC' ] omni_uvfiles = [ "/data4/paper/2012EoR/psa_live/forlstbinning_omnical_2/zen.2456266.64714.uvcRREcACO" ] info = omni.read_redundantinfo( '/data2/home/hz2ug/omnical/doc/redundantinfo_PSA64_ba19_37_50.bin') nant = 64 wantpols = {} wantpols['xx'] = ap.miriad.str2pol['xx'] papercal_data, _, _, _, rawflag = omni.importuvs(paper_uvfiles, wantpols, timingTolerance=100) omnical_data, _, _, _, rawflag = omni.importuvs(omni_uvfiles, wantpols, timingTolerance=100) data = omni.omniview( np.array([papercal_data[0, 0, 120], omnical_data[0, 0, 120]]), info) np.savez('/data2/home/zakiali/jeff_psa64/omniview.npz', data)
def test_all(self): ##FILENAME = "test.py" ###################################################################### ##############Config parameters################################### ###################################################################### ano = 'test'##This is the file name difference for final calibration parameter result file. Result will be saved in miriadextract_xx_ano.omnical uvfiles = [os.path.dirname(os.path.realpath(__file__)) + '/test.uv'] wantpols = {'xx':-5}#, 'yy':-6} #infopaths = {'xx':os.path.dirname(os.path.realpath(__file__)) + '/../doc/redundantinfo_PSA32.txt', 'yy':os.path.dirname(os.path.realpath(__file__)) + '/../doc/redundantinfo_PSA32.txt'} arrayinfos = {'xx':os.path.dirname(os.path.realpath(__file__)) + '/../doc/arrayinfo_apprx_PAPER32_badUBLpair.txt', 'yy':os.path.dirname(os.path.realpath(__file__)) + '/../doc/arrayinfo_apprx_PAPER32_badUBLpair.txt'} oppath = os.path.dirname(os.path.realpath(__file__)) + '/results/' if not os.path.isdir(oppath): os.makedirs(oppath) removedegen = True removeadditive = False needrawcal = True #if true, (generally true for raw data) you need to take care of having raw calibration parameters in float32 binary format freq x nant rawpaths = {'xx':os.path.dirname(os.path.realpath(__file__)) + "/testrawphasecalparrad_xx", 'yy':os.path.dirname(os.path.realpath(__file__)) + "/testrawphasecalparrad_yy"} keep_binary_data = True keep_binary_calpar = True converge_percent = 0.000001 max_iter = 1000 step_size = .3 ###################################################################### ###################################################################### ###################################################################### ########Massage user parameters################################### oppath += '/' ####get some info from the first uvfile ################ uv=ap.miriad.UV(uvfiles[0]) nfreq = uv.nchan; nant = uv['nants'] / 2 # 'nants' counting ant-pols, so divide 2 startfreq = uv['sfreq'] dfreq = uv['sdf'] del(uv) ####create redundant calibrators################ #calibrators = [omni.RedundantCalibrator(nant, info = infopaths[key]) for key in wantpols.keys()] calibrators = [omni.RedundantCalibrator(nant) for key in wantpols.keys()] for calibrator, key in zip(calibrators, wantpols.keys()): calibrator.compute_redundantinfo(arrayinfoPath = arrayinfos[key]) #calibrator.write_redundantinfo(infoPath = './redundantinfo_test_' + key + '.txt', overwrite = True) ###start reading miriads################ ##print FILENAME + " MSG:", len(uvfiles), "uv files to be processed for " + ano data, t, timing, lst = omni.importuvs(uvfiles, calibrators[0].totalVisibilityId, wantpols, nTotalAntenna = 32,timingTolerance = 2*math.pi, init_mem = 5.e7) ##print FILENAME + " MSG:", len(t), "slices read." ###raw calibration################ if needrawcal: for p, key in zip(range(len(wantpols)), wantpols.keys()): rawcalpar = np.fromfile(rawpaths[key], dtype="complex64").reshape(nfreq, nant) data[p] = omni.apply_calpar(data[p], rawcalpar, calibrators[p].totalVisibilityId) #####Save various files read################ ###np.savetxt('miriadextract_' + ano + "_sunpos.dat", sunpos[:len(t)], fmt='%8.5f') ##f = open(oppath + 'miriadextract_' + ano + "_localtime.dat",'w') ##for time in timing: ##f.write("%s\n"%time) ##f.close() ##f = open(oppath + 'miriadextract_' + ano + "_lsthour.dat",'w') ##for l in lst: ##f.write("%s\n"%l) ##f.close() ####calibrate################ ##print FILENAME + " MSG: starting calibration." for p, calibrator in zip(range(len(wantpols)), calibrators): calibrator.removeDegeneracy = removedegen calibrator.removeAdditive = removeadditive calibrator.keepData = keep_binary_data calibrator.keepCalpar = keep_binary_calpar calibrator.convergePercent = converge_percent calibrator.maxIteration = max_iter calibrator.stepSize = step_size calibrator.computeUBLFit = False calibrator.logcal(data[p], np.zeros_like(data[p]), verbose=True) calibrator.lincal(data[p], np.zeros_like(data[p]), verbose=True) calibrator.utctimes = timing calibrator.get_calibrated_data(data[p]) calibrator.get_omnichisq() calibrator.get_omnigain() calibrator.get_omnifit() #########Test results############ correctresult = np.fromfile(os.path.dirname(os.path.realpath(__file__)) + '/test.omnical', dtype = 'float32').reshape(14,203,165)[:,:,:3+2*nant] nanmask = ~np.isnan(np.sum(correctresult,axis=2))#mask the last dimension because when data contains some 0 and some -0, C++ code return various phasecalibration parameters on different systems, when all other numbers are nan. I do the summation to avoid it failing the euqality check when the input is trivially 0s. newresult = calibrators[-1].rawCalpar[:,:,:3+2*nant]#[:,:,3:] #calibrators[-1].rawCalpar.tofile(os.path.dirname(os.path.realpath(__file__)) + '/test.omnical') np.testing.assert_almost_equal(correctresult[nanmask], newresult[nanmask])#decimal=
lst = [] for i, uvfile in enumerate(uvfiles): rawdata[:, len(timing):len(timing) + nTimes[i]] = np.fromfile( uvfile + '/visibilities', dtype='complex64').reshape( (4, nTimes[i], nfreq, nant * (nant + 1) / 2))[pol_select] with open(uvfile.replace('.odf', '.odfa') + '/timing.dat') as f: for l in f.readlines(): sa.date = l.replace('\n', '') sa.date += ephem.second * 3600 * 4 lst += [sa.sidereal_time()] timing += [sa.date.__str__()] else: if len(uvfiles) == 1: rawdata, t, timing, lst, rawflag = omni.importuvs( uvfiles, wantpols, totalVisibilityId=np.concatenate([[[i, j] for i in range(j + 1)] for j in range(len(aa))]), timingTolerance=100) #, nTotalAntenna = len(aa)) else: for p, pol in enumerate(wantpols.keys()): if p == 0: rawdata, t, timing, lst, rawflag = omni.importuvs( [uvfiles_dic[pol]], {pol: wantpols[pol]}, totalVisibilityId=np.concatenate([[[i, j] for i in range(j + 1)] for j in range(len(aa)) ]), timingTolerance=100) else: tmpdata, t, timing, lst, tmpflag = omni.importuvs(
sa.lon = aa.lon sa.lat = aa.lat startfreq = uv['sfreq']#GHz dfreq = uv['sdf']#GHz wantpols = {} for pol in omni.get_uv_pols(uv): wantpols[pol] = ap.miriad.str2pol[pol] del(uv) print "Done." sys.stdout.flush() #######import data file########################### print FILENAME + " MSG:", len(uvfiles), "uv files to be processed for " + ano sys.stdout.flush() data, jd, timing, lst, rawflag = omni.importuvs(uvfiles, wantpols, totalVisibilityId = np.concatenate([[[i,j] for i in range(j + 1)] for j in range(len(aa))]), timingTolerance=100, init_mem=init_mem, lat = sa.lat, lon=sa.lon)#, nTotalAntenna = len(aa)) if delay_compression > len(jd) / len(uvfiles): raise ValueError("Desired time bins %i is larger than input stamp count per uvfile."%len(jd)) data[rawflag] = 0 print FILENAME + " MSG:", len(jd), "slices read. data shape: ", data.shape sys.stdout.flush() ########compress########################### compr_shape = list(data.shape) compr_shape[1] = len(uvfiles) * delay_compression compressed_data = np.zeros(compr_shape, dtype=data.dtype) compr_flags = np.zeros(compr_shape, dtype='bool') compr_error = np.zeros((compr_shape[0], compr_shape[2], compr_shape[-1]), dtype='float32') for p, pol in enumerate(wantpols.keys()):
import matplotlib.pyplot as plt import aipy as ap paper_uvfiles = [ '/data4/paper/2012EoR/psa_live/psa6266/zen.2456266.64714.uvcRREcAC' ] omni_uvfiles = [ "/data4/paper/2012EoR/psa_live/forlstbinning_omnical_2/zen.2456266.64714.uvcRREcACO" ] info = omni.read_redundantinfo( '/data2/home/hz2ug/omnical/doc/redundantinfo_PSA64_ba19_37_50.bin') nant = 64 wantpols = {} wantpols['xx'] = ap.miriad.str2pol['xx'] papercal_data, _, _, _ = omni.importuvs(paper_uvfiles, np.concatenate([[[i, j] for i in range(j + 1)] for j in range(nant)]), wantpols, timingTolerance=100) omnical_data, _, _, _ = omni.importuvs(omni_uvfiles, np.concatenate([[[i, j] for i in range(j + 1)] for j in range(nant)]), wantpols, timingTolerance=100) omni.omniview(np.array([papercal_data[0, 0, 120], omnical_data[0, 0, 120]]), info)
sa.lon = uv['longitu'] sa.lat = uv['latitud'] sa.pressure = 0 startfreq = uv['sfreq'] dfreq = uv['sdf'] del(uv) print "Done." sys.stdout.flush() ###start reading miriads################ print FILENAME + " MSG:", len(uvfiles), "uv files to be processed" sys.stdout.flush() rawdata, t, timing, lst = omni.importuvs(uvfiles, np.concatenate([[[i,j] for i in range(j + 1)] for j in range(len(aa))]), wantpols, timingTolerance=100)#, nTotalAntenna = len(aa)) print FILENAME + " MSG:", len(t), "slices read." sys.stdout.flush() sun = ephem.Sun() sunpos = np.zeros((len(timing), 2)) southern_points = {'hyd':{'ra': '09:18:05.7', 'dec': '-12:05:44'}, 'cen':{'ra': '13:25:27.6', 'dec': '-43:01:09'}, 'cyg':{'ra': '19:59:28.3', 'dec': '40:44:02'}, 'pic':{'ra': '05:19:49.7', 'dec': '-45:46:44'}, 'vir':{'ra': '12:30:49.4', 'dec': '12:23:28'}, 'for':{'ra': '03:22:41.7', 'dec': '-37:12:30'}} for source in southern_points.keys(): southern_points[source]['body'] = ephem.FixedBody() southern_points[source]['body']._ra = southern_points[source]['ra']
import numpy as np import omnical.calibration_omni as omni import matplotlib.pyplot as plt import aipy as ap paper_uvfiles = ['/data4/paper/2012EoR/psa_live/psa6266/zen.2456266.64714.uvcRREcAC'] omni_uvfiles = ["/data4/paper/2012EoR/psa_live/forlstbinning_omnical_2/zen.2456266.64714.uvcRREcACO"] info=omni.read_redundantinfo('/data2/home/hz2ug/omnical/doc/redundantinfo_PSA64_ba19_37_50.bin') nant = 64 wantpols = {} wantpols['xx'] = ap.miriad.str2pol['xx'] papercal_data, _, _, _, rawflag = omni.importuvs(paper_uvfiles, wantpols, timingTolerance=100) omnical_data, _, _, _, rawflag = omni.importuvs(omni_uvfiles, wantpols, timingTolerance=100) data = omni.omniview(np.array([papercal_data[0,0,120], omnical_data[0,0,120]]), info) np.savez('/data2/home/zakiali/jeff_psa64/omniview.npz', data)
def chisq4tau(filename, tau_lim=30., delta_tau=0.01, save_npz=True): """ define array of possible tau values [always symmetric about zero] and impose freqency coupling. Use these to calculate the chi squared statistic: X^2(nu,t) = sum_{baselines b} V^xy_b - V^{yx}_b exp(-2 pi i nu tau) for a sweep of taus; i.e. the x-to-y delay/angle that minimzes Stokes V across the array. Arguments: -- tau_lim: sweep of taus runs (-tau_lim, taulim) -- delta_tau: spacing of the sweep -- save_npz: saves an npz of the chi square for each tau, time and frequency Returns: an array of chi squared values for every tau, time and frequency. The minimum of the tau axis will give the required minimization for V. """ print '==== ' + filename + ' ====' dataxy, txy, timingxy, lstxy, flagsxy = omni.importuvs([filename], {'xy': -7}, timingTolerance=1e9, init_mem=.5e9) datayx, tyx, timingyx, lstyx, flagsyx = omni.importuvs([filename], {'yx': -8}, timingTolerance=1e9, init_mem=.5e9) global tau_arr global angle_arr tau_arr = np.arange(-1 * tau_lim, tau_lim, delta_tau) #nanseconds angle_arr = np.exp(-2 * np.pi * 1.j * np.outer(nu_arr, tau_arr)) #calculate the chi for each baseline; sum over the array chisquared = np.zeros((tau_arr.shape[0], dataxy.shape[1], nu_arr.shape[0])) #tau x time x freq ###XXX TODO: VECTORIZE THIS BABY ###XXX update: I get memory errors when I try to create a baseline axis by ###### outer-producting with angle_arr. How to fix? print 'Begin stupid tau/bl loop (vectorizing is giving Mem Errs)' for i in range(angle_arr.shape[1]): # for each tau calc = 0 for bl in range(dataxy.shape[3]): #for each baseline XY = dataxy[0, :, :, bl] YX = datayx[0, :, :, bl] calc += pow(np.absolute(XY - YX * angle_arr[:, i]), 2.) chisquared[i, :, :] = calc a = filename.split('.') a[0] = 'tau' a = '.'.join(a) + '.npz' if save_npz: np.savez(a, chisquared=chisquared, angle=angle_arr, tau=tau_arr) return chisquared
def test_all(self): ##FILENAME = "test.py" ###################################################################### ##############Config parameters################################### ###################################################################### ano = 'test' ##This is the file name difference for final calibration parameter result file. Result will be saved in miriadextract_xx_ano.omnical uvfiles = [os.path.dirname(os.path.realpath(__file__)) + '/test.uv'] wantpols = {'xx': -5} #, 'yy':-6} #infopaths = {'xx':os.path.dirname(os.path.realpath(__file__)) + '/../doc/redundantinfo_PSA32.txt', 'yy':os.path.dirname(os.path.realpath(__file__)) + '/../doc/redundantinfo_PSA32.txt'} arrayinfos = { 'xx': os.path.dirname(os.path.realpath(__file__)) + '/../doc/arrayinfo_apprx_PAPER32_badUBLpair.txt', 'yy': os.path.dirname(os.path.realpath(__file__)) + '/../doc/arrayinfo_apprx_PAPER32_badUBLpair.txt' } oppath = os.path.dirname(os.path.realpath(__file__)) + '/results/' if not os.path.isdir(oppath): os.makedirs(oppath) removedegen = True removeadditive = False needrawcal = True #if true, (generally true for raw data) you need to take care of having raw calibration parameters in float32 binary format freq x nant rawpaths = { 'xx': os.path.dirname(os.path.realpath(__file__)) + "/testrawphasecalparrad_xx", 'yy': os.path.dirname(os.path.realpath(__file__)) + "/testrawphasecalparrad_yy" } keep_binary_data = True keep_binary_calpar = True converge_percent = 0.000001 max_iter = 1000 step_size = .3 ###################################################################### ###################################################################### ###################################################################### ########Massage user parameters################################### oppath += '/' ####get some info from the first uvfile ################ uv = ap.miriad.UV(uvfiles[0]) nfreq = uv.nchan nant = uv['nants'] / 2 # 'nants' counting ant-pols, so divide 2 startfreq = uv['sfreq'] dfreq = uv['sdf'] del (uv) ####create redundant calibrators################ #calibrators = [omni.RedundantCalibrator(nant, info = infopaths[key]) for key in wantpols.keys()] calibrators = [ omni.RedundantCalibrator(nant) for key in wantpols.keys() ] for calibrator, key in zip(calibrators, wantpols.keys()): calibrator.compute_redundantinfo(arrayinfoPath=arrayinfos[key]) #calibrator.write_redundantinfo(infoPath = './redundantinfo_test_' + key + '.txt', overwrite = True) ###start reading miriads################ ##print FILENAME + " MSG:", len(uvfiles), "uv files to be processed for " + ano data, t, timing, lst = omni.importuvs(uvfiles, calibrators[0].totalVisibilityId, wantpols, nTotalAntenna=32, timingTolerance=2 * math.pi, init_mem=5.e7) ##print FILENAME + " MSG:", len(t), "slices read." ###raw calibration################ if needrawcal: for p, key in zip(range(len(wantpols)), wantpols.keys()): rawcalpar = np.fromfile(rawpaths[key], dtype="complex64").reshape( nfreq, nant) data[p] = omni.apply_calpar(data[p], rawcalpar, calibrators[p].totalVisibilityId) #####Save various files read################ ###np.savetxt('miriadextract_' + ano + "_sunpos.dat", sunpos[:len(t)], fmt='%8.5f') ##f = open(oppath + 'miriadextract_' + ano + "_localtime.dat",'w') ##for time in timing: ##f.write("%s\n"%time) ##f.close() ##f = open(oppath + 'miriadextract_' + ano + "_lsthour.dat",'w') ##for l in lst: ##f.write("%s\n"%l) ##f.close() ####calibrate################ ##print FILENAME + " MSG: starting calibration." for p, calibrator in zip(range(len(wantpols)), calibrators): calibrator.removeDegeneracy = removedegen calibrator.removeAdditive = removeadditive calibrator.keepData = keep_binary_data calibrator.keepCalpar = keep_binary_calpar calibrator.convergePercent = converge_percent calibrator.maxIteration = max_iter calibrator.stepSize = step_size calibrator.computeUBLFit = False calibrator.logcal(data[p], np.zeros_like(data[p]), verbose=False) additiveout = calibrator.lincal(data[p], np.zeros_like(data[p]), verbose=False) calibrator.utctimes = timing calibrator.get_calibrated_data(data[p]) calibrator.get_omnichisq() calibrator.get_omnigain() calibrator.get_omnifit() #########Test results############ correctresult = np.fromfile( os.path.dirname(os.path.realpath(__file__)) + '/test.omnical', dtype='float32').reshape(14, 203, 165)[:, :, :] nanmask = ~np.isnan( np.sum(correctresult, axis=2) ) #mask the last dimension because when data contains some 0 and some -0, C++ code return various phasecalibration parameters on different systems, when all other numbers are nan. I do the summation to avoid it failing the euqality check when the input is trivially 0s. calibrators[-1].rawCalpar.tofile( os.path.dirname(os.path.realpath(__file__)) + '/results/new_result.omnical') omni.write_redundantinfo(calibrators[-1].Info.get_info(), os.path.dirname(os.path.realpath(__file__)) + '/results/new_info.bin', overwrite=True) newresult = calibrators[-1].rawCalpar[:, :, :] np.testing.assert_almost_equal(correctresult[:, :, 1:3][nanmask], newresult[:, :, 1:3][nanmask], decimal=5) np.testing.assert_almost_equal(np.sum( np.abs(data[-1] - calibrators[-1].get_modeled_data()) [:, :, calibrators[-1].Info.subsetbl[calibrators[-1].Info.crossindex]]** 2, axis=2)[nanmask], newresult[:, :, 2][nanmask], decimal=5) np.testing.assert_almost_equal(np.sum( np.abs(additiveout)[:, :, calibrators[-1].Info.crossindex]**2, axis=2)[nanmask], newresult[:, :, 2][nanmask], decimal=5) np.testing.assert_almost_equal(correctresult[:, :, 3:67][nanmask], newresult[:, :, 3:67][nanmask], decimal=5) np.testing.assert_almost_equal( np.sort(np.abs(correctresult[:, :, 67:][nanmask])), np.sort(np.abs(newresult[:, :, 67:][nanmask])), decimal=5)
dfreq = uv['sdf'] #GHz wantpols = {} for pol in omni.get_uv_pols(uv): wantpols[pol] = ap.miriad.str2pol[pol] del (uv) print "Done." sys.stdout.flush() #######import data file########################### print FILENAME + " MSG:", len(uvfiles), "uv files to be processed for " + ano sys.stdout.flush() data, jd, timing, lst, rawflag = omni.importuvs( uvfiles, wantpols, totalVisibilityId=np.concatenate([[[i, j] for i in range(j + 1)] for j in range(len(aa))]), timingTolerance=100, init_mem=init_mem, lat=sa.lat, lon=sa.lon) #, nTotalAntenna = len(aa)) if delay_compression > len(jd) / len(uvfiles): raise ValueError( "Desired time bins %i is larger than input stamp count per uvfile." % len(jd)) data[rawflag] = 0 print FILENAME + " MSG:", len(jd), "slices read. data shape: ", data.shape sys.stdout.flush() ########compress########################### compr_shape = list(data.shape)