####create redundant calibrators################ #calibrators = [omni.RedundantCalibrator(nant, info = infopaths[key]) for key in wantpols.keys()] calibrators = [omni.RedundantCalibrator(nant) for key in wantpols.keys()] for calibrator, key in zip(calibrators, wantpols.keys()): calibrator.compute_redundantinfo(arrayinfoPath = arrayinfos[key]) #calibrator.write_redundantinfo(infoPath = './redundantinfo_test_' + key + '.txt', overwrite = True) ###start reading miriads################ ##print FILENAME + " MSG:", len(uvfiles), "uv files to be processed for " + ano data, t, timing, lst = omni.importuvs(uvfiles, calibrators[0].totalVisibilityId, wantpols, nTotalAntenna = 32,timingTolerance = 2*math.pi, init_mem = 5.e7) ##print FILENAME + " MSG:", len(t), "slices read." ###raw calibration################ if needrawcal: for p, key in zip(range(len(wantpols)), wantpols.keys()): rawcalpar = np.fromfile(rawpaths[key], dtype="complex64").reshape(nfreq, nant) data[p] = omni.apply_calpar(data[p], rawcalpar, calibrators[p].totalVisibilityId) #####Save various files read################ ###np.savetxt('miriadextract_' + ano + "_sunpos.dat", sunpos[:len(t)], fmt='%8.5f') ##f = open(oppath + 'miriadextract_' + ano + "_localtime.dat",'w') ##for time in timing: ##f.write("%s\n"%time) ##f.close() ##f = open(oppath + 'miriadextract_' + ano + "_lsthour.dat",'w') ##for l in lst: ##f.write("%s\n"%l) ##f.close() ####calibrate################ ##print FILENAME + " MSG: starting calibration."
calibrators[pol].removeDegeneracy = removedegen calibrators[pol].convergePercent = converge_percent calibrators[pol].maxIteration = max_iter calibrators[pol].stepSize = step_size timer = time.time() calibrators[pol].compute_redundantinfo(badAntenna = badAntenna, badUBLpair = badUBLpair, antennaLocationTolerance = redundancy_tol) print "Redundant info on %s computed in %f minutes."%(pol, (time.time() - timer)/60.) info = calibrators[pol].Info.get_info() ###prepare rawCalpar for each calibrator and consider, if needed, raw calibration################ if need_crude_cal: initant, solution_path, additional_solution_path, degen, _ = omni.find_solution_path(info, tol = calibrators[pol].antennaLocationTolerance, verbose = False) crude_calpar[pol] = np.array([omni.raw_calibrate(rawdata[p, 0, f], info, initant, solution_path, additional_solution_path, degen) for f in range(calibrators[pol].nFrequency)]) data[p] = omni.apply_calpar(rawdata[p], crude_calpar[pol], calibrators[pol].totalVisibilityId) else: data[p] = rawdata[p] #calibrators[pol].rawCalpar = np.zeros((calibrators[pol].nTime, calibrators[pol].nFrequency, 3 + 2 * (calibrators[pol].Info.nAntenna + calibrators[pol].Info.nUBL)),dtype='float32') ################################ ########calibrate################ ################################ ################first round of calibration ######################### print FILENAME + " MSG: starting calibration on %s. nTime = %i, nFrequency = %i ..."%(pol, calibrators[pol].nTime, calibrators[pol].nFrequency), sys.stdout.flush() timer = time.time() additivein = np.zeros_like(data[p]) calibrators[pol].logcal(data[p], additivein, verbose=True) additiveout = calibrators[pol].lincal(data[p], additivein, verbose=True)
badAntenna=badAntenna, badUBLpair=badUBLpair) info = calibrators[key].Info.get_info() calibrators[key].nTime = nt calibrators[key].nFrequency = nfreq ###prepare rawCalpar for each calibrator and consider, if needed, raw calibration################ if need_crude_cal: initant, solution_path, additional_solution_path, degen, _ = omni.find_solution_path( info) crude_calpar[key] = np.array([ omni.raw_calibrate(data[p, 0, f], info, initant, solution_path, additional_solution_path, degen) for f in range(calibrators[key].nFrequency) ]) data[p] = omni.apply_calpar(data[p], crude_calpar[key], calibrators[key].totalVisibilityId) calibrators[key].rawCalpar = np.zeros( (calibrators[key].nTime, calibrators[key].nFrequency, 3 + 2 * (calibrators[key].Info.nAntenna + calibrators[key].Info.nUBL)), dtype='float32') ####calibrate################ calibrators[key].removeDegeneracy = removedegen calibrators[key].convergePercent = converge_percent calibrators[key].maxIteration = max_iter calibrators[key].stepSize = step_size ################first round of calibration ######################### print FILENAME + " MSG: starting calibration on %s %s. nTime = %i, nFrequency = %i ..." % ( dataano, key, calibrators[key].nTime, calibrators[key].nFrequency),
if input_type == 'uv': calibrators[pol] = omni.RedundantCalibrator_PAPER(aa) elif input_type == 'odf': calibrators[pol] = omni.RedundantCalibrator_X5(header_antloc) calibrators[pol].read_redundantinfo(infopaths[pol], verbose=False) info = calibrators[pol].Info.get_info() calibrators[pol].nTime = len(timing) calibrators[pol].nFrequency = nfreq ####consolidate 3D flags from uv files into per time/freq 2D flags uvflags[pol] = np.any(rawflag[p,:,:,calibrators[pol].subsetbl[calibrators[pol].crossindex]], axis = 0)# aweird transpose happens when slicing ###apply, if needed, raw calibration################ if needrawcal: original_data = np.copy(data[p]) data[p] = omni.apply_calpar(data[p], crude_calpar[pol], calibrators[pol].totalVisibilityId) ####calibrate################ calibrators[pol].removeDegeneracy = removedegen calibrators[pol].convergePercent = converge_percent calibrators[pol].maxIteration = max_iter calibrators[pol].stepSize = step_size ################first round of calibration ######################### print FILENAME + " MSG: starting calibration on %s %s. nTime = %i, nFrequency = %i ..."%(dataano, pol, calibrators[pol].nTime, calibrators[pol].nFrequency), sys.stdout.flush() timer = time.time() additivein = np.zeros_like(data[p]) #######################Logcal############################### calibrators[pol].logcal(data[p], additivein, nthread = nthread, verbose=True)
omnigains = {} adds = {} cdata = [None,None] for p, key in zip(range(len(data)), wantpols.keys()): calibrators[key] = RedundantCalibrator_CHIME(len(data_pack['feed_positions'])) calibrators[key].compute_redundantinfo(data_pack, badAntenna = badAntenna, badUBLpair = badUBLpair) info = calibrators[key].Info.get_info() calibrators[key].nTime = nt calibrators[key].nFrequency = nfreq ###prepare rawCalpar for each calibrator and consider, if needed, raw calibration################ if need_crude_cal: initant, solution_path, additional_solution_path, degen, _ = omni.find_solution_path(info) crude_calpar[key] = np.array([omni.raw_calibrate(data[p, 0, f], info, initant, solution_path, additional_solution_path, degen) for f in range(calibrators[key].nFrequency)]) data[p] = omni.apply_calpar(data[p], crude_calpar[key], calibrators[key].totalVisibilityId) calibrators[key].rawCalpar = np.zeros((calibrators[key].nTime, calibrators[key].nFrequency, 3 + 2 * (calibrators[key].Info.nAntenna + calibrators[key].Info.nUBL)),dtype='float32') ####calibrate################ calibrators[key].removeDegeneracy = removedegen calibrators[key].convergePercent = converge_percent calibrators[key].maxIteration = max_iter calibrators[key].stepSize = step_size ################first round of calibration ######################### print FILENAME + " MSG: starting calibration on %s %s. nTime = %i, nFrequency = %i ..."%(dataano, key, calibrators[key].nTime, calibrators[key].nFrequency), sys.stdout.flush() timer = time.time() additivein = np.zeros_like(data[p]) calibrators[key].logcal(data[p], additivein, verbose=True)
def test_all(self): ##FILENAME = "test.py" ###################################################################### ##############Config parameters################################### ###################################################################### ano = 'test'##This is the file name difference for final calibration parameter result file. Result will be saved in miriadextract_xx_ano.omnical uvfiles = [os.path.dirname(os.path.realpath(__file__)) + '/test.uv'] wantpols = {'xx':-5}#, 'yy':-6} #infopaths = {'xx':os.path.dirname(os.path.realpath(__file__)) + '/../doc/redundantinfo_PSA32.txt', 'yy':os.path.dirname(os.path.realpath(__file__)) + '/../doc/redundantinfo_PSA32.txt'} arrayinfos = {'xx':os.path.dirname(os.path.realpath(__file__)) + '/../doc/arrayinfo_apprx_PAPER32_badUBLpair.txt', 'yy':os.path.dirname(os.path.realpath(__file__)) + '/../doc/arrayinfo_apprx_PAPER32_badUBLpair.txt'} oppath = os.path.dirname(os.path.realpath(__file__)) + '/results/' if not os.path.isdir(oppath): os.makedirs(oppath) removedegen = True removeadditive = False needrawcal = True #if true, (generally true for raw data) you need to take care of having raw calibration parameters in float32 binary format freq x nant rawpaths = {'xx':os.path.dirname(os.path.realpath(__file__)) + "/testrawphasecalparrad_xx", 'yy':os.path.dirname(os.path.realpath(__file__)) + "/testrawphasecalparrad_yy"} keep_binary_data = True keep_binary_calpar = True converge_percent = 0.000001 max_iter = 1000 step_size = .3 ###################################################################### ###################################################################### ###################################################################### ########Massage user parameters################################### oppath += '/' ####get some info from the first uvfile ################ uv=ap.miriad.UV(uvfiles[0]) nfreq = uv.nchan; nant = uv['nants'] / 2 # 'nants' counting ant-pols, so divide 2 startfreq = uv['sfreq'] dfreq = uv['sdf'] del(uv) ####create redundant calibrators################ #calibrators = [omni.RedundantCalibrator(nant, info = infopaths[key]) for key in wantpols.keys()] calibrators = [omni.RedundantCalibrator(nant) for key in wantpols.keys()] for calibrator, key in zip(calibrators, wantpols.keys()): calibrator.compute_redundantinfo(arrayinfoPath = arrayinfos[key]) #calibrator.write_redundantinfo(infoPath = './redundantinfo_test_' + key + '.txt', overwrite = True) ###start reading miriads################ ##print FILENAME + " MSG:", len(uvfiles), "uv files to be processed for " + ano data, t, timing, lst = omni.importuvs(uvfiles, calibrators[0].totalVisibilityId, wantpols, nTotalAntenna = 32,timingTolerance = 2*math.pi, init_mem = 5.e7) ##print FILENAME + " MSG:", len(t), "slices read." ###raw calibration################ if needrawcal: for p, key in zip(range(len(wantpols)), wantpols.keys()): rawcalpar = np.fromfile(rawpaths[key], dtype="complex64").reshape(nfreq, nant) data[p] = omni.apply_calpar(data[p], rawcalpar, calibrators[p].totalVisibilityId) #####Save various files read################ ###np.savetxt('miriadextract_' + ano + "_sunpos.dat", sunpos[:len(t)], fmt='%8.5f') ##f = open(oppath + 'miriadextract_' + ano + "_localtime.dat",'w') ##for time in timing: ##f.write("%s\n"%time) ##f.close() ##f = open(oppath + 'miriadextract_' + ano + "_lsthour.dat",'w') ##for l in lst: ##f.write("%s\n"%l) ##f.close() ####calibrate################ ##print FILENAME + " MSG: starting calibration." for p, calibrator in zip(range(len(wantpols)), calibrators): calibrator.removeDegeneracy = removedegen calibrator.removeAdditive = removeadditive calibrator.keepData = keep_binary_data calibrator.keepCalpar = keep_binary_calpar calibrator.convergePercent = converge_percent calibrator.maxIteration = max_iter calibrator.stepSize = step_size calibrator.computeUBLFit = False calibrator.logcal(data[p], np.zeros_like(data[p]), verbose=True) calibrator.lincal(data[p], np.zeros_like(data[p]), verbose=True) calibrator.utctimes = timing calibrator.get_calibrated_data(data[p]) calibrator.get_omnichisq() calibrator.get_omnigain() calibrator.get_omnifit() #########Test results############ correctresult = np.fromfile(os.path.dirname(os.path.realpath(__file__)) + '/test.omnical', dtype = 'float32').reshape(14,203,165)[:,:,:3+2*nant] nanmask = ~np.isnan(np.sum(correctresult,axis=2))#mask the last dimension because when data contains some 0 and some -0, C++ code return various phasecalibration parameters on different systems, when all other numbers are nan. I do the summation to avoid it failing the euqality check when the input is trivially 0s. newresult = calibrators[-1].rawCalpar[:,:,:3+2*nant]#[:,:,3:] #calibrators[-1].rawCalpar.tofile(os.path.dirname(os.path.realpath(__file__)) + '/test.omnical') np.testing.assert_almost_equal(correctresult[nanmask], newresult[nanmask])#decimal=
pol, (time.time() - timer) / 60.) info = calibrators[pol].Info.get_info() ###prepare rawCalpar for each calibrator and consider, if needed, raw calibration################ if need_crude_cal: initant, solution_path, additional_solution_path, degen, _ = omni.find_solution_path( info, tol=calibrators[pol].antennaLocationTolerance, verbose=False) crude_calpar[pol] = np.array([ omni.raw_calibrate(rawdata[p, 0, f], info, initant, solution_path, additional_solution_path, degen) for f in range(calibrators[pol].nFrequency) ]) data[p] = omni.apply_calpar(rawdata[p], crude_calpar[pol], calibrators[pol].totalVisibilityId) else: data[p] = rawdata[p] #calibrators[pol].rawCalpar = np.zeros((calibrators[pol].nTime, calibrators[pol].nFrequency, 3 + 2 * (calibrators[pol].Info.nAntenna + calibrators[pol].Info.nUBL)),dtype='float32') ################################ ########calibrate################ ################################ ################first round of calibration ######################### print FILENAME + " MSG: starting calibration on %s. nTime = %i, nFrequency = %i ..." % ( pol, calibrators[pol].nTime, calibrators[pol].nFrequency), sys.stdout.flush() timer = time.time() additivein = np.zeros_like(data[p]) calibrators[pol].logcal(data[p], additivein, verbose=True)
def test_all(self): ##FILENAME = "test.py" ###################################################################### ##############Config parameters################################### ###################################################################### ano = 'test' ##This is the file name difference for final calibration parameter result file. Result will be saved in miriadextract_xx_ano.omnical uvfiles = [os.path.dirname(os.path.realpath(__file__)) + '/test.uv'] wantpols = {'xx': -5} #, 'yy':-6} #infopaths = {'xx':os.path.dirname(os.path.realpath(__file__)) + '/../doc/redundantinfo_PSA32.txt', 'yy':os.path.dirname(os.path.realpath(__file__)) + '/../doc/redundantinfo_PSA32.txt'} arrayinfos = { 'xx': os.path.dirname(os.path.realpath(__file__)) + '/../doc/arrayinfo_apprx_PAPER32_badUBLpair.txt', 'yy': os.path.dirname(os.path.realpath(__file__)) + '/../doc/arrayinfo_apprx_PAPER32_badUBLpair.txt' } oppath = os.path.dirname(os.path.realpath(__file__)) + '/results/' if not os.path.isdir(oppath): os.makedirs(oppath) removedegen = True removeadditive = False needrawcal = True #if true, (generally true for raw data) you need to take care of having raw calibration parameters in float32 binary format freq x nant rawpaths = { 'xx': os.path.dirname(os.path.realpath(__file__)) + "/testrawphasecalparrad_xx", 'yy': os.path.dirname(os.path.realpath(__file__)) + "/testrawphasecalparrad_yy" } keep_binary_data = True keep_binary_calpar = True converge_percent = 0.000001 max_iter = 1000 step_size = .3 ###################################################################### ###################################################################### ###################################################################### ########Massage user parameters################################### oppath += '/' ####get some info from the first uvfile ################ uv = ap.miriad.UV(uvfiles[0]) nfreq = uv.nchan nant = uv['nants'] / 2 # 'nants' counting ant-pols, so divide 2 startfreq = uv['sfreq'] dfreq = uv['sdf'] del (uv) ####create redundant calibrators################ #calibrators = [omni.RedundantCalibrator(nant, info = infopaths[key]) for key in wantpols.keys()] calibrators = [ omni.RedundantCalibrator(nant) for key in wantpols.keys() ] for calibrator, key in zip(calibrators, wantpols.keys()): calibrator.compute_redundantinfo(arrayinfoPath=arrayinfos[key]) #calibrator.write_redundantinfo(infoPath = './redundantinfo_test_' + key + '.txt', overwrite = True) ###start reading miriads################ ##print FILENAME + " MSG:", len(uvfiles), "uv files to be processed for " + ano data, t, timing, lst = omni.importuvs(uvfiles, calibrators[0].totalVisibilityId, wantpols, nTotalAntenna=32, timingTolerance=2 * math.pi, init_mem=5.e7) ##print FILENAME + " MSG:", len(t), "slices read." ###raw calibration################ if needrawcal: for p, key in zip(range(len(wantpols)), wantpols.keys()): rawcalpar = np.fromfile(rawpaths[key], dtype="complex64").reshape( nfreq, nant) data[p] = omni.apply_calpar(data[p], rawcalpar, calibrators[p].totalVisibilityId) #####Save various files read################ ###np.savetxt('miriadextract_' + ano + "_sunpos.dat", sunpos[:len(t)], fmt='%8.5f') ##f = open(oppath + 'miriadextract_' + ano + "_localtime.dat",'w') ##for time in timing: ##f.write("%s\n"%time) ##f.close() ##f = open(oppath + 'miriadextract_' + ano + "_lsthour.dat",'w') ##for l in lst: ##f.write("%s\n"%l) ##f.close() ####calibrate################ ##print FILENAME + " MSG: starting calibration." for p, calibrator in zip(range(len(wantpols)), calibrators): calibrator.removeDegeneracy = removedegen calibrator.removeAdditive = removeadditive calibrator.keepData = keep_binary_data calibrator.keepCalpar = keep_binary_calpar calibrator.convergePercent = converge_percent calibrator.maxIteration = max_iter calibrator.stepSize = step_size calibrator.computeUBLFit = False calibrator.logcal(data[p], np.zeros_like(data[p]), verbose=False) additiveout = calibrator.lincal(data[p], np.zeros_like(data[p]), verbose=False) calibrator.utctimes = timing calibrator.get_calibrated_data(data[p]) calibrator.get_omnichisq() calibrator.get_omnigain() calibrator.get_omnifit() #########Test results############ correctresult = np.fromfile( os.path.dirname(os.path.realpath(__file__)) + '/test.omnical', dtype='float32').reshape(14, 203, 165)[:, :, :] nanmask = ~np.isnan( np.sum(correctresult, axis=2) ) #mask the last dimension because when data contains some 0 and some -0, C++ code return various phasecalibration parameters on different systems, when all other numbers are nan. I do the summation to avoid it failing the euqality check when the input is trivially 0s. calibrators[-1].rawCalpar.tofile( os.path.dirname(os.path.realpath(__file__)) + '/results/new_result.omnical') omni.write_redundantinfo(calibrators[-1].Info.get_info(), os.path.dirname(os.path.realpath(__file__)) + '/results/new_info.bin', overwrite=True) newresult = calibrators[-1].rawCalpar[:, :, :] np.testing.assert_almost_equal(correctresult[:, :, 1:3][nanmask], newresult[:, :, 1:3][nanmask], decimal=5) np.testing.assert_almost_equal(np.sum( np.abs(data[-1] - calibrators[-1].get_modeled_data()) [:, :, calibrators[-1].Info.subsetbl[calibrators[-1].Info.crossindex]]** 2, axis=2)[nanmask], newresult[:, :, 2][nanmask], decimal=5) np.testing.assert_almost_equal(np.sum( np.abs(additiveout)[:, :, calibrators[-1].Info.crossindex]**2, axis=2)[nanmask], newresult[:, :, 2][nanmask], decimal=5) np.testing.assert_almost_equal(correctresult[:, :, 3:67][nanmask], newresult[:, :, 3:67][nanmask], decimal=5) np.testing.assert_almost_equal( np.sort(np.abs(correctresult[:, :, 67:][nanmask])), np.sort(np.abs(newresult[:, :, 67:][nanmask])), decimal=5)
calibrators[pol] = omni.RedundantCalibrator_X5(header_antloc) calibrators[pol].read_redundantinfo(infopaths[pol], verbose=False) info = calibrators[pol].Info.get_info() calibrators[pol].nTime = len(timing) calibrators[pol].nFrequency = nfreq ####consolidate 3D flags from uv files into per time/freq 2D flags uvflags[pol] = np.any( rawflag[p, :, :, calibrators[pol].subsetbl[calibrators[pol].crossindex]], axis=0) # aweird transpose happens when slicing ###apply, if needed, raw calibration################ if needrawcal: original_data = np.copy(data[p]) data[p] = omni.apply_calpar(data[p], crude_calpar[pol], calibrators[pol].totalVisibilityId) ####calibrate################ calibrators[pol].removeDegeneracy = removedegen calibrators[pol].convergePercent = converge_percent calibrators[pol].maxIteration = max_iter calibrators[pol].stepSize = step_size ################first round of calibration ######################### print FILENAME + " MSG: starting calibration on %s %s. nTime = %i, nFrequency = %i ..." % ( dataano, pol, calibrators[pol].nTime, calibrators[pol].nFrequency), sys.stdout.flush() timer = time.time() additivein = np.zeros_like(data[p]) calibrators[pol].logcal(data[p], additivein, nthread=nthread, verbose=True)