####create redundant calibrators################
#calibrators = [omni.RedundantCalibrator(nant, info = infopaths[key]) for key in wantpols.keys()]
calibrators = [omni.RedundantCalibrator(nant) for key in wantpols.keys()]
for calibrator, key in zip(calibrators, wantpols.keys()):
calibrator.compute_redundantinfo(arrayinfoPath = arrayinfos[key])
#calibrator.write_redundantinfo(infoPath = './redundantinfo_test_' + key + '.txt', overwrite = True)
###start reading miriads################
##print FILENAME + " MSG:", len(uvfiles), "uv files to be processed for " + ano
data, t, timing, lst = omni.importuvs(uvfiles, calibrators[0].totalVisibilityId, wantpols, nTotalAntenna = 32,timingTolerance = 2*math.pi, init_mem = 5.e7)
##print FILENAME + " MSG:", len(t), "slices read."
###raw calibration################
if needrawcal:
for p, key in zip(range(len(wantpols)), wantpols.keys()):
rawcalpar = np.fromfile(rawpaths[key], dtype="complex64").reshape(nfreq, nant)
data[p] = omni.apply_calpar(data[p], rawcalpar, calibrators[p].totalVisibilityId)
#####Save various files read################
###np.savetxt('miriadextract_' + ano + "_sunpos.dat", sunpos[:len(t)], fmt='%8.5f')
##f = open(oppath + 'miriadextract_' + ano + "_localtime.dat",'w')
##for time in timing:
##f.write("%s\n"%time)
##f.close()
##f = open(oppath + 'miriadextract_' + ano + "_lsthour.dat",'w')
##for l in lst:
##f.write("%s\n"%l)
##f.close()
####calibrate################
##print FILENAME + " MSG: starting calibration."
calibrators[pol].removeDegeneracy = removedegen
calibrators[pol].convergePercent = converge_percent
calibrators[pol].maxIteration = max_iter
calibrators[pol].stepSize = step_size
timer = time.time()
calibrators[pol].compute_redundantinfo(badAntenna = badAntenna, badUBLpair = badUBLpair, antennaLocationTolerance = redundancy_tol)
print "Redundant info on %s computed in %f minutes."%(pol, (time.time() - timer)/60.)
info = calibrators[pol].Info.get_info()
###prepare rawCalpar for each calibrator and consider, if needed, raw calibration################
if need_crude_cal:
initant, solution_path, additional_solution_path, degen, _ = omni.find_solution_path(info, tol = calibrators[pol].antennaLocationTolerance, verbose = False)
crude_calpar[pol] = np.array([omni.raw_calibrate(rawdata[p, 0, f], info, initant, solution_path, additional_solution_path, degen) for f in range(calibrators[pol].nFrequency)])
data[p] = omni.apply_calpar(rawdata[p], crude_calpar[pol], calibrators[pol].totalVisibilityId)
else:
data[p] = rawdata[p]
#calibrators[pol].rawCalpar = np.zeros((calibrators[pol].nTime, calibrators[pol].nFrequency, 3 + 2 * (calibrators[pol].Info.nAntenna + calibrators[pol].Info.nUBL)),dtype='float32')
################################
########calibrate################
################################
################first round of calibration #########################
print FILENAME + " MSG: starting calibration on %s. nTime = %i, nFrequency = %i ..."%(pol, calibrators[pol].nTime, calibrators[pol].nFrequency),
sys.stdout.flush()
timer = time.time()
additivein = np.zeros_like(data[p])
calibrators[pol].logcal(data[p], additivein, verbose=True)
additiveout = calibrators[pol].lincal(data[p], additivein, verbose=True)
badAntenna=badAntenna,
badUBLpair=badUBLpair)
info = calibrators[key].Info.get_info()
calibrators[key].nTime = nt
calibrators[key].nFrequency = nfreq
###prepare rawCalpar for each calibrator and consider, if needed, raw calibration################
if need_crude_cal:
initant, solution_path, additional_solution_path, degen, _ = omni.find_solution_path(
info)
crude_calpar[key] = np.array([
omni.raw_calibrate(data[p, 0, f], info, initant, solution_path,
additional_solution_path, degen)
for f in range(calibrators[key].nFrequency)
])
data[p] = omni.apply_calpar(data[p], crude_calpar[key],
calibrators[key].totalVisibilityId)
calibrators[key].rawCalpar = np.zeros(
(calibrators[key].nTime, calibrators[key].nFrequency, 3 + 2 *
(calibrators[key].Info.nAntenna + calibrators[key].Info.nUBL)),
dtype='float32')
####calibrate################
calibrators[key].removeDegeneracy = removedegen
calibrators[key].convergePercent = converge_percent
calibrators[key].maxIteration = max_iter
calibrators[key].stepSize = step_size
################first round of calibration #########################
print FILENAME + " MSG: starting calibration on %s %s. nTime = %i, nFrequency = %i ..." % (
dataano, key, calibrators[key].nTime, calibrators[key].nFrequency),
if input_type == 'uv':
calibrators[pol] = omni.RedundantCalibrator_PAPER(aa)
elif input_type == 'odf':
calibrators[pol] = omni.RedundantCalibrator_X5(header_antloc)
calibrators[pol].read_redundantinfo(infopaths[pol], verbose=False)
info = calibrators[pol].Info.get_info()
calibrators[pol].nTime = len(timing)
calibrators[pol].nFrequency = nfreq
####consolidate 3D flags from uv files into per time/freq 2D flags
uvflags[pol] = np.any(rawflag[p,:,:,calibrators[pol].subsetbl[calibrators[pol].crossindex]], axis = 0)# aweird transpose happens when slicing
###apply, if needed, raw calibration################
if needrawcal:
original_data = np.copy(data[p])
data[p] = omni.apply_calpar(data[p], crude_calpar[pol], calibrators[pol].totalVisibilityId)
####calibrate################
calibrators[pol].removeDegeneracy = removedegen
calibrators[pol].convergePercent = converge_percent
calibrators[pol].maxIteration = max_iter
calibrators[pol].stepSize = step_size
################first round of calibration #########################
print FILENAME + " MSG: starting calibration on %s %s. nTime = %i, nFrequency = %i ..."%(dataano, pol, calibrators[pol].nTime, calibrators[pol].nFrequency),
sys.stdout.flush()
timer = time.time()
additivein = np.zeros_like(data[p])
#######################Logcal###############################
calibrators[pol].logcal(data[p], additivein, nthread = nthread, verbose=True)
omnigains = {}
adds = {}
cdata = [None,None]
for p, key in zip(range(len(data)), wantpols.keys()):
calibrators[key] = RedundantCalibrator_CHIME(len(data_pack['feed_positions']))
calibrators[key].compute_redundantinfo(data_pack, badAntenna = badAntenna, badUBLpair = badUBLpair)
info = calibrators[key].Info.get_info()
calibrators[key].nTime = nt
calibrators[key].nFrequency = nfreq
###prepare rawCalpar for each calibrator and consider, if needed, raw calibration################
if need_crude_cal:
initant, solution_path, additional_solution_path, degen, _ = omni.find_solution_path(info)
crude_calpar[key] = np.array([omni.raw_calibrate(data[p, 0, f], info, initant, solution_path, additional_solution_path, degen) for f in range(calibrators[key].nFrequency)])
data[p] = omni.apply_calpar(data[p], crude_calpar[key], calibrators[key].totalVisibilityId)
calibrators[key].rawCalpar = np.zeros((calibrators[key].nTime, calibrators[key].nFrequency, 3 + 2 * (calibrators[key].Info.nAntenna + calibrators[key].Info.nUBL)),dtype='float32')
####calibrate################
calibrators[key].removeDegeneracy = removedegen
calibrators[key].convergePercent = converge_percent
calibrators[key].maxIteration = max_iter
calibrators[key].stepSize = step_size
################first round of calibration #########################
print FILENAME + " MSG: starting calibration on %s %s. nTime = %i, nFrequency = %i ..."%(dataano, key, calibrators[key].nTime, calibrators[key].nFrequency),
sys.stdout.flush()
timer = time.time()
additivein = np.zeros_like(data[p])
calibrators[key].logcal(data[p], additivein, verbose=True)
def test_all(self):
##FILENAME = "test.py"
######################################################################
##############Config parameters###################################
######################################################################
ano = 'test'##This is the file name difference for final calibration parameter result file. Result will be saved in miriadextract_xx_ano.omnical
uvfiles = [os.path.dirname(os.path.realpath(__file__)) + '/test.uv']
wantpols = {'xx':-5}#, 'yy':-6}
#infopaths = {'xx':os.path.dirname(os.path.realpath(__file__)) + '/../doc/redundantinfo_PSA32.txt', 'yy':os.path.dirname(os.path.realpath(__file__)) + '/../doc/redundantinfo_PSA32.txt'}
arrayinfos = {'xx':os.path.dirname(os.path.realpath(__file__)) + '/../doc/arrayinfo_apprx_PAPER32_badUBLpair.txt', 'yy':os.path.dirname(os.path.realpath(__file__)) + '/../doc/arrayinfo_apprx_PAPER32_badUBLpair.txt'}
oppath = os.path.dirname(os.path.realpath(__file__)) + '/results/'
if not os.path.isdir(oppath):
os.makedirs(oppath)
removedegen = True
removeadditive = False
needrawcal = True #if true, (generally true for raw data) you need to take care of having raw calibration parameters in float32 binary format freq x nant
rawpaths = {'xx':os.path.dirname(os.path.realpath(__file__)) + "/testrawphasecalparrad_xx", 'yy':os.path.dirname(os.path.realpath(__file__)) + "/testrawphasecalparrad_yy"}
keep_binary_data = True
keep_binary_calpar = True
converge_percent = 0.000001
max_iter = 1000
step_size = .3
######################################################################
######################################################################
######################################################################
########Massage user parameters###################################
oppath += '/'
####get some info from the first uvfile ################
uv=ap.miriad.UV(uvfiles[0])
nfreq = uv.nchan;
nant = uv['nants'] / 2 # 'nants' counting ant-pols, so divide 2
startfreq = uv['sfreq']
dfreq = uv['sdf']
del(uv)
####create redundant calibrators################
#calibrators = [omni.RedundantCalibrator(nant, info = infopaths[key]) for key in wantpols.keys()]
calibrators = [omni.RedundantCalibrator(nant) for key in wantpols.keys()]
for calibrator, key in zip(calibrators, wantpols.keys()):
calibrator.compute_redundantinfo(arrayinfoPath = arrayinfos[key])
#calibrator.write_redundantinfo(infoPath = './redundantinfo_test_' + key + '.txt', overwrite = True)
###start reading miriads################
##print FILENAME + " MSG:", len(uvfiles), "uv files to be processed for " + ano
data, t, timing, lst = omni.importuvs(uvfiles, calibrators[0].totalVisibilityId, wantpols, nTotalAntenna = 32,timingTolerance = 2*math.pi, init_mem = 5.e7)
##print FILENAME + " MSG:", len(t), "slices read."
###raw calibration################
if needrawcal:
for p, key in zip(range(len(wantpols)), wantpols.keys()):
rawcalpar = np.fromfile(rawpaths[key], dtype="complex64").reshape(nfreq, nant)
data[p] = omni.apply_calpar(data[p], rawcalpar, calibrators[p].totalVisibilityId)
#####Save various files read################
###np.savetxt('miriadextract_' + ano + "_sunpos.dat", sunpos[:len(t)], fmt='%8.5f')
##f = open(oppath + 'miriadextract_' + ano + "_localtime.dat",'w')
##for time in timing:
##f.write("%s\n"%time)
##f.close()
##f = open(oppath + 'miriadextract_' + ano + "_lsthour.dat",'w')
##for l in lst:
##f.write("%s\n"%l)
##f.close()
####calibrate################
##print FILENAME + " MSG: starting calibration."
for p, calibrator in zip(range(len(wantpols)), calibrators):
calibrator.removeDegeneracy = removedegen
calibrator.removeAdditive = removeadditive
calibrator.keepData = keep_binary_data
calibrator.keepCalpar = keep_binary_calpar
calibrator.convergePercent = converge_percent
calibrator.maxIteration = max_iter
calibrator.stepSize = step_size
calibrator.computeUBLFit = False
calibrator.logcal(data[p], np.zeros_like(data[p]), verbose=True)
calibrator.lincal(data[p], np.zeros_like(data[p]), verbose=True)
calibrator.utctimes = timing
calibrator.get_calibrated_data(data[p])
calibrator.get_omnichisq()
calibrator.get_omnigain()
calibrator.get_omnifit()
#########Test results############
correctresult = np.fromfile(os.path.dirname(os.path.realpath(__file__)) + '/test.omnical', dtype = 'float32').reshape(14,203,165)[:,:,:3+2*nant]
nanmask = ~np.isnan(np.sum(correctresult,axis=2))#mask the last dimension because when data contains some 0 and some -0, C++ code return various phasecalibration parameters on different systems, when all other numbers are nan. I do the summation to avoid it failing the euqality check when the input is trivially 0s.
newresult = calibrators[-1].rawCalpar[:,:,:3+2*nant]#[:,:,3:]
#calibrators[-1].rawCalpar.tofile(os.path.dirname(os.path.realpath(__file__)) + '/test.omnical')
np.testing.assert_almost_equal(correctresult[nanmask], newresult[nanmask])#decimal=
pol, (time.time() - timer) / 60.)
info = calibrators[pol].Info.get_info()
###prepare rawCalpar for each calibrator and consider, if needed, raw calibration################
if need_crude_cal:
initant, solution_path, additional_solution_path, degen, _ = omni.find_solution_path(
info,
tol=calibrators[pol].antennaLocationTolerance,
verbose=False)
crude_calpar[pol] = np.array([
omni.raw_calibrate(rawdata[p, 0,
f], info, initant, solution_path,
additional_solution_path, degen)
for f in range(calibrators[pol].nFrequency)
])
data[p] = omni.apply_calpar(rawdata[p], crude_calpar[pol],
calibrators[pol].totalVisibilityId)
else:
data[p] = rawdata[p]
#calibrators[pol].rawCalpar = np.zeros((calibrators[pol].nTime, calibrators[pol].nFrequency, 3 + 2 * (calibrators[pol].Info.nAntenna + calibrators[pol].Info.nUBL)),dtype='float32')
################################
########calibrate################
################################
################first round of calibration #########################
print FILENAME + " MSG: starting calibration on %s. nTime = %i, nFrequency = %i ..." % (
pol, calibrators[pol].nTime, calibrators[pol].nFrequency),
sys.stdout.flush()
timer = time.time()
additivein = np.zeros_like(data[p])
calibrators[pol].logcal(data[p], additivein, verbose=True)
def test_all(self):
##FILENAME = "test.py"
######################################################################
##############Config parameters###################################
######################################################################
ano = 'test' ##This is the file name difference for final calibration parameter result file. Result will be saved in miriadextract_xx_ano.omnical
uvfiles = [os.path.dirname(os.path.realpath(__file__)) + '/test.uv']
wantpols = {'xx': -5} #, 'yy':-6}
#infopaths = {'xx':os.path.dirname(os.path.realpath(__file__)) + '/../doc/redundantinfo_PSA32.txt', 'yy':os.path.dirname(os.path.realpath(__file__)) + '/../doc/redundantinfo_PSA32.txt'}
arrayinfos = {
'xx':
os.path.dirname(os.path.realpath(__file__)) +
'/../doc/arrayinfo_apprx_PAPER32_badUBLpair.txt',
'yy':
os.path.dirname(os.path.realpath(__file__)) +
'/../doc/arrayinfo_apprx_PAPER32_badUBLpair.txt'
}
oppath = os.path.dirname(os.path.realpath(__file__)) + '/results/'
if not os.path.isdir(oppath):
os.makedirs(oppath)
removedegen = True
removeadditive = False
needrawcal = True #if true, (generally true for raw data) you need to take care of having raw calibration parameters in float32 binary format freq x nant
rawpaths = {
'xx':
os.path.dirname(os.path.realpath(__file__)) +
"/testrawphasecalparrad_xx",
'yy':
os.path.dirname(os.path.realpath(__file__)) +
"/testrawphasecalparrad_yy"
}
keep_binary_data = True
keep_binary_calpar = True
converge_percent = 0.000001
max_iter = 1000
step_size = .3
######################################################################
######################################################################
######################################################################
########Massage user parameters###################################
oppath += '/'
####get some info from the first uvfile ################
uv = ap.miriad.UV(uvfiles[0])
nfreq = uv.nchan
nant = uv['nants'] / 2 # 'nants' counting ant-pols, so divide 2
startfreq = uv['sfreq']
dfreq = uv['sdf']
del (uv)
####create redundant calibrators################
#calibrators = [omni.RedundantCalibrator(nant, info = infopaths[key]) for key in wantpols.keys()]
calibrators = [
omni.RedundantCalibrator(nant) for key in wantpols.keys()
]
for calibrator, key in zip(calibrators, wantpols.keys()):
calibrator.compute_redundantinfo(arrayinfoPath=arrayinfos[key])
#calibrator.write_redundantinfo(infoPath = './redundantinfo_test_' + key + '.txt', overwrite = True)
###start reading miriads################
##print FILENAME + " MSG:", len(uvfiles), "uv files to be processed for " + ano
data, t, timing, lst = omni.importuvs(uvfiles,
calibrators[0].totalVisibilityId,
wantpols,
nTotalAntenna=32,
timingTolerance=2 * math.pi,
init_mem=5.e7)
##print FILENAME + " MSG:", len(t), "slices read."
###raw calibration################
if needrawcal:
for p, key in zip(range(len(wantpols)), wantpols.keys()):
rawcalpar = np.fromfile(rawpaths[key],
dtype="complex64").reshape(
nfreq, nant)
data[p] = omni.apply_calpar(data[p], rawcalpar,
calibrators[p].totalVisibilityId)
#####Save various files read################
###np.savetxt('miriadextract_' + ano + "_sunpos.dat", sunpos[:len(t)], fmt='%8.5f')
##f = open(oppath + 'miriadextract_' + ano + "_localtime.dat",'w')
##for time in timing:
##f.write("%s\n"%time)
##f.close()
##f = open(oppath + 'miriadextract_' + ano + "_lsthour.dat",'w')
##for l in lst:
##f.write("%s\n"%l)
##f.close()
####calibrate################
##print FILENAME + " MSG: starting calibration."
for p, calibrator in zip(range(len(wantpols)), calibrators):
calibrator.removeDegeneracy = removedegen
calibrator.removeAdditive = removeadditive
calibrator.keepData = keep_binary_data
calibrator.keepCalpar = keep_binary_calpar
calibrator.convergePercent = converge_percent
calibrator.maxIteration = max_iter
calibrator.stepSize = step_size
calibrator.computeUBLFit = False
calibrator.logcal(data[p], np.zeros_like(data[p]), verbose=False)
additiveout = calibrator.lincal(data[p],
np.zeros_like(data[p]),
verbose=False)
calibrator.utctimes = timing
calibrator.get_calibrated_data(data[p])
calibrator.get_omnichisq()
calibrator.get_omnigain()
calibrator.get_omnifit()
#########Test results############
correctresult = np.fromfile(
os.path.dirname(os.path.realpath(__file__)) + '/test.omnical',
dtype='float32').reshape(14, 203, 165)[:, :, :]
nanmask = ~np.isnan(
np.sum(correctresult, axis=2)
) #mask the last dimension because when data contains some 0 and some -0, C++ code return various phasecalibration parameters on different systems, when all other numbers are nan. I do the summation to avoid it failing the euqality check when the input is trivially 0s.
calibrators[-1].rawCalpar.tofile(
os.path.dirname(os.path.realpath(__file__)) +
'/results/new_result.omnical')
omni.write_redundantinfo(calibrators[-1].Info.get_info(),
os.path.dirname(os.path.realpath(__file__)) +
'/results/new_info.bin',
overwrite=True)
newresult = calibrators[-1].rawCalpar[:, :, :]
np.testing.assert_almost_equal(correctresult[:, :, 1:3][nanmask],
newresult[:, :, 1:3][nanmask],
decimal=5)
np.testing.assert_almost_equal(np.sum(
np.abs(data[-1] - calibrators[-1].get_modeled_data())
[:, :,
calibrators[-1].Info.subsetbl[calibrators[-1].Info.crossindex]]**
2,
axis=2)[nanmask],
newresult[:, :, 2][nanmask],
decimal=5)
np.testing.assert_almost_equal(np.sum(
np.abs(additiveout)[:, :, calibrators[-1].Info.crossindex]**2,
axis=2)[nanmask],
newresult[:, :, 2][nanmask],
decimal=5)
np.testing.assert_almost_equal(correctresult[:, :, 3:67][nanmask],
newresult[:, :, 3:67][nanmask],
decimal=5)
np.testing.assert_almost_equal(
np.sort(np.abs(correctresult[:, :, 67:][nanmask])),
np.sort(np.abs(newresult[:, :, 67:][nanmask])),
decimal=5)
calibrators[pol] = omni.RedundantCalibrator_X5(header_antloc)
calibrators[pol].read_redundantinfo(infopaths[pol], verbose=False)
info = calibrators[pol].Info.get_info()
calibrators[pol].nTime = len(timing)
calibrators[pol].nFrequency = nfreq
####consolidate 3D flags from uv files into per time/freq 2D flags
uvflags[pol] = np.any(
rawflag[p, :, :,
calibrators[pol].subsetbl[calibrators[pol].crossindex]],
axis=0) # aweird transpose happens when slicing
###apply, if needed, raw calibration################
if needrawcal:
original_data = np.copy(data[p])
data[p] = omni.apply_calpar(data[p], crude_calpar[pol],
calibrators[pol].totalVisibilityId)
####calibrate################
calibrators[pol].removeDegeneracy = removedegen
calibrators[pol].convergePercent = converge_percent
calibrators[pol].maxIteration = max_iter
calibrators[pol].stepSize = step_size
################first round of calibration #########################
print FILENAME + " MSG: starting calibration on %s %s. nTime = %i, nFrequency = %i ..." % (
dataano, pol, calibrators[pol].nTime, calibrators[pol].nFrequency),
sys.stdout.flush()
timer = time.time()
additivein = np.zeros_like(data[p])
calibrators[pol].logcal(data[p], additivein, nthread=nthread, verbose=True)