########Massage user parameters################################### oppath += '/' ####get some info from the first uvfile ################ uv=ap.miriad.UV(uvfiles[0]) nfreq = uv.nchan; nant = uv['nants'] / 2 # 'nants' counting ant-pols, so divide 2 startfreq = uv['sfreq'] dfreq = uv['sdf'] del(uv) ####create redundant calibrators################ #calibrators = [omni.RedundantCalibrator(nant, info = infopaths[key]) for key in wantpols.keys()] calibrators = [omni.RedundantCalibrator(nant) for key in wantpols.keys()] for calibrator, key in zip(calibrators, wantpols.keys()): calibrator.compute_redundantinfo(arrayinfoPath = arrayinfos[key]) #calibrator.write_redundantinfo(infoPath = './redundantinfo_test_' + key + '.txt', overwrite = True) ###start reading miriads################ ##print FILENAME + " MSG:", len(uvfiles), "uv files to be processed for " + ano data, t, timing, lst = omni.importuvs(uvfiles, calibrators[0].totalVisibilityId, wantpols, nTotalAntenna = 32,timingTolerance = 2*math.pi, init_mem = 5.e7) ##print FILENAME + " MSG:", len(t), "slices read." ###raw calibration################ if needrawcal: for p, key in zip(range(len(wantpols)), wantpols.keys()): rawcalpar = np.fromfile(rawpaths[key], dtype="complex64").reshape(nfreq, nant) data[p] = omni.apply_calpar(data[p], rawcalpar, calibrators[p].totalVisibilityId) #####Save various files read################
#if os.path.isfile(opts.path):
#raise Exception('Error: output filename exists!')
try:
badAntenna = [int(i) for i in opts.ba.split(',')]
except:
badAntenna = []
try:
if opts.bu != '':
badUBLpair = [[int(j) for j in i.split('.')] for i in opts.bu.split(',')]
else:
badUBLpair = []
except:
badUBLpair = []
print 'Bad Antennas:', badAntenna
print 'Bad unique baselines:', badUBLpair
nant = opts.antenna
calibrator = omni.RedundantCalibrator(nant)
calibrator.badAntenna = badAntenna
calibrator.badUBLpair = badUBLpair
calibrator.antennaLocationTolerance = opts.tol
timer = time.time()
calibrator.compute_redundantinfo(verbose = True)
print "Redundant info computed in %f minutes. %i good antenna, %i good UBL. Writing to %s..."%((time.time() - timer)/60., calibrator.Info.nAntenna, calibrator.Info.nUBL, opts.path),
calibrator.write_redundantinfo(infoPath = opts.path, overwrite = opts.overwrite, verbose = True)
print "Done."
type='int',
default=203,
help='Number of frequency bins.')
o.add_option('-o',
'--outputpath',
action='store',
default=None,
help='output folder. Same folder by default.')
o.add_option('--overwrite',
action='store_true',
help='overwrite if file exists')
opts, args = o.parse_args(sys.argv[1:])
NCHAN = opts.nFrequency
calibrator = omni.RedundantCalibrator(opts.nAntenna)
print "Reading redundant info", opts.infopath, "...",
sys.stdout.flush()
calibrator.read_redundantinfo(opts.infopath)
print "Done."
sys.stdout.flush()
NANT = calibrator.nAntenna
NUBL = calibrator.nUBL
NPRM = 3 + 2 * (calibrator.nAntenna + calibrator.nUBL)
for omnical_file in args:
print "Processing", omnical_file, "..."
sys.stdout.flush()
if opts.outputpath is None:
def test_testinfo_logcal(self):
#check that logcal give 0 chi2 for all 20 testinfos
diff = np.zeros(20)
for index in range(20):
fileindex = index + 1 #filenames have index that start with 1
####Import arrayinfo##########################
arrayinfopath = os.path.dirname(
os.path.realpath(__file__)) + '/testinfo/test' + str(
fileindex) + '_array_info.txt'
nant = 56
calibrator = omni.RedundantCalibrator(nant)
calibrator.compute_redundantinfo(arrayinfopath)
info = calibrator.Info.get_info()
####Config parameters###################################
removedegen = True
removeadditive = False
needrawcal = True #if true, (generally true for raw data) you need to take care of having raw calibration parameters in float32 binary format freq x nant
keep_binary_data = True
keep_binary_calpar = True
converge_percent = 0.00001
max_iter = 50
step_size = .2
####import data##################
datapath = os.path.dirname(os.path.realpath(
__file__)) + '/testinfo/test' + str(fileindex) + '_data.txt'
with open(datapath) as f:
rawinfo = [[float(x) for x in line.split()] for line in f]
data = np.array(
[i[0] + 1.0j * i[1] for i in rawinfo[:-1]],
dtype='complex64') #last element of rawinfo is empty
data = data.reshape((1, 1, len(data)))
####do calibration################
calibrator.removeDegeneracy = removedegen
calibrator.removeAdditive = removeadditive
calibrator.keepData = keep_binary_data
calibrator.keepCalpar = keep_binary_calpar
calibrator.convergePercent = converge_percent
calibrator.maxIteration = max_iter
calibrator.stepSize = step_size
calibrator.computeUBLFit = True
calibrator.logcal(data, np.zeros_like(data), verbose=True)
log = np.copy(calibrator.rawCalpar)
log = np.copy(calibrator.rawCalpar)
ampcal = log[0, 0, 3:info['nAntenna'] + 3]
phasecal = log[0, 0, info['nAntenna'] + 3:info['nAntenna'] * 2 + 3]
calpar = 10**(ampcal) * np.exp(1.0j * phasecal)
ublfit = log[0, 0, 3 + 2 * info['nAntenna']::2] + 1.0j * log[
0, 0, 3 + 2 * info['nAntenna'] + 1::2]
####import real calibration parameter
calparpath = os.path.dirname(os.path.realpath(
__file__)) + '/testinfo/test' + str(fileindex) + '_calpar.txt'
with open(calparpath) as f:
rawinfo = [[float(x) for x in line.split()] for line in f]
temp = np.array(rawinfo[:-1])
correctcalpar = (np.array(temp[:, 0]) +
1.0j * np.array(temp[:, 1]))[info['subsetant']]
###compare calpar with correct calpar
overallfactor = np.real(np.mean(ublfit))**0.5
diffnorm = la.norm(calpar * overallfactor - correctcalpar)
diff[index] = la.norm(diffnorm)
self.assertAlmostEqual(la.norm(diff), 0, 4)
def test_all(self):
##FILENAME = "test.py"
######################################################################
##############Config parameters###################################
######################################################################
ano = 'test' ##This is the file name difference for final calibration parameter result file. Result will be saved in miriadextract_xx_ano.omnical
uvfiles = [os.path.dirname(os.path.realpath(__file__)) + '/test.uv']
wantpols = {'xx': -5} #, 'yy':-6}
#infopaths = {'xx':os.path.dirname(os.path.realpath(__file__)) + '/../doc/redundantinfo_PSA32.txt', 'yy':os.path.dirname(os.path.realpath(__file__)) + '/../doc/redundantinfo_PSA32.txt'}
arrayinfos = {
'xx':
os.path.dirname(os.path.realpath(__file__)) +
'/../doc/arrayinfo_apprx_PAPER32_badUBLpair.txt',
'yy':
os.path.dirname(os.path.realpath(__file__)) +
'/../doc/arrayinfo_apprx_PAPER32_badUBLpair.txt'
}
oppath = os.path.dirname(os.path.realpath(__file__)) + '/results/'
if not os.path.isdir(oppath):
os.makedirs(oppath)
removedegen = True
removeadditive = False
needrawcal = True #if true, (generally true for raw data) you need to take care of having raw calibration parameters in float32 binary format freq x nant
rawpaths = {
'xx':
os.path.dirname(os.path.realpath(__file__)) +
"/testrawphasecalparrad_xx",
'yy':
os.path.dirname(os.path.realpath(__file__)) +
"/testrawphasecalparrad_yy"
}
keep_binary_data = True
keep_binary_calpar = True
converge_percent = 0.000001
max_iter = 1000
step_size = .3
######################################################################
######################################################################
######################################################################
########Massage user parameters###################################
oppath += '/'
####get some info from the first uvfile ################
uv = ap.miriad.UV(uvfiles[0])
nfreq = uv.nchan
nant = uv['nants'] / 2 # 'nants' counting ant-pols, so divide 2
startfreq = uv['sfreq']
dfreq = uv['sdf']
del (uv)
####create redundant calibrators################
#calibrators = [omni.RedundantCalibrator(nant, info = infopaths[key]) for key in wantpols.keys()]
calibrators = [
omni.RedundantCalibrator(nant) for key in wantpols.keys()
]
for calibrator, key in zip(calibrators, wantpols.keys()):
calibrator.compute_redundantinfo(arrayinfoPath=arrayinfos[key])
#calibrator.write_redundantinfo(infoPath = './redundantinfo_test_' + key + '.txt', overwrite = True)
###start reading miriads################
##print FILENAME + " MSG:", len(uvfiles), "uv files to be processed for " + ano
data, t, timing, lst = omni.importuvs(uvfiles,
calibrators[0].totalVisibilityId,
wantpols,
nTotalAntenna=32,
timingTolerance=2 * math.pi,
init_mem=5.e7)
##print FILENAME + " MSG:", len(t), "slices read."
###raw calibration################
if needrawcal:
for p, key in zip(range(len(wantpols)), wantpols.keys()):
rawcalpar = np.fromfile(rawpaths[key],
dtype="complex64").reshape(
nfreq, nant)
data[p] = omni.apply_calpar(data[p], rawcalpar,
calibrators[p].totalVisibilityId)
#####Save various files read################
###np.savetxt('miriadextract_' + ano + "_sunpos.dat", sunpos[:len(t)], fmt='%8.5f')
##f = open(oppath + 'miriadextract_' + ano + "_localtime.dat",'w')
##for time in timing:
##f.write("%s\n"%time)
##f.close()
##f = open(oppath + 'miriadextract_' + ano + "_lsthour.dat",'w')
##for l in lst:
##f.write("%s\n"%l)
##f.close()
####calibrate################
##print FILENAME + " MSG: starting calibration."
for p, calibrator in zip(range(len(wantpols)), calibrators):
calibrator.removeDegeneracy = removedegen
calibrator.removeAdditive = removeadditive
calibrator.keepData = keep_binary_data
calibrator.keepCalpar = keep_binary_calpar
calibrator.convergePercent = converge_percent
calibrator.maxIteration = max_iter
calibrator.stepSize = step_size
calibrator.computeUBLFit = False
calibrator.logcal(data[p], np.zeros_like(data[p]), verbose=False)
additiveout = calibrator.lincal(data[p],
np.zeros_like(data[p]),
verbose=False)
calibrator.utctimes = timing
calibrator.get_calibrated_data(data[p])
calibrator.get_omnichisq()
calibrator.get_omnigain()
calibrator.get_omnifit()
#########Test results############
correctresult = np.fromfile(
os.path.dirname(os.path.realpath(__file__)) + '/test.omnical',
dtype='float32').reshape(14, 203, 165)[:, :, :]
nanmask = ~np.isnan(
np.sum(correctresult, axis=2)
) #mask the last dimension because when data contains some 0 and some -0, C++ code return various phasecalibration parameters on different systems, when all other numbers are nan. I do the summation to avoid it failing the euqality check when the input is trivially 0s.
calibrators[-1].rawCalpar.tofile(
os.path.dirname(os.path.realpath(__file__)) +
'/results/new_result.omnical')
omni.write_redundantinfo(calibrators[-1].Info.get_info(),
os.path.dirname(os.path.realpath(__file__)) +
'/results/new_info.bin',
overwrite=True)
newresult = calibrators[-1].rawCalpar[:, :, :]
np.testing.assert_almost_equal(correctresult[:, :, 1:3][nanmask],
newresult[:, :, 1:3][nanmask],
decimal=5)
np.testing.assert_almost_equal(np.sum(
np.abs(data[-1] - calibrators[-1].get_modeled_data())
[:, :,
calibrators[-1].Info.subsetbl[calibrators[-1].Info.crossindex]]**
2,
axis=2)[nanmask],
newresult[:, :, 2][nanmask],
decimal=5)
np.testing.assert_almost_equal(np.sum(
np.abs(additiveout)[:, :, calibrators[-1].Info.crossindex]**2,
axis=2)[nanmask],
newresult[:, :, 2][nanmask],
decimal=5)
np.testing.assert_almost_equal(correctresult[:, :, 3:67][nanmask],
newresult[:, :, 3:67][nanmask],
decimal=5)
np.testing.assert_almost_equal(
np.sort(np.abs(correctresult[:, :, 67:][nanmask])),
np.sort(np.abs(newresult[:, :, 67:][nanmask])),
decimal=5)
def test_testinfo_lincal(self):
fileindex = 3 #use the 3rd file to do the test, can also change this to any number from 1 to 20
length = 100
loglist = np.zeros(length)
linlist = np.zeros(length)
####import arrayinfo################
arrayinfopath = os.path.dirname(os.path.realpath(
__file__)) + '/testinfo/test' + str(fileindex) + '_array_info.txt'
nant = 56
calibrator = omni.RedundantCalibrator(nant)
calibrator.compute_redundantinfo(arrayinfopath)
info = calibrator.Info.get_info()
####Config parameters###################################
removedegen = True
removeadditive = False
needrawcal = True #if true, (generally true for raw data) you need to take care of having raw calibration parameters in float32 binary format freq x nant
keep_binary_data = True
keep_binary_calpar = True
converge_percent = 0.00001
max_iter = 50
step_size = .2
####import data##################
datapath = os.path.dirname(os.path.realpath(
__file__)) + '/testinfo/test' + str(fileindex) + '_data.txt'
with open(datapath) as f:
rawinfo = [[float(x) for x in line.split()] for line in f]
data = np.array([i[0] + 1.0j * i[1] for i in rawinfo[:-1]],
dtype='complex64') #last element of rawinfo is empty
truedata = data.reshape((1, 1, len(data)))
std = 0.1
####do calibration################
calibrator.removeDegeneracy = removedegen
calibrator.removeAdditive = removeadditive
calibrator.keepData = keep_binary_data
calibrator.keepCalpar = keep_binary_calpar
calibrator.convergePercent = converge_percent
calibrator.maxIteration = max_iter
calibrator.stepSize = step_size
calibrator.computeUBLFit = True
for i in range(length):
noise = (np.random.normal(scale=std, size=data.shape) +
1.0j * np.random.normal(scale=std, size=data.shape)
).astype('complex64')
data = truedata + noise
calibrator.logcal(data, np.zeros_like(data), verbose=True)
calibrator.lincal(data, np.zeros_like(data), verbose=True)
linchi2 = (calibrator.rawCalpar[0, 0, 2] /
(info['A'].shape[0] - info['A'].shape[1]) /
(2 * std**2))**0.5
logchi2 = (calibrator.rawCalpar[0, 0, 1] /
(info['A'].shape[0] - info['A'].shape[1]) /
(2 * std**2))**0.5
linlist[i] = linchi2
loglist[i] = logchi2
self.assertTrue(abs(np.mean(linlist) - 1.0) <
0.01) #check that chi2 of lincal is close enough to 1
self.assertTrue(
np.mean(linlist) < np.mean(loglist)
) #chick that chi2 of lincal is smaller than chi2 of logcal