def chisq4tau(filename, tau_lim=30., delta_tau=0.01, save_npz=True):
"""
define array of possible tau values [always symmetric about zero]
and impose freqency coupling.
Use these to calculate the chi squared statistic:
X^2(nu,t) = sum_{baselines b} V^xy_b - V^{yx}_b exp(-2 pi i nu tau)
for a sweep of taus; i.e. the x-to-y delay/angle that minimzes Stokes V
across the array.
Arguments:
-- tau_lim: sweep of taus runs (-tau_lim, taulim)
-- delta_tau: spacing of the sweep
-- save_npz: saves an npz of the chi square for each tau, time and frequency
Returns: an array of chi squared values for every tau, time and frequency.
The minimum of the tau axis will give the required minimization for V.
"""
print '==== '+filename+' ===='
dataxy,txy,timingxy,lstxy,flagsxy = omni.importuvs([filename],{'xy':-7}, timingTolerance=1e9, init_mem=.5e9)
datayx,tyx,timingyx,lstyx,flagsyx = omni.importuvs([filename],{'yx':-8}, timingTolerance=1e9, init_mem=.5e9)
global tau_arr
global angle_arr
tau_arr = np.arange(-1*tau_lim, tau_lim, delta_tau) #nanseconds
angle_arr = np.exp(-2*np.pi*1.j*np.outer(nu_arr,tau_arr))
#calculate the chi for each baseline; sum over the array
chisquared = np.zeros( (tau_arr.shape[0], dataxy.shape[1], nu_arr.shape[0]) ) #tau x time x freq
###XXX TODO: VECTORIZE THIS BABY
###XXX update: I get memory errors when I try to create a baseline axis by
###### outer-producting with angle_arr. How to fix?
print 'Begin stupid tau/bl loop (vectorizing is giving Mem Errs)'
for i in range(angle_arr.shape[1]): # for each tau
calc = 0
for bl in range(dataxy.shape[3]): #for each baseline
XY = dataxy[0,:,:,bl]
YX = datayx[0,:,:,bl]
calc += pow(np.absolute(XY - YX*angle_arr[:,i]),2.)
chisquared[i,:,:]=calc
a=filename.split('.')
a[0]='tau'
a='.'.join(a)+'.npz'
if save_npz: np.savez(a, chisquared=chisquared, angle=angle_arr, tau=tau_arr)
return chisquared
import numpy as np
import omnical.calibration_omni as omni
import matplotlib.pyplot as plt
import aipy as ap
paper_uvfiles = ['/data4/paper/2012EoR/psa_live/psa6266/zen.2456266.64714.uvcRREcAC']
omni_uvfiles = ["/data4/paper/2012EoR/psa_live/forlstbinning_omnical_2/zen.2456266.64714.uvcRREcACO"]
info=omni.read_redundantinfo('/data2/home/hz2ug/omnical/doc/redundantinfo_PSA64_ba19_37_50.bin')
nant = 64
wantpols = {}
wantpols['xx'] = ap.miriad.str2pol['xx']
papercal_data, _, _, _ = omni.importuvs(paper_uvfiles, np.concatenate([[[i,j] for i in range(j + 1)] for j in range(nant)]), wantpols, timingTolerance=100)
omnical_data, _, _, _ = omni.importuvs(omni_uvfiles, np.concatenate([[[i,j] for i in range(j + 1)] for j in range(nant)]), wantpols, timingTolerance=100)
omni.omniview(np.array([papercal_data[0,0,120], omnical_data[0,0,120]]), info)
elif 'yy' == key:
pol_select = pol_select + [3]
rawdata = np.zeros((len(pol_select), np.sum(nTimes), nfreq, nant * (nant + 1) / 2), dtype='complex64')
timing = []
lst = []
for i, uvfile in enumerate(uvfiles):
rawdata[:, len(timing):len(timing)+nTimes[i]] = np.fromfile(uvfile+'/visibilities', dtype='complex64').reshape((4, nTimes[i], nfreq, nant * (nant + 1) / 2))[pol_select]
with open(uvfile.replace('.odf', '.odfa') + '/timing.dat') as f:
for l in f.readlines():
sa.date = l.replace('\n','')
sa.date += ephem.second * 3600 * 4
lst += [sa.sidereal_time()]
timing += [sa.date.__str__()]
else:
if len(uvfiles) == 1:
rawdata, t, timing, lst, rawflag = omni.importuvs(uvfiles, wantpols, totalVisibilityId = np.concatenate([[[i,j] for i in range(j + 1)] for j in range(len(aa))]), timingTolerance=100)#, nTotalAntenna = len(aa))
else:
for p, pol in enumerate(wantpols.keys()):
if p == 0:
rawdata, t, timing, lst, rawflag = omni.importuvs([uvfiles_dic[pol]], {pol:wantpols[pol]}, totalVisibilityId = np.concatenate([[[i,j] for i in range(j + 1)] for j in range(len(aa))]), timingTolerance=100)
else:
tmpdata, t, timing, lst, tmpflag = omni.importuvs([uvfiles_dic[pol]], {pol:wantpols[pol]}, totalVisibilityId = np.concatenate([[[i,j] for i in range(j + 1)] for j in range(len(aa))]), timingTolerance=100)
rawdata = np.concatenate((rawdata, tmpdata))
print FILENAME + " MSG:", len(t), "slices read."
sys.stdout.flush()
sun = ephem.Sun()
sunpos = np.zeros((len(timing), 2))
southern_points = {'hyd':{'ra': '09:18:05.7', 'dec': '-12:05:44'},
'cen':{'ra': '13:25:27.6', 'dec': '-43:01:09'},
uv=ap.miriad.UV(uvfiles[0])
nfreq = uv.nchan;
nant = uv['nants'] / 2 # 'nants' counting ant-pols, so divide 2
startfreq = uv['sfreq']
dfreq = uv['sdf']
del(uv)
####create redundant calibrators################
#calibrators = [omni.RedundantCalibrator(nant, info = infopaths[key]) for key in wantpols.keys()]
calibrators = [omni.RedundantCalibrator(nant) for key in wantpols.keys()]
for calibrator, key in zip(calibrators, wantpols.keys()):
calibrator.compute_redundantinfo(arrayinfoPath = arrayinfos[key])
#calibrator.write_redundantinfo(infoPath = './redundantinfo_test_' + key + '.txt', overwrite = True)
###start reading miriads################
##print FILENAME + " MSG:", len(uvfiles), "uv files to be processed for " + ano
data, t, timing, lst = omni.importuvs(uvfiles, calibrators[0].totalVisibilityId, wantpols, nTotalAntenna = 32,timingTolerance = 2*math.pi, init_mem = 5.e7)
##print FILENAME + " MSG:", len(t), "slices read."
###raw calibration################
if needrawcal:
for p, key in zip(range(len(wantpols)), wantpols.keys()):
rawcalpar = np.fromfile(rawpaths[key], dtype="complex64").reshape(nfreq, nant)
data[p] = omni.apply_calpar(data[p], rawcalpar, calibrators[p].totalVisibilityId)
#####Save various files read################
###np.savetxt('miriadextract_' + ano + "_sunpos.dat", sunpos[:len(t)], fmt='%8.5f')
##f = open(oppath + 'miriadextract_' + ano + "_localtime.dat",'w')
##for time in timing:
##f.write("%s\n"%time)
##f.close()
##f = open(oppath + 'miriadextract_' + ano + "_lsthour.dat",'w')
badUBLpair.append([aa.ant_layout[0, 0], aa.ant_layout[r, c]])
badUBLpair.append([aa.ant_layout[-1, 0], aa.ant_layout[r, c]])
badAntenna_x = [8, 14, 15, 16, 26, 27, 28, 34, 37, 38, 42, 46, 50, 53, 72, 74, 82, 84, 85, 110]
badAntenna_y = [3, 7, 15, 16, 23, 26, 27, 34, 38, 46, 50, 56, 57, 72, 100]
badAntenna = sorted(np.unique(badAntenna_x + badAntenna_y).tolist())
rc.compute_redundantinfo(badAntenna=badAntenna, badUBLpair=badUBLpair, antennaLocationTolerance=.3)
##start reading data
rawdata = {}
rawflag = {}
for p, pol in enumerate(['xx', 'yy']):
data_list = glob.glob(datadir+'*.%s.uv'%pol)
print len(data_list)
rawdata[pol], t, timing, raw_lst, rawflag[pol] = omni.importuvs(data_list, {pol: ap.miriad.str2pol[pol]}, totalVisibilityId=rc.Info.totalVisibilityId[rc.Info.subsetbl[rc.Info.crossindex]], timingTolerance=0.001)
#reorder and sort lst
rawdata[pol] = rawdata[pol][:, np.argsort(raw_lst)]
rawflag[pol] = rawflag[pol][:, np.argsort(raw_lst)]
raw_lst = sorted(raw_lst)
plt.plot(raw_lst)
plt.show()
nf = rawdata['xx'].shape[2]
nbl = rawdata['xx'].shape[3]
uv = ap.miriad.UV(data_list[0])
freqs = (uv['freq'] + np.arange(nf) * uv['sdf']) * 1.e3
t_chunks = len(raw_lst) / t_avg
lst = np.mean(np.array(raw_lst)[:t_chunks * t_avg].reshape((t_chunks, t_avg)), axis=-1)
sa = ephem.Observer()
sa.lon = uv['longitu']
sa.lat = uv['latitud']
sa.pressure = 0
startfreq = uv['sfreq']
dfreq = uv['sdf']
del (uv)
print "Done."
sys.stdout.flush()
###start reading miriads################
print FILENAME + " MSG:", len(uvfiles), "uv files to be processed"
sys.stdout.flush()
rawdata, t, timing, lst = omni.importuvs(
uvfiles,
np.concatenate([[[i, j] for i in range(j + 1)] for j in range(len(aa))]),
wantpols,
timingTolerance=100) #, nTotalAntenna = len(aa))
print FILENAME + " MSG:", len(t), "slices read."
sys.stdout.flush()
sun = ephem.Sun()
sunpos = np.zeros((len(timing), 2))
southern_points = {
'hyd': {
'ra': '09:18:05.7',
'dec': '-12:05:44'
},
'cen': {
'ra': '13:25:27.6',
'dec': '-43:01:09'
import numpy as np
import omnical.calibration_omni as omni
import matplotlib.pyplot as plt
import aipy as ap
paper_uvfiles = [
'/data4/paper/2012EoR/psa_live/psa6266/zen.2456266.64714.uvcRREcAC'
]
omni_uvfiles = [
"/data4/paper/2012EoR/psa_live/forlstbinning_omnical_2/zen.2456266.64714.uvcRREcACO"
]
info = omni.read_redundantinfo(
'/data2/home/hz2ug/omnical/doc/redundantinfo_PSA64_ba19_37_50.bin')
nant = 64
wantpols = {}
wantpols['xx'] = ap.miriad.str2pol['xx']
papercal_data, _, _, _, rawflag = omni.importuvs(paper_uvfiles,
wantpols,
timingTolerance=100)
omnical_data, _, _, _, rawflag = omni.importuvs(omni_uvfiles,
wantpols,
timingTolerance=100)
data = omni.omniview(
np.array([papercal_data[0, 0, 120], omnical_data[0, 0, 120]]), info)
np.savez('/data2/home/zakiali/jeff_psa64/omniview.npz', data)
def test_all(self):
##FILENAME = "test.py"
######################################################################
##############Config parameters###################################
######################################################################
ano = 'test'##This is the file name difference for final calibration parameter result file. Result will be saved in miriadextract_xx_ano.omnical
uvfiles = [os.path.dirname(os.path.realpath(__file__)) + '/test.uv']
wantpols = {'xx':-5}#, 'yy':-6}
#infopaths = {'xx':os.path.dirname(os.path.realpath(__file__)) + '/../doc/redundantinfo_PSA32.txt', 'yy':os.path.dirname(os.path.realpath(__file__)) + '/../doc/redundantinfo_PSA32.txt'}
arrayinfos = {'xx':os.path.dirname(os.path.realpath(__file__)) + '/../doc/arrayinfo_apprx_PAPER32_badUBLpair.txt', 'yy':os.path.dirname(os.path.realpath(__file__)) + '/../doc/arrayinfo_apprx_PAPER32_badUBLpair.txt'}
oppath = os.path.dirname(os.path.realpath(__file__)) + '/results/'
if not os.path.isdir(oppath):
os.makedirs(oppath)
removedegen = True
removeadditive = False
needrawcal = True #if true, (generally true for raw data) you need to take care of having raw calibration parameters in float32 binary format freq x nant
rawpaths = {'xx':os.path.dirname(os.path.realpath(__file__)) + "/testrawphasecalparrad_xx", 'yy':os.path.dirname(os.path.realpath(__file__)) + "/testrawphasecalparrad_yy"}
keep_binary_data = True
keep_binary_calpar = True
converge_percent = 0.000001
max_iter = 1000
step_size = .3
######################################################################
######################################################################
######################################################################
########Massage user parameters###################################
oppath += '/'
####get some info from the first uvfile ################
uv=ap.miriad.UV(uvfiles[0])
nfreq = uv.nchan;
nant = uv['nants'] / 2 # 'nants' counting ant-pols, so divide 2
startfreq = uv['sfreq']
dfreq = uv['sdf']
del(uv)
####create redundant calibrators################
#calibrators = [omni.RedundantCalibrator(nant, info = infopaths[key]) for key in wantpols.keys()]
calibrators = [omni.RedundantCalibrator(nant) for key in wantpols.keys()]
for calibrator, key in zip(calibrators, wantpols.keys()):
calibrator.compute_redundantinfo(arrayinfoPath = arrayinfos[key])
#calibrator.write_redundantinfo(infoPath = './redundantinfo_test_' + key + '.txt', overwrite = True)
###start reading miriads################
##print FILENAME + " MSG:", len(uvfiles), "uv files to be processed for " + ano
data, t, timing, lst = omni.importuvs(uvfiles, calibrators[0].totalVisibilityId, wantpols, nTotalAntenna = 32,timingTolerance = 2*math.pi, init_mem = 5.e7)
##print FILENAME + " MSG:", len(t), "slices read."
###raw calibration################
if needrawcal:
for p, key in zip(range(len(wantpols)), wantpols.keys()):
rawcalpar = np.fromfile(rawpaths[key], dtype="complex64").reshape(nfreq, nant)
data[p] = omni.apply_calpar(data[p], rawcalpar, calibrators[p].totalVisibilityId)
#####Save various files read################
###np.savetxt('miriadextract_' + ano + "_sunpos.dat", sunpos[:len(t)], fmt='%8.5f')
##f = open(oppath + 'miriadextract_' + ano + "_localtime.dat",'w')
##for time in timing:
##f.write("%s\n"%time)
##f.close()
##f = open(oppath + 'miriadextract_' + ano + "_lsthour.dat",'w')
##for l in lst:
##f.write("%s\n"%l)
##f.close()
####calibrate################
##print FILENAME + " MSG: starting calibration."
for p, calibrator in zip(range(len(wantpols)), calibrators):
calibrator.removeDegeneracy = removedegen
calibrator.removeAdditive = removeadditive
calibrator.keepData = keep_binary_data
calibrator.keepCalpar = keep_binary_calpar
calibrator.convergePercent = converge_percent
calibrator.maxIteration = max_iter
calibrator.stepSize = step_size
calibrator.computeUBLFit = False
calibrator.logcal(data[p], np.zeros_like(data[p]), verbose=True)
calibrator.lincal(data[p], np.zeros_like(data[p]), verbose=True)
calibrator.utctimes = timing
calibrator.get_calibrated_data(data[p])
calibrator.get_omnichisq()
calibrator.get_omnigain()
calibrator.get_omnifit()
#########Test results############
correctresult = np.fromfile(os.path.dirname(os.path.realpath(__file__)) + '/test.omnical', dtype = 'float32').reshape(14,203,165)[:,:,:3+2*nant]
nanmask = ~np.isnan(np.sum(correctresult,axis=2))#mask the last dimension because when data contains some 0 and some -0, C++ code return various phasecalibration parameters on different systems, when all other numbers are nan. I do the summation to avoid it failing the euqality check when the input is trivially 0s.
newresult = calibrators[-1].rawCalpar[:,:,:3+2*nant]#[:,:,3:]
#calibrators[-1].rawCalpar.tofile(os.path.dirname(os.path.realpath(__file__)) + '/test.omnical')
np.testing.assert_almost_equal(correctresult[nanmask], newresult[nanmask])#decimal=
lst = []
for i, uvfile in enumerate(uvfiles):
rawdata[:, len(timing):len(timing) + nTimes[i]] = np.fromfile(
uvfile + '/visibilities', dtype='complex64').reshape(
(4, nTimes[i], nfreq, nant * (nant + 1) / 2))[pol_select]
with open(uvfile.replace('.odf', '.odfa') + '/timing.dat') as f:
for l in f.readlines():
sa.date = l.replace('\n', '')
sa.date += ephem.second * 3600 * 4
lst += [sa.sidereal_time()]
timing += [sa.date.__str__()]
else:
if len(uvfiles) == 1:
rawdata, t, timing, lst, rawflag = omni.importuvs(
uvfiles,
wantpols,
totalVisibilityId=np.concatenate([[[i, j] for i in range(j + 1)]
for j in range(len(aa))]),
timingTolerance=100) #, nTotalAntenna = len(aa))
else:
for p, pol in enumerate(wantpols.keys()):
if p == 0:
rawdata, t, timing, lst, rawflag = omni.importuvs(
[uvfiles_dic[pol]], {pol: wantpols[pol]},
totalVisibilityId=np.concatenate([[[i, j]
for i in range(j + 1)]
for j in range(len(aa))
]),
timingTolerance=100)
else:
tmpdata, t, timing, lst, tmpflag = omni.importuvs(
sa.lon = aa.lon
sa.lat = aa.lat
startfreq = uv['sfreq']#GHz
dfreq = uv['sdf']#GHz
wantpols = {}
for pol in omni.get_uv_pols(uv):
wantpols[pol] = ap.miriad.str2pol[pol]
del(uv)
print "Done."
sys.stdout.flush()
#######import data file###########################
print FILENAME + " MSG:", len(uvfiles), "uv files to be processed for " + ano
sys.stdout.flush()
data, jd, timing, lst, rawflag = omni.importuvs(uvfiles, wantpols, totalVisibilityId = np.concatenate([[[i,j] for i in range(j + 1)] for j in range(len(aa))]), timingTolerance=100, init_mem=init_mem, lat = sa.lat, lon=sa.lon)#, nTotalAntenna = len(aa))
if delay_compression > len(jd) / len(uvfiles):
raise ValueError("Desired time bins %i is larger than input stamp count per uvfile."%len(jd))
data[rawflag] = 0
print FILENAME + " MSG:", len(jd), "slices read. data shape: ", data.shape
sys.stdout.flush()
########compress###########################
compr_shape = list(data.shape)
compr_shape[1] = len(uvfiles) * delay_compression
compressed_data = np.zeros(compr_shape, dtype=data.dtype)
compr_flags = np.zeros(compr_shape, dtype='bool')
compr_error = np.zeros((compr_shape[0], compr_shape[2], compr_shape[-1]), dtype='float32')
for p, pol in enumerate(wantpols.keys()):
import matplotlib.pyplot as plt
import aipy as ap
paper_uvfiles = [
'/data4/paper/2012EoR/psa_live/psa6266/zen.2456266.64714.uvcRREcAC'
]
omni_uvfiles = [
"/data4/paper/2012EoR/psa_live/forlstbinning_omnical_2/zen.2456266.64714.uvcRREcACO"
]
info = omni.read_redundantinfo(
'/data2/home/hz2ug/omnical/doc/redundantinfo_PSA64_ba19_37_50.bin')
nant = 64
wantpols = {}
wantpols['xx'] = ap.miriad.str2pol['xx']
papercal_data, _, _, _ = omni.importuvs(paper_uvfiles,
np.concatenate([[[i, j]
for i in range(j + 1)]
for j in range(nant)]),
wantpols,
timingTolerance=100)
omnical_data, _, _, _ = omni.importuvs(omni_uvfiles,
np.concatenate([[[i, j]
for i in range(j + 1)]
for j in range(nant)]),
wantpols,
timingTolerance=100)
omni.omniview(np.array([papercal_data[0, 0, 120], omnical_data[0, 0, 120]]),
info)
sa.lon = uv['longitu']
sa.lat = uv['latitud']
sa.pressure = 0
startfreq = uv['sfreq']
dfreq = uv['sdf']
del(uv)
print "Done."
sys.stdout.flush()
###start reading miriads################
print FILENAME + " MSG:", len(uvfiles), "uv files to be processed"
sys.stdout.flush()
rawdata, t, timing, lst = omni.importuvs(uvfiles, np.concatenate([[[i,j] for i in range(j + 1)] for j in range(len(aa))]), wantpols, timingTolerance=100)#, nTotalAntenna = len(aa))
print FILENAME + " MSG:", len(t), "slices read."
sys.stdout.flush()
sun = ephem.Sun()
sunpos = np.zeros((len(timing), 2))
southern_points = {'hyd':{'ra': '09:18:05.7', 'dec': '-12:05:44'},
'cen':{'ra': '13:25:27.6', 'dec': '-43:01:09'},
'cyg':{'ra': '19:59:28.3', 'dec': '40:44:02'},
'pic':{'ra': '05:19:49.7', 'dec': '-45:46:44'},
'vir':{'ra': '12:30:49.4', 'dec': '12:23:28'},
'for':{'ra': '03:22:41.7', 'dec': '-37:12:30'}}
for source in southern_points.keys():
southern_points[source]['body'] = ephem.FixedBody()
southern_points[source]['body']._ra = southern_points[source]['ra']
import numpy as np
import omnical.calibration_omni as omni
import matplotlib.pyplot as plt
import aipy as ap
paper_uvfiles = ['/data4/paper/2012EoR/psa_live/psa6266/zen.2456266.64714.uvcRREcAC']
omni_uvfiles = ["/data4/paper/2012EoR/psa_live/forlstbinning_omnical_2/zen.2456266.64714.uvcRREcACO"]
info=omni.read_redundantinfo('/data2/home/hz2ug/omnical/doc/redundantinfo_PSA64_ba19_37_50.bin')
nant = 64
wantpols = {}
wantpols['xx'] = ap.miriad.str2pol['xx']
papercal_data, _, _, _, rawflag = omni.importuvs(paper_uvfiles, wantpols, timingTolerance=100)
omnical_data, _, _, _, rawflag = omni.importuvs(omni_uvfiles, wantpols, timingTolerance=100)
data = omni.omniview(np.array([papercal_data[0,0,120], omnical_data[0,0,120]]), info)
np.savez('/data2/home/zakiali/jeff_psa64/omniview.npz', data)
def chisq4tau(filename, tau_lim=30., delta_tau=0.01, save_npz=True):
"""
define array of possible tau values [always symmetric about zero]
and impose freqency coupling.
Use these to calculate the chi squared statistic:
X^2(nu,t) = sum_{baselines b} V^xy_b - V^{yx}_b exp(-2 pi i nu tau)
for a sweep of taus; i.e. the x-to-y delay/angle that minimzes Stokes V
across the array.
Arguments:
-- tau_lim: sweep of taus runs (-tau_lim, taulim)
-- delta_tau: spacing of the sweep
-- save_npz: saves an npz of the chi square for each tau, time and frequency
Returns: an array of chi squared values for every tau, time and frequency.
The minimum of the tau axis will give the required minimization for V.
"""
print '==== ' + filename + ' ===='
dataxy, txy, timingxy, lstxy, flagsxy = omni.importuvs([filename],
{'xy': -7},
timingTolerance=1e9,
init_mem=.5e9)
datayx, tyx, timingyx, lstyx, flagsyx = omni.importuvs([filename],
{'yx': -8},
timingTolerance=1e9,
init_mem=.5e9)
global tau_arr
global angle_arr
tau_arr = np.arange(-1 * tau_lim, tau_lim, delta_tau) #nanseconds
angle_arr = np.exp(-2 * np.pi * 1.j * np.outer(nu_arr, tau_arr))
#calculate the chi for each baseline; sum over the array
chisquared = np.zeros((tau_arr.shape[0], dataxy.shape[1],
nu_arr.shape[0])) #tau x time x freq
###XXX TODO: VECTORIZE THIS BABY
###XXX update: I get memory errors when I try to create a baseline axis by
###### outer-producting with angle_arr. How to fix?
print 'Begin stupid tau/bl loop (vectorizing is giving Mem Errs)'
for i in range(angle_arr.shape[1]): # for each tau
calc = 0
for bl in range(dataxy.shape[3]): #for each baseline
XY = dataxy[0, :, :, bl]
YX = datayx[0, :, :, bl]
calc += pow(np.absolute(XY - YX * angle_arr[:, i]), 2.)
chisquared[i, :, :] = calc
a = filename.split('.')
a[0] = 'tau'
a = '.'.join(a) + '.npz'
if save_npz:
np.savez(a, chisquared=chisquared, angle=angle_arr, tau=tau_arr)
return chisquared
def test_all(self):
##FILENAME = "test.py"
######################################################################
##############Config parameters###################################
######################################################################
ano = 'test' ##This is the file name difference for final calibration parameter result file. Result will be saved in miriadextract_xx_ano.omnical
uvfiles = [os.path.dirname(os.path.realpath(__file__)) + '/test.uv']
wantpols = {'xx': -5} #, 'yy':-6}
#infopaths = {'xx':os.path.dirname(os.path.realpath(__file__)) + '/../doc/redundantinfo_PSA32.txt', 'yy':os.path.dirname(os.path.realpath(__file__)) + '/../doc/redundantinfo_PSA32.txt'}
arrayinfos = {
'xx':
os.path.dirname(os.path.realpath(__file__)) +
'/../doc/arrayinfo_apprx_PAPER32_badUBLpair.txt',
'yy':
os.path.dirname(os.path.realpath(__file__)) +
'/../doc/arrayinfo_apprx_PAPER32_badUBLpair.txt'
}
oppath = os.path.dirname(os.path.realpath(__file__)) + '/results/'
if not os.path.isdir(oppath):
os.makedirs(oppath)
removedegen = True
removeadditive = False
needrawcal = True #if true, (generally true for raw data) you need to take care of having raw calibration parameters in float32 binary format freq x nant
rawpaths = {
'xx':
os.path.dirname(os.path.realpath(__file__)) +
"/testrawphasecalparrad_xx",
'yy':
os.path.dirname(os.path.realpath(__file__)) +
"/testrawphasecalparrad_yy"
}
keep_binary_data = True
keep_binary_calpar = True
converge_percent = 0.000001
max_iter = 1000
step_size = .3
######################################################################
######################################################################
######################################################################
########Massage user parameters###################################
oppath += '/'
####get some info from the first uvfile ################
uv = ap.miriad.UV(uvfiles[0])
nfreq = uv.nchan
nant = uv['nants'] / 2 # 'nants' counting ant-pols, so divide 2
startfreq = uv['sfreq']
dfreq = uv['sdf']
del (uv)
####create redundant calibrators################
#calibrators = [omni.RedundantCalibrator(nant, info = infopaths[key]) for key in wantpols.keys()]
calibrators = [
omni.RedundantCalibrator(nant) for key in wantpols.keys()
]
for calibrator, key in zip(calibrators, wantpols.keys()):
calibrator.compute_redundantinfo(arrayinfoPath=arrayinfos[key])
#calibrator.write_redundantinfo(infoPath = './redundantinfo_test_' + key + '.txt', overwrite = True)
###start reading miriads################
##print FILENAME + " MSG:", len(uvfiles), "uv files to be processed for " + ano
data, t, timing, lst = omni.importuvs(uvfiles,
calibrators[0].totalVisibilityId,
wantpols,
nTotalAntenna=32,
timingTolerance=2 * math.pi,
init_mem=5.e7)
##print FILENAME + " MSG:", len(t), "slices read."
###raw calibration################
if needrawcal:
for p, key in zip(range(len(wantpols)), wantpols.keys()):
rawcalpar = np.fromfile(rawpaths[key],
dtype="complex64").reshape(
nfreq, nant)
data[p] = omni.apply_calpar(data[p], rawcalpar,
calibrators[p].totalVisibilityId)
#####Save various files read################
###np.savetxt('miriadextract_' + ano + "_sunpos.dat", sunpos[:len(t)], fmt='%8.5f')
##f = open(oppath + 'miriadextract_' + ano + "_localtime.dat",'w')
##for time in timing:
##f.write("%s\n"%time)
##f.close()
##f = open(oppath + 'miriadextract_' + ano + "_lsthour.dat",'w')
##for l in lst:
##f.write("%s\n"%l)
##f.close()
####calibrate################
##print FILENAME + " MSG: starting calibration."
for p, calibrator in zip(range(len(wantpols)), calibrators):
calibrator.removeDegeneracy = removedegen
calibrator.removeAdditive = removeadditive
calibrator.keepData = keep_binary_data
calibrator.keepCalpar = keep_binary_calpar
calibrator.convergePercent = converge_percent
calibrator.maxIteration = max_iter
calibrator.stepSize = step_size
calibrator.computeUBLFit = False
calibrator.logcal(data[p], np.zeros_like(data[p]), verbose=False)
additiveout = calibrator.lincal(data[p],
np.zeros_like(data[p]),
verbose=False)
calibrator.utctimes = timing
calibrator.get_calibrated_data(data[p])
calibrator.get_omnichisq()
calibrator.get_omnigain()
calibrator.get_omnifit()
#########Test results############
correctresult = np.fromfile(
os.path.dirname(os.path.realpath(__file__)) + '/test.omnical',
dtype='float32').reshape(14, 203, 165)[:, :, :]
nanmask = ~np.isnan(
np.sum(correctresult, axis=2)
) #mask the last dimension because when data contains some 0 and some -0, C++ code return various phasecalibration parameters on different systems, when all other numbers are nan. I do the summation to avoid it failing the euqality check when the input is trivially 0s.
calibrators[-1].rawCalpar.tofile(
os.path.dirname(os.path.realpath(__file__)) +
'/results/new_result.omnical')
omni.write_redundantinfo(calibrators[-1].Info.get_info(),
os.path.dirname(os.path.realpath(__file__)) +
'/results/new_info.bin',
overwrite=True)
newresult = calibrators[-1].rawCalpar[:, :, :]
np.testing.assert_almost_equal(correctresult[:, :, 1:3][nanmask],
newresult[:, :, 1:3][nanmask],
decimal=5)
np.testing.assert_almost_equal(np.sum(
np.abs(data[-1] - calibrators[-1].get_modeled_data())
[:, :,
calibrators[-1].Info.subsetbl[calibrators[-1].Info.crossindex]]**
2,
axis=2)[nanmask],
newresult[:, :, 2][nanmask],
decimal=5)
np.testing.assert_almost_equal(np.sum(
np.abs(additiveout)[:, :, calibrators[-1].Info.crossindex]**2,
axis=2)[nanmask],
newresult[:, :, 2][nanmask],
decimal=5)
np.testing.assert_almost_equal(correctresult[:, :, 3:67][nanmask],
newresult[:, :, 3:67][nanmask],
decimal=5)
np.testing.assert_almost_equal(
np.sort(np.abs(correctresult[:, :, 67:][nanmask])),
np.sort(np.abs(newresult[:, :, 67:][nanmask])),
decimal=5)
dfreq = uv['sdf'] #GHz
wantpols = {}
for pol in omni.get_uv_pols(uv):
wantpols[pol] = ap.miriad.str2pol[pol]
del (uv)
print "Done."
sys.stdout.flush()
#######import data file###########################
print FILENAME + " MSG:", len(uvfiles), "uv files to be processed for " + ano
sys.stdout.flush()
data, jd, timing, lst, rawflag = omni.importuvs(
uvfiles,
wantpols,
totalVisibilityId=np.concatenate([[[i, j] for i in range(j + 1)]
for j in range(len(aa))]),
timingTolerance=100,
init_mem=init_mem,
lat=sa.lat,
lon=sa.lon) #, nTotalAntenna = len(aa))
if delay_compression > len(jd) / len(uvfiles):
raise ValueError(
"Desired time bins %i is larger than input stamp count per uvfile." %
len(jd))
data[rawflag] = 0
print FILENAME + " MSG:", len(jd), "slices read. data shape: ", data.shape
sys.stdout.flush()
########compress###########################
compr_shape = list(data.shape)
