force_recompute = False
nside = 32
nt = 80
nside_standard = 32 #64
nt_standard = 80 #160
nUBL = 75 #34
#nside = 16
#nUBL = 75
if ps:
nps = 2 #number of point sources to include
else:
nps = 0
freq = 160.
thresh = 1
ubls = np.array([
bl for bl in omni.read_redundantinfo(
'/home/omniscope/omnical/doc/redundantinfo_X5_q3x.bin')['ubl'] *
[1.5, 1.5, 0] if la.norm(bl) < thresh * (nside * 299.792458 / freq)
])
if len(ubls) != nUBL:
raise Exception('%i != %i!' % (len(ubls), nUBL))
pca1 = hp.fitsfunc.read_map(
'/home/omniscope/simulate_visibilities/data/gsm1.fits' +
str(nside_standard))
pca2 = hp.fitsfunc.read_map(
'/home/omniscope/simulate_visibilities/data/gsm2.fits' +
str(nside_standard))
pca3 = hp.fitsfunc.read_map(
'/home/omniscope/simulate_visibilities/data/gsm3.fits' +
str(nside_standard))
gsm_standard = 422.952 * (0.307706 * pca1 + -0.281772 * pca2 +
import sys, os
import omnical.calibration_omni as omni
tavg = 2
force_recompute = False
nside = 16
nt = 80
nside_standard = 16#64
nt_standard = 80#160
nUBL = 75#34
#nside = 16
#nUBL = 75
nps = 2#number of point sources to include
freq = 160.
thresh = 1
ubls = np.array([bl for bl in omni.read_redundantinfo('/home/omniscope/omnical/doc/redundantinfo_X5_q3x.bin')['ubl']*[1.5, 1.5, 0] if la.norm(bl) < thresh * (nside * 299.792458 / freq)])
if len(ubls) != nUBL:
raise Exception('%i != %i!'%(len(ubls), nUBL))
#compute S
angular_scale = 1/(freq/300*np.max([la.norm(ubl) for ubl in ubls]))
print "Computing S matrix...",
sys.stdout.flush()
S = np.identity(12 * nside**2)
S = np.maximum(np.array([hp.sphtfunc.smoothing(pix_vec, sigma = angular_scale, verbose = False) for pix_vec in S]), 0)
pca1 = hp.fitsfunc.read_map('/home/omniscope/simulate_visibilities/data/gsm1.fits' + str(nside_standard))
pca2 = hp.fitsfunc.read_map('/home/omniscope/simulate_visibilities/data/gsm2.fits' + str(nside_standard))
pca3 = hp.fitsfunc.read_map('/home/omniscope/simulate_visibilities/data/gsm3.fits' + str(nside_standard))
gsm_standard = 422.952*(0.307706*pca1+-0.281772*pca2+0.0123976*pca3)
import numpy as np
import omnical.calibration_omni as omni
import matplotlib.pyplot as plt
import aipy as ap
paper_uvfiles = ['/data4/paper/2012EoR/psa_live/psa6266/zen.2456266.64714.uvcRREcAC']
omni_uvfiles = ["/data4/paper/2012EoR/psa_live/forlstbinning_omnical_2/zen.2456266.64714.uvcRREcACO"]
info=omni.read_redundantinfo('/data2/home/hz2ug/omnical/doc/redundantinfo_PSA64_ba19_37_50.bin')
nant = 64
wantpols = {}
wantpols['xx'] = ap.miriad.str2pol['xx']
papercal_data, _, _, _ = omni.importuvs(paper_uvfiles, np.concatenate([[[i,j] for i in range(j + 1)] for j in range(nant)]), wantpols, timingTolerance=100)
omnical_data, _, _, _ = omni.importuvs(omni_uvfiles, np.concatenate([[[i,j] for i in range(j + 1)] for j in range(nant)]), wantpols, timingTolerance=100)
omni.omniview(np.array([papercal_data[0,0,120], omnical_data[0,0,120]]), info)
else:
compr_uvfile = oppath + '/' + dataano + ".uvOE"
uv=ap.miriad.UV(uvfiles[0])
omni.exportuv(compr_uvfile, compressed_data, compr_flags, list(wantpols.values()) + list(crosspols.values()), jd, uv['inttime'], uv['sfreq'], uv['sdf'], sa.lat, sa.lon, overwrite=False, comment=ano)
del(uv)
print "Done."
sys.stdout.flush()
if create_new_uvs:
print FILENAME + " MSG: saving new uv files",
sys.stdout.flush()
infos = {}
for pol in wantpols.keys():
infos[pol[0]] = omni.read_redundantinfo(infopaths[pol])
for uvfile in uvfiles:
omni.apply_omnigain_uvs([uvfile], omnigains, calibrators[wantpols.keys()[0]].totalVisibilityId, infos, oppath, ano, adds= adds, verbose = True, comment = '_'.join(sys.argv), flags = flags, overwrite = overwrite_uvs)
print "Done."
sys.stdout.flush()
TIMER.tick()
if make_plots:
for p,pol in zip(range(len(wantpols)), wantpols.keys()):
#plt.subplot(2, len(wantpols), 2 * p + 1)
#plot_data = (calibrators[pol].rawCalpar[:,:,2]/(len(calibrators[pol].Info.subsetbl)-calibrators[pol].Info.nAntenna - calibrators[pol].Info.nUBL))**.5
#plt.imshow(plot_data, vmin = 0, vmax = (np.nanmax(calibrators[wantpols.keys()[0]].rawCalpar[:,:,2][~flags[wantpols.keys()[0]]])/(len(calibrators[pol].Info.subsetbl)-calibrators[pol].Info.nAntenna - calibrators[pol].Info.nUBL))**.5, interpolation='nearest')
#plt.title('RMS fitting error per baseline')
#plt.colorbar()
import simulate_visibilities.simulate_visibilities as sv
import numpy as np
import numpy.linalg as la
import time, ephem, sys
import aipy as ap
import matplotlib.pyplot as plt
import healpy as hp
import healpy.pixelfunc as hpf
import scipy.interpolate as si
import omnical.calibration_omni as omni
tlist = np.arange(16, 24, .1)
infofile = '/home/omniscope/omnical/doc/redundantinfo_X5_q3y.bin' #'/home/omniscope/omnical/doc/redundantinfo_PSA128_26ba_6bu_08-15-2014.bin'#
pol = 'yy'
info = omni.read_redundantinfo(infofile)
nside = 8
nside_target = 4
inclusion_thresh = 1 #betweennnn 0 and 1. ubl/lambda must be this times nside_target less
all_ubl = False
bnside = 8
freq = 160.
vs = sv.Visibility_Simulator()
if 'X5' in infofile and 'PSA' in infofile:
raise Exception('Cant tell if the iinfo is for PAPER or MITEoR')
elif 'X5' in infofile:
lat_degree = 45.2977
infoubl = info['ubl'].dot([[1.5, 0, 0], [0, 1.5, 0], [0, 0, 0]])
opts,args = o.parse_args(sys.argv[1:])
uvfiles = [os.path.expanduser(arg) for arg in args]
oppath = os.path.expanduser(opts.outputpath)
if not os.path.isdir(oppath):
os.makedirs(oppath)
overwrite_uvs = opts.overwrite
ano = opts.tag
if opts.xgain == "NOTGIVEN" and opts.ygain == "NOTGIVEN":
raise IOError("At least -x and -y omnigain files are required.")
gainpaths = {}
omnigains = {}
infos = {}
flags = {}
if opts.xgain != "NOTGIVEN":
gainpaths['x'] = os.path.expanduser(opts.xgain)
if opts.ygain != "NOTGIVEN":
gainpaths['y'] = os.path.expanduser(opts.ygain)
for pol in gainpaths.keys():
infos[pol] = omni.read_redundantinfo(gainpaths[pol].replace('.omnigain', '.binfo'))
omnigains[pol] = omni.load_omnigain(gainpaths[pol], infos[pol])
flags[pol] = np.fromfile(gainpaths[pol].replace('.omnigain', '.omniflag'), dtype='bool').reshape((len(omnigains[pol]), int(omnigains[pol][0,0,3])))
##########################################################################
omni.apply_omnigain_uvs(uvfiles, omnigains, [[0,0]], infos, oppath, ano, verbose = True, comment = '_'.join(sys.argv), flags = flags, overwrite = overwrite_uvs)
raise IOError("File format not recognized. Must contain .omnigain, .omnifit, .omnichisq, or .uv")
######################################################################
############## read redundant info and antenna numbers; perform sanity checks ###################################
######################################################################
if plottype == 'uv' or plottype == 'odf':
if len(datafiles) != 1:
raise IOError("For uv file plotting only 1 file accepted. Received %i."%len(datafiles))
if not os.path.isdir(datafiles[0]):
raise IOError("UV file not found: %s."%(datafiles[0]))
if infopath is None:
raise IOError("For uv file plotting you must supply a redundant info file.")
elif not os.path.isfile(infopath):
raise IOError("Redundant info file not found: %s."%infopath)
else:
info = omni.read_redundantinfo(infopath)
if time is None or frequency is None:
raise IOError("For uv file plotting you must supply both a time index (-t) and a frequency index (-f).")
else:
if time is not None and frequency is not None:
raise IOError("Cannot accept both -t and -f options for %s file. Accepts at most one out of the two options."%(plottype))
if mode not in ['amp', 'phs', 'real', 'imag']:
raise IOError("Specified mode %s not recognized. Only allow amp, phs, real, imag.")
for datafile in datafiles:
if not os.path.isfile(datafile):
raise IOError("%s file not found: %s."%(plottype, datafile))
treasure_path = args[0]
nLST = int(args[1])
nFrequency = int(args[2])
if os.path.isdir(os.path.expanduser(treasure_path)):
raise IOError("Treasure path %s exists."%treasure_path)
pols = opts.pol
if pols is None:
raise TypeError("polarization not set. Use -p to set polarizations like -p xx,yy")
else:
pols = pols.split(',')
if np.max([len(pol) for pol in pols]) > 9:
raise TypeError("polarization string cannot exceed 9 characters. Use -p to set polarizations like -p xx,yy")
treasure = omni.Treasure(treasure_path, nlst = nLST, nfreq = nFrequency)
try:
if opts.infopath is not None:
info = omni.read_redundantinfo(os.path.expanduser(opts.infopath))
for p, pol in enumerate(pols):
for i, ublvec in enumerate(info['ubl']):
print pol, ublvec, "%i/%i"%(p * len(info['ubl']) + i, len(pols) * len(info['ubl']))
sys.stdout.flush()
treasure.add_coin((pol, ublvec))
print "Done."
except KeyboardInterrupt:
treasure.burn()
print "Aborted due to KeyboardInterrupt."
print nTimes, nFrequencies
jds = np.concatenate([omni.get_omnitime(fit) for fit in omnifits['xx']], axis=0)
flags = {}
for pol in ['xx', 'yy']:
flags[pol] = np.concatenate([np.fromfile(fname, dtype = 'bool').reshape((nTimes[i], nFrequencies[i])) for i, fname in enumerate(sorted(glob.glob("/home/omniscope/data/X5/2015calibration/*_%s_*%s.omniflag"%(Q, pol))))], axis=0)
flag = flags['xx']|flags['yy']
for p, pol in enumerate(['xx', 'xy', 'yx', 'yy']):
omnifits[pol] = np.concatenate([fit[..., 6::2] + 1.j*fit[..., 7::2] for fit in omnifits[pol]], axis=0).transpose((0,2,1))
omnifits[pol][flag] = np.nan
info = omni.read_redundantinfo(glob.glob("/home/omniscope/data/X5/2015calibration/*_%s_*xx.binfo"%Q)[0])
redundancy_sort = np.argsort(info['ublcount'])
raw_vars = {}
for pol in ['xx', 'xy', 'yx', 'yy']:
omnichisq = np.concatenate([omni.load_omnichisq(fname)[:,3:] for fname in sorted(glob.glob("/home/omniscope/data/X5/2015calibration/*_%s_*%s.omnichisq"%(Q,pol)))], axis=0)
if pol[0] == pol[1]:
raw_vars[pol] = omnichisq / (info['nBaseline'] - info['nAntenna'] - info['nUBL'] + 2)
else:
raw_vars[pol] = omnichisq / info['nBaseline']
raw_vars[pol] = np.outer(raw_vars[pol], 1./info['ublcount']).reshape(list(raw_vars[pol].shape) + [info['nUBL']])
####process the pi jump in xy and yx
per_file_avg = {}
pi_flip = {}
flip_use_ubls = redundancy_sort[-10:]
import numpy as np
import omnical.calibration_omni as omni
import matplotlib.pyplot as plt
import aipy as ap
paper_uvfiles = [
'/data4/paper/2012EoR/psa_live/psa6266/zen.2456266.64714.uvcRREcAC'
]
omni_uvfiles = [
"/data4/paper/2012EoR/psa_live/forlstbinning_omnical_2/zen.2456266.64714.uvcRREcACO"
]
info = omni.read_redundantinfo(
'/data2/home/hz2ug/omnical/doc/redundantinfo_PSA64_ba19_37_50.bin')
nant = 64
wantpols = {}
wantpols['xx'] = ap.miriad.str2pol['xx']
papercal_data, _, _, _, rawflag = omni.importuvs(paper_uvfiles,
wantpols,
timingTolerance=100)
omnical_data, _, _, _, rawflag = omni.importuvs(omni_uvfiles,
wantpols,
timingTolerance=100)
data = omni.omniview(
np.array([papercal_data[0, 0, 120], omnical_data[0, 0, 120]]), info)
np.savez('/data2/home/zakiali/jeff_psa64/omniview.npz', data)
tlast = None
for uvfile in uvfiles:
print "Processing", uvfile
uv = a.miriad.UV(uvfile)
for (uvw,t,bl), d in uv.all():
if t != tlast:
tlast = t
aa.set_jultime(t)
lsts.append(aa.sidereal_time())
np.savetxt(lst_file, np.array(lsts))
lsts = np.array(lsts)
#info=omni.read_redundantinfo('/data2/home/hz2ug/omnical/doc/redundantinfo_PSA64_ba19_37_50.bin')
#chis = [np.fromfile("/data4/paper/2012EoR/psa_live/forlstbinning_omnical_2/data_psa%i_245%i.%i_xx_add7.omnical"%(D, D, i), dtype = 'float32')[2::3+2*(info['nAntenna'] + info['nUBL'])] for i in range(1,7)]
info=omni.read_redundantinfo('/data2/home/hz2ug/omnical/doc/redundantinfo_PSA64_7ba_7bu_08-15-2014.bin')
chis = [np.fromfile("/data4/paper/2012EoR/psa_live/PSA64_omnical_results_Aug_2014/psa%i_245%i.%i_aug2014_xx.omnical"%(D, D, i), dtype = 'float32')[2::3+2*(info['nAntenna'] + info['nUBL'])] for i in range(1,7)]
chis = [chi.reshape((len(chi)/nchan, nchan)) for chi in chis]
chi2 = np.concatenate((chis), axis = 0)
dof = len(info['crossindex']) * 2*nadd / (2*nadd + 1) - (info['nAntenna'] + info['nUBL'] - 2)
f = np.load('/data2/home/hz2ug/PSA64_plot_for_zaki/tsys_model_jy_corrected.npz')
noise = np.array(f['tsys_jy'] **2) / (43*100./nchan*1e6)
noise_model = interpolate.interp2d(np.arange(0, nchan), f['lsts']/12.*np.pi, noise)
#if min(lsts[1:] - lsts[:-1]) >= 0:
# modeled_noise = noise_model(range(0, nchan), lsts)
import numpy as np
import numpy.linalg as la
import time, ephem, sys
import aipy as ap
import matplotlib.pyplot as plt
import healpy as hp
import healpy.pixelfunc as hpf
import scipy.interpolate as si
import omnical.calibration_omni as omni
tlist = np.arange(16, 24, .1)
p = 'y'
infofile = '/home/omniscope/omnical/doc/redundantinfo_X5_q3%s.bin'%(p)#'/home/omniscope/omnical/doc/redundantinfo_PSA128_26ba_6bu_08-15-2014.bin'#
pol = p+p
info = omni.read_redundantinfo(infofile)
nside = 16
nside_target = 16
inclusion_thresh = 1 #betweennnn 0 and 1. ubl/lambda must be this times nside_target less
all_ubl = False
bnside = 8
freq = 160.
vs = sv.Visibility_Simulator()
if 'X5' in infofile and 'PSA' in infofile:
raise Exception('Cant tell if the iinfo is for PAPER or MITEoR')
elif 'X5' in infofile:
lat_degree = 45.2977
infoubl = info['ubl'].dot([[1.5,0,0],[0,1.5,0],[0,0,0]])
sa.lat = uv['latitud']
sun = ephem.Sun()
del(uv)
#get data for all freq and lst, only care abt first time slice of each file
NT = 100
NF = 203
CEILING = 2.e8
for POL in ['xx', 'yy']:
chisqfiles=sorted(glob.glob('/data4/paper/hz2ug/2015PSA64/*%s.omnichisq'%POL))
data = np.zeros((NT, NF)) + CEILING
print "WARNING! assuming same dof for all chisqs"
dof = omni.read_redundantinfo(chisqfiles[0].replace('.omnichisq','.binfo'), DoF_only=True)
for i, chisqfile in enumerate(chisqfiles):
rawd = np.fromfile(chisqfile,dtype='float32')
sa.date = struct.unpack('d', struct.pack('ff', *rawd[:2].tolist()))[0] - julDelta
lst = sa.sidereal_time()
sun.compute(sa)
if sun.alt < -0.1:
nf = rawd[2]
d = rawd[3:3+nf:int(np.floor(nf/NF))][:NF]/dof
flag = np.fromfile(chisqfile.replace('.omnichisq','.omniflag'),dtype='bool')[:nf:int(np.floor(nf/NF))][:NF]
#print flag.dtype, (d==0).dtype
flag = flag|(d==0)
script_ver = "2"
# get data for all freq and lst, only care abt first time slice of each file
NT = 100
NF = 203
CEILING = 2.0e8
MIN_FILTER_SIZE = (1, 1)
HEAVY_FILTER_FREQ_RANGE = [60, 150]
HEAVY_MIN_FILTER_SIZE = (1, 1) # (1, 15)
MEDIAN_FILTER_SIZE = (6, 3)
for POL in ["xx", "yy"]:
chisqfiles = sorted(glob.glob("/data4/paper/hz2ug/2015PSA64/zen*%s*%s.omnichisq" % (tag, POL)))
data = np.zeros((NT, NF)) + CEILING
print "WARNING! assuming same dof for all chisqs"
dof = omni.read_redundantinfo(chisqfiles[0].replace(".omnichisq", ".binfo"), DoF_only=True)
for i, chisqfile in enumerate(chisqfiles):
# print '.',
rawd = np.fromfile(chisqfile, dtype="float32")
sa.date = struct.unpack("d", struct.pack("ff", *rawd[:2].tolist()))[0] - julDelta
lst = sa.sidereal_time()
sun.compute(sa)
if sun.alt < -0.1:
nf = rawd[2]
nt = len(rawd) / (3 + nf)
rawd.shape = (nt, 3 + nf)
flag = (
np.fromfile(chisqfile.replace(".omnichisq", ".omniflag"), dtype="bool")
.reshape((nt, nf))[:, :: int(np.floor(nf / NF))][:, :NF]
if removeadditive:
adds[key].tofile(oppath + '/' + dataano + '_' + ano + "_%s.omniadd"%key + str(removeadditiveperiod))
#calibrators[key].get_calibrated_data(data[p])
#calibrators[key].get_omnichisq()
#calibrators[key].get_omnifit()
print "Done"
sys.stdout.flush()
diag_txt = calibrators[key].diagnose(data = data[p], additiveout = additiveout, healthbar = healthbar, ubl_healthbar = ubl_healthbar, ouput_txt = True)
text_file = open(oppath + '/' + dataano + '_' + ano + "_%s.diagtxt"%key, "a")
text_file.write(diag_txt)
text_file.close()
if create_new_uvs:
print FILENAME + " MSG: saving new uv files",
sys.stdout.flush()
infos = {}
for key in wantpols.keys():
infos[key] = omni.read_redundantinfo(infopaths[key])
omni.apply_omnigain_uvs(uvfiles, omnigains, calibrators[wantpols.keys()[0]].totalVisibilityId, infos, wantpols, oppath, ano, adds= adds, verbose = True, comment = '_'.join(sys.argv), overwrite = overwrite_uvs)
print "Done"
sys.stdout.flush()
if make_plots:
import matplotlib.pyplot as plt
for p,pol in zip(range(len(wantpols)), wantpols.keys()):
plt.subplot(1,len(wantpols),p+1)
plot_data = (calibrators[pol].rawCalpar[:,:,2]/(len(calibrators[pol].Info.subsetbl)-calibrators[pol].Info.nAntenna - calibrators[pol].Info.nUBL))**.5
plt.imshow(plot_data, vmin = 0, vmax = (np.nanmax(calibrators[wantpols.keys()[0]].rawCalpar[:,30:-30:5,2])/(len(calibrators[pol].Info.subsetbl)-calibrators[pol].Info.nAntenna - calibrators[pol].Info.nUBL))**.5, interpolation='nearest')
plt.title('RMS fitting error per baseline')
plt.colorbar()
plt.show()
uv['sfreq'],
uv['sdf'],
sa.lat,
sa.lon,
overwrite=False,
comment=ano)
del (uv)
print "Done."
sys.stdout.flush()
if create_new_uvs:
print FILENAME + " MSG: saving new uv files",
sys.stdout.flush()
infos = {}
for pol in wantpols.keys():
infos[pol[0]] = omni.read_redundantinfo(infopaths[pol])
omni.apply_omnigain_uvs(uvfiles,
omnigains,
calibrators[wantpols.keys()[0]].totalVisibilityId,
infos,
oppath,
ano,
adds=adds,
verbose=True,
comment='_'.join(sys.argv),
flags=flags,
overwrite=overwrite_uvs)
print "Done."
sys.stdout.flush()
TIMER.tick()