force_recompute = False
nside = 32
nt = 80
nside_standard = 32  #64
nt_standard = 80  #160
nUBL = 75  #34
#nside = 16
#nUBL = 75
if ps:
    nps = 2  #number of point sources to include
else:
    nps = 0
freq = 160.
thresh = 1
ubls = np.array([
    bl for bl in omni.read_redundantinfo(
        '/home/omniscope/omnical/doc/redundantinfo_X5_q3x.bin')['ubl'] *
    [1.5, 1.5, 0] if la.norm(bl) < thresh * (nside * 299.792458 / freq)
])
if len(ubls) != nUBL:
    raise Exception('%i != %i!' % (len(ubls), nUBL))

pca1 = hp.fitsfunc.read_map(
    '/home/omniscope/simulate_visibilities/data/gsm1.fits' +
    str(nside_standard))
pca2 = hp.fitsfunc.read_map(
    '/home/omniscope/simulate_visibilities/data/gsm2.fits' +
    str(nside_standard))
pca3 = hp.fitsfunc.read_map(
    '/home/omniscope/simulate_visibilities/data/gsm3.fits' +
    str(nside_standard))
gsm_standard = 422.952 * (0.307706 * pca1 + -0.281772 * pca2 +
コード例 #2
0
import sys, os
import omnical.calibration_omni as omni

tavg = 2
force_recompute = False
nside = 16
nt = 80
nside_standard = 16#64
nt_standard = 80#160
nUBL = 75#34
#nside = 16
#nUBL = 75
nps = 2#number of point sources to include
freq = 160.
thresh = 1
ubls = np.array([bl for bl in omni.read_redundantinfo('/home/omniscope/omnical/doc/redundantinfo_X5_q3x.bin')['ubl']*[1.5, 1.5, 0] if la.norm(bl) < thresh * (nside * 299.792458 / freq)])
if len(ubls) != nUBL:
    raise Exception('%i != %i!'%(len(ubls), nUBL))

#compute S
angular_scale = 1/(freq/300*np.max([la.norm(ubl) for ubl in ubls]))
print "Computing S matrix...",
sys.stdout.flush()
S = np.identity(12 * nside**2)
S = np.maximum(np.array([hp.sphtfunc.smoothing(pix_vec, sigma = angular_scale, verbose = False) for pix_vec in S]), 0)


pca1 = hp.fitsfunc.read_map('/home/omniscope/simulate_visibilities/data/gsm1.fits' + str(nside_standard))
pca2 = hp.fitsfunc.read_map('/home/omniscope/simulate_visibilities/data/gsm2.fits' + str(nside_standard))
pca3 = hp.fitsfunc.read_map('/home/omniscope/simulate_visibilities/data/gsm3.fits' + str(nside_standard))
gsm_standard = 422.952*(0.307706*pca1+-0.281772*pca2+0.0123976*pca3)
コード例 #3
0
import numpy as np
import omnical.calibration_omni as omni
import matplotlib.pyplot as plt
import aipy as ap


paper_uvfiles = ['/data4/paper/2012EoR/psa_live/psa6266/zen.2456266.64714.uvcRREcAC']
omni_uvfiles = ["/data4/paper/2012EoR/psa_live/forlstbinning_omnical_2/zen.2456266.64714.uvcRREcACO"]
info=omni.read_redundantinfo('/data2/home/hz2ug/omnical/doc/redundantinfo_PSA64_ba19_37_50.bin')
nant = 64
wantpols = {}
wantpols['xx'] = ap.miriad.str2pol['xx']

papercal_data, _, _, _ = omni.importuvs(paper_uvfiles, np.concatenate([[[i,j] for i in range(j + 1)] for j in range(nant)]), wantpols, timingTolerance=100)
omnical_data, _, _, _ = omni.importuvs(omni_uvfiles, np.concatenate([[[i,j] for i in range(j + 1)] for j in range(nant)]), wantpols, timingTolerance=100)

omni.omniview(np.array([papercal_data[0,0,120], omnical_data[0,0,120]]), info)
コード例 #4
0
ファイル: omnical4.py プロジェクト: JIANSHULI/omnical
    else:
        compr_uvfile = oppath + '/' + dataano + ".uvOE"


    uv=ap.miriad.UV(uvfiles[0])
    omni.exportuv(compr_uvfile, compressed_data, compr_flags, list(wantpols.values()) + list(crosspols.values()), jd, uv['inttime'], uv['sfreq'], uv['sdf'], sa.lat, sa.lon, overwrite=False, comment=ano)
    del(uv)
    print "Done."
    sys.stdout.flush()

if create_new_uvs:
    print FILENAME + " MSG: saving new uv files",
    sys.stdout.flush()
    infos = {}
    for pol in wantpols.keys():
        infos[pol[0]] = omni.read_redundantinfo(infopaths[pol])
    for uvfile in uvfiles:
        omni.apply_omnigain_uvs([uvfile], omnigains, calibrators[wantpols.keys()[0]].totalVisibilityId, infos, oppath, ano, adds= adds, verbose = True, comment = '_'.join(sys.argv), flags = flags, overwrite = overwrite_uvs)
    print "Done."
    sys.stdout.flush()

TIMER.tick()

if make_plots:
    for p,pol in zip(range(len(wantpols)), wantpols.keys()):
        #plt.subplot(2, len(wantpols), 2 * p + 1)
        #plot_data = (calibrators[pol].rawCalpar[:,:,2]/(len(calibrators[pol].Info.subsetbl)-calibrators[pol].Info.nAntenna - calibrators[pol].Info.nUBL))**.5
        #plt.imshow(plot_data, vmin = 0, vmax = (np.nanmax(calibrators[wantpols.keys()[0]].rawCalpar[:,:,2][~flags[wantpols.keys()[0]]])/(len(calibrators[pol].Info.subsetbl)-calibrators[pol].Info.nAntenna - calibrators[pol].Info.nUBL))**.5, interpolation='nearest')
        #plt.title('RMS fitting error per baseline')
        #plt.colorbar()
コード例 #5
0
import simulate_visibilities.simulate_visibilities as sv
import numpy as np
import numpy.linalg as la
import time, ephem, sys
import aipy as ap
import matplotlib.pyplot as plt
import healpy as hp
import healpy.pixelfunc as hpf
import scipy.interpolate as si
import omnical.calibration_omni as omni

tlist = np.arange(16, 24, .1)

infofile = '/home/omniscope/omnical/doc/redundantinfo_X5_q3y.bin'  #'/home/omniscope/omnical/doc/redundantinfo_PSA128_26ba_6bu_08-15-2014.bin'#
pol = 'yy'
info = omni.read_redundantinfo(infofile)

nside = 8
nside_target = 4
inclusion_thresh = 1  #betweennnn 0 and 1. ubl/lambda must be this times nside_target less
all_ubl = False
bnside = 8

freq = 160.

vs = sv.Visibility_Simulator()
if 'X5' in infofile and 'PSA' in infofile:
    raise Exception('Cant tell if the iinfo is for PAPER or MITEoR')
elif 'X5' in infofile:
    lat_degree = 45.2977
    infoubl = info['ubl'].dot([[1.5, 0, 0], [0, 1.5, 0], [0, 0, 0]])
コード例 #6
0
ファイル: apply_omnigain.py プロジェクト: JIANSHULI/omnical
opts,args = o.parse_args(sys.argv[1:])
uvfiles = [os.path.expanduser(arg) for arg in args]

oppath = os.path.expanduser(opts.outputpath)
if not os.path.isdir(oppath):
	os.makedirs(oppath)

overwrite_uvs = opts.overwrite
ano = opts.tag

if opts.xgain == "NOTGIVEN" and opts.ygain == "NOTGIVEN":
	raise IOError("At least -x and -y  omnigain files are required.")

gainpaths = {}
omnigains = {}
infos = {}
flags = {}
if opts.xgain != "NOTGIVEN":
	gainpaths['x'] = os.path.expanduser(opts.xgain)

if opts.ygain != "NOTGIVEN":
	gainpaths['y'] = os.path.expanduser(opts.ygain)


for pol in gainpaths.keys():
	infos[pol] = omni.read_redundantinfo(gainpaths[pol].replace('.omnigain', '.binfo'))
	omnigains[pol] = omni.load_omnigain(gainpaths[pol], infos[pol])
	flags[pol] = np.fromfile(gainpaths[pol].replace('.omnigain', '.omniflag'), dtype='bool').reshape((len(omnigains[pol]), int(omnigains[pol][0,0,3])))
##########################################################################
omni.apply_omnigain_uvs(uvfiles, omnigains, [[0,0]], infos, oppath, ano, verbose = True, comment = '_'.join(sys.argv), flags = flags, overwrite = overwrite_uvs)
コード例 #7
0
    raise IOError("File format not recognized. Must contain .omnigain, .omnifit, .omnichisq, or .uv")

######################################################################
############## read redundant info and antenna numbers; perform sanity checks ###################################
######################################################################
if plottype == 'uv' or plottype == 'odf':
    if len(datafiles) != 1:
        raise IOError("For uv file plotting only 1 file accepted. Received %i."%len(datafiles))
    if not os.path.isdir(datafiles[0]):
        raise IOError("UV file not found: %s."%(datafiles[0]))
    if infopath is None:
        raise IOError("For uv file plotting you must supply a redundant info file.")
    elif not os.path.isfile(infopath):
        raise IOError("Redundant info file not found: %s."%infopath)
    else:
        info = omni.read_redundantinfo(infopath)

    if time is None or frequency is None:
        raise IOError("For uv file plotting you must supply both a time index (-t) and a frequency index (-f).")

else:
    if time is not None and frequency is not None:
        raise IOError("Cannot accept both -t and -f options for %s file. Accepts at most one out of the two options."%(plottype))

    if mode not in ['amp', 'phs', 'real', 'imag']:
        raise IOError("Specified mode %s not recognized. Only allow amp, phs, real, imag.")

    for datafile in datafiles:
        if not os.path.isfile(datafile):
            raise IOError("%s file not found: %s."%(plottype, datafile))
コード例 #8
0
ファイル: bury_treasure.py プロジェクト: JIANSHULI/omnical
treasure_path = args[0]
nLST = int(args[1])
nFrequency = int(args[2])

if os.path.isdir(os.path.expanduser(treasure_path)):
	raise IOError("Treasure path %s exists."%treasure_path)

pols = opts.pol
if pols is None:
	raise TypeError("polarization not set. Use -p to set polarizations like -p xx,yy")
else:
	pols = pols.split(',')

if np.max([len(pol) for pol in pols]) > 9:
	raise TypeError("polarization string cannot exceed 9 characters. Use -p to set polarizations like -p xx,yy")

treasure = omni.Treasure(treasure_path, nlst = nLST, nfreq = nFrequency)
try:
	if opts.infopath is not None:
		info = omni.read_redundantinfo(os.path.expanduser(opts.infopath))
		for p, pol in enumerate(pols):
			for i, ublvec in enumerate(info['ubl']):
				print pol, ublvec, "%i/%i"%(p * len(info['ubl']) + i, len(pols) * len(info['ubl']))
				sys.stdout.flush()
				treasure.add_coin((pol, ublvec))
		print "Done."
except KeyboardInterrupt:
	treasure.burn()
	print "Aborted due to KeyboardInterrupt."
コード例 #9
0
    print nTimes, nFrequencies

    jds = np.concatenate([omni.get_omnitime(fit) for fit in omnifits['xx']], axis=0)



    flags = {}
    for pol in ['xx', 'yy']:
        flags[pol] = np.concatenate([np.fromfile(fname, dtype = 'bool').reshape((nTimes[i], nFrequencies[i])) for i, fname in enumerate(sorted(glob.glob("/home/omniscope/data/X5/2015calibration/*_%s_*%s.omniflag"%(Q, pol))))], axis=0)
    flag = flags['xx']|flags['yy']


    for p, pol in enumerate(['xx', 'xy', 'yx', 'yy']):
        omnifits[pol] = np.concatenate([fit[..., 6::2] + 1.j*fit[..., 7::2] for fit in omnifits[pol]], axis=0).transpose((0,2,1))
        omnifits[pol][flag] = np.nan
    info = omni.read_redundantinfo(glob.glob("/home/omniscope/data/X5/2015calibration/*_%s_*xx.binfo"%Q)[0])
    redundancy_sort = np.argsort(info['ublcount'])

    raw_vars = {}
    for pol in ['xx', 'xy', 'yx', 'yy']:
        omnichisq = np.concatenate([omni.load_omnichisq(fname)[:,3:] for fname in sorted(glob.glob("/home/omniscope/data/X5/2015calibration/*_%s_*%s.omnichisq"%(Q,pol)))], axis=0)
        if pol[0] == pol[1]:
            raw_vars[pol] = omnichisq / (info['nBaseline'] - info['nAntenna'] - info['nUBL'] + 2)
        else:
            raw_vars[pol] = omnichisq / info['nBaseline']
        raw_vars[pol] = np.outer(raw_vars[pol], 1./info['ublcount']).reshape(list(raw_vars[pol].shape) + [info['nUBL']])

    ####process the pi jump in xy and yx
    per_file_avg = {}
    pi_flip = {}
    flip_use_ubls = redundancy_sort[-10:]
コード例 #10
0
import numpy as np
import omnical.calibration_omni as omni
import matplotlib.pyplot as plt
import aipy as ap

paper_uvfiles = [
    '/data4/paper/2012EoR/psa_live/psa6266/zen.2456266.64714.uvcRREcAC'
]
omni_uvfiles = [
    "/data4/paper/2012EoR/psa_live/forlstbinning_omnical_2/zen.2456266.64714.uvcRREcACO"
]
info = omni.read_redundantinfo(
    '/data2/home/hz2ug/omnical/doc/redundantinfo_PSA64_ba19_37_50.bin')
nant = 64
wantpols = {}
wantpols['xx'] = ap.miriad.str2pol['xx']

papercal_data, _, _, _, rawflag = omni.importuvs(paper_uvfiles,
                                                 wantpols,
                                                 timingTolerance=100)
omnical_data, _, _, _, rawflag = omni.importuvs(omni_uvfiles,
                                                wantpols,
                                                timingTolerance=100)

data = omni.omniview(
    np.array([papercal_data[0, 0, 120], omnical_data[0, 0, 120]]), info)

np.savez('/data2/home/zakiali/jeff_psa64/omniview.npz', data)
コード例 #11
0
    tlast = None
    for uvfile in uvfiles:
        print "Processing", uvfile
        uv = a.miriad.UV(uvfile)
        for (uvw,t,bl), d in uv.all():
            if t != tlast:
                tlast = t
                aa.set_jultime(t)
                lsts.append(aa.sidereal_time())
    np.savetxt(lst_file, np.array(lsts))
    lsts = np.array(lsts)

#info=omni.read_redundantinfo('/data2/home/hz2ug/omnical/doc/redundantinfo_PSA64_ba19_37_50.bin')
#chis = [np.fromfile("/data4/paper/2012EoR/psa_live/forlstbinning_omnical_2/data_psa%i_245%i.%i_xx_add7.omnical"%(D, D, i), dtype = 'float32')[2::3+2*(info['nAntenna'] + info['nUBL'])] for i in range(1,7)]

info=omni.read_redundantinfo('/data2/home/hz2ug/omnical/doc/redundantinfo_PSA64_7ba_7bu_08-15-2014.bin')
chis = [np.fromfile("/data4/paper/2012EoR/psa_live/PSA64_omnical_results_Aug_2014/psa%i_245%i.%i_aug2014_xx.omnical"%(D, D, i), dtype = 'float32')[2::3+2*(info['nAntenna'] + info['nUBL'])] for i in range(1,7)]


chis = [chi.reshape((len(chi)/nchan, nchan)) for chi in chis]
chi2 = np.concatenate((chis), axis = 0)
dof = len(info['crossindex']) * 2*nadd / (2*nadd + 1) - (info['nAntenna'] + info['nUBL'] - 2)



f = np.load('/data2/home/hz2ug/PSA64_plot_for_zaki/tsys_model_jy_corrected.npz')
noise = np.array(f['tsys_jy'] **2) / (43*100./nchan*1e6)

noise_model = interpolate.interp2d(np.arange(0, nchan), f['lsts']/12.*np.pi, noise)
#if min(lsts[1:] - lsts[:-1]) >= 0:
#    modeled_noise = noise_model(range(0, nchan), lsts)
コード例 #12
0
import numpy as np
import numpy.linalg as la
import time, ephem, sys
import aipy as ap
import matplotlib.pyplot as plt
import healpy as hp
import healpy.pixelfunc as hpf
import scipy.interpolate as si
import omnical.calibration_omni as omni

tlist = np.arange(16, 24, .1)

p = 'y'
infofile = '/home/omniscope/omnical/doc/redundantinfo_X5_q3%s.bin'%(p)#'/home/omniscope/omnical/doc/redundantinfo_PSA128_26ba_6bu_08-15-2014.bin'#
pol = p+p
info = omni.read_redundantinfo(infofile)
nside = 16
nside_target = 16

inclusion_thresh = 1 #betweennnn 0 and 1. ubl/lambda must be this times nside_target less
all_ubl = False
bnside = 8

freq = 160.

vs = sv.Visibility_Simulator()
if 'X5' in infofile and 'PSA' in infofile:
    raise Exception('Cant tell if the iinfo is for PAPER or MITEoR')
elif 'X5' in infofile:
    lat_degree = 45.2977
    infoubl = info['ubl'].dot([[1.5,0,0],[0,1.5,0],[0,0,0]])
コード例 #13
0
sa.lat = uv['latitud']
sun = ephem.Sun()
del(uv)

#get data for all freq and lst, only care abt first time slice of each file
NT = 100
NF = 203
CEILING = 2.e8

for POL in ['xx', 'yy']:
    chisqfiles=sorted(glob.glob('/data4/paper/hz2ug/2015PSA64/*%s.omnichisq'%POL))

    data = np.zeros((NT, NF)) + CEILING

    print "WARNING! assuming same dof for all chisqs"
    dof = omni.read_redundantinfo(chisqfiles[0].replace('.omnichisq','.binfo'), DoF_only=True)
    for i, chisqfile in enumerate(chisqfiles):
        rawd = np.fromfile(chisqfile,dtype='float32')
        
        sa.date = struct.unpack('d', struct.pack('ff', *rawd[:2].tolist()))[0] - julDelta
        lst = sa.sidereal_time()
        sun.compute(sa)
        if sun.alt < -0.1:
            nf = rawd[2]
            
            d = rawd[3:3+nf:int(np.floor(nf/NF))][:NF]/dof
            flag = np.fromfile(chisqfile.replace('.omnichisq','.omniflag'),dtype='bool')[:nf:int(np.floor(nf/NF))][:NF]
            
            
            #print flag.dtype, (d==0).dtype
            flag = flag|(d==0)
コード例 #14
0
script_ver = "2"
# get data for all freq and lst, only care abt first time slice of each file
NT = 100
NF = 203
CEILING = 2.0e8
MIN_FILTER_SIZE = (1, 1)
HEAVY_FILTER_FREQ_RANGE = [60, 150]
HEAVY_MIN_FILTER_SIZE = (1, 1)  # (1, 15)
MEDIAN_FILTER_SIZE = (6, 3)
for POL in ["xx", "yy"]:
    chisqfiles = sorted(glob.glob("/data4/paper/hz2ug/2015PSA64/zen*%s*%s.omnichisq" % (tag, POL)))

    data = np.zeros((NT, NF)) + CEILING

    print "WARNING! assuming same dof for all chisqs"
    dof = omni.read_redundantinfo(chisqfiles[0].replace(".omnichisq", ".binfo"), DoF_only=True)
    for i, chisqfile in enumerate(chisqfiles):
        # print '.',
        rawd = np.fromfile(chisqfile, dtype="float32")

        sa.date = struct.unpack("d", struct.pack("ff", *rawd[:2].tolist()))[0] - julDelta
        lst = sa.sidereal_time()
        sun.compute(sa)
        if sun.alt < -0.1:
            nf = rawd[2]
            nt = len(rawd) / (3 + nf)
            rawd.shape = (nt, 3 + nf)

            flag = (
                np.fromfile(chisqfile.replace(".omnichisq", ".omniflag"), dtype="bool")
                .reshape((nt, nf))[:, :: int(np.floor(nf / NF))][:, :NF]
コード例 #15
0
        if removeadditive:
            adds[key].tofile(oppath + '/' + dataano + '_' + ano + "_%s.omniadd"%key + str(removeadditiveperiod))
        #calibrators[key].get_calibrated_data(data[p])
        #calibrators[key].get_omnichisq()
        #calibrators[key].get_omnifit()
        print "Done"
        sys.stdout.flush()
    diag_txt = calibrators[key].diagnose(data = data[p], additiveout = additiveout, healthbar = healthbar, ubl_healthbar = ubl_healthbar, ouput_txt = True)
    text_file = open(oppath + '/' + dataano + '_' + ano + "_%s.diagtxt"%key, "a")
    text_file.write(diag_txt)
    text_file.close()

if create_new_uvs:
    print FILENAME + " MSG: saving new uv files",
    sys.stdout.flush()
    infos = {}
    for key in wantpols.keys():
        infos[key] = omni.read_redundantinfo(infopaths[key])
    omni.apply_omnigain_uvs(uvfiles, omnigains, calibrators[wantpols.keys()[0]].totalVisibilityId, infos, wantpols, oppath, ano, adds= adds, verbose = True, comment = '_'.join(sys.argv), overwrite = overwrite_uvs)
    print "Done"
    sys.stdout.flush()
if make_plots:
    import matplotlib.pyplot as plt
    for p,pol in zip(range(len(wantpols)), wantpols.keys()):
        plt.subplot(1,len(wantpols),p+1)
        plot_data = (calibrators[pol].rawCalpar[:,:,2]/(len(calibrators[pol].Info.subsetbl)-calibrators[pol].Info.nAntenna - calibrators[pol].Info.nUBL))**.5
        plt.imshow(plot_data, vmin = 0, vmax = (np.nanmax(calibrators[wantpols.keys()[0]].rawCalpar[:,30:-30:5,2])/(len(calibrators[pol].Info.subsetbl)-calibrators[pol].Info.nAntenna - calibrators[pol].Info.nUBL))**.5, interpolation='nearest')
    plt.title('RMS fitting error per baseline')
    plt.colorbar()
    plt.show()
コード例 #16
0
                  uv['sfreq'],
                  uv['sdf'],
                  sa.lat,
                  sa.lon,
                  overwrite=False,
                  comment=ano)
    del (uv)
    print "Done."
    sys.stdout.flush()

if create_new_uvs:
    print FILENAME + " MSG: saving new uv files",
    sys.stdout.flush()
    infos = {}
    for pol in wantpols.keys():
        infos[pol[0]] = omni.read_redundantinfo(infopaths[pol])
    omni.apply_omnigain_uvs(uvfiles,
                            omnigains,
                            calibrators[wantpols.keys()[0]].totalVisibilityId,
                            infos,
                            oppath,
                            ano,
                            adds=adds,
                            verbose=True,
                            comment='_'.join(sys.argv),
                            flags=flags,
                            overwrite=overwrite_uvs)
    print "Done."
    sys.stdout.flush()

TIMER.tick()