Ejemplo n.º 1
0
####create redundant calibrators################
#calibrators = [omni.RedundantCalibrator(nant, info = infopaths[key]) for key in wantpols.keys()]
calibrators = [omni.RedundantCalibrator(nant) for key in wantpols.keys()]
for calibrator, key in zip(calibrators, wantpols.keys()):
	calibrator.compute_redundantinfo(arrayinfoPath = arrayinfos[key])
	#calibrator.write_redundantinfo(infoPath = './redundantinfo_test_' + key + '.txt', overwrite = True)
###start reading miriads################
##print FILENAME + " MSG:",  len(uvfiles), "uv files to be processed for " + ano
data, t, timing, lst = omni.importuvs(uvfiles, calibrators[0].totalVisibilityId, wantpols, nTotalAntenna = 32,timingTolerance = 2*math.pi, init_mem = 5.e7)
##print FILENAME + " MSG:",  len(t), "slices read."

###raw calibration################
if needrawcal:
	for p, key in zip(range(len(wantpols)), wantpols.keys()):
		rawcalpar = np.fromfile(rawpaths[key], dtype="complex64").reshape(nfreq, nant)
		data[p] = omni.apply_calpar(data[p], rawcalpar, calibrators[p].totalVisibilityId)

#####Save various files read################
###np.savetxt('miriadextract_' + ano + "_sunpos.dat", sunpos[:len(t)], fmt='%8.5f')
##f = open(oppath + 'miriadextract_' + ano + "_localtime.dat",'w')
##for time in timing:
	##f.write("%s\n"%time)
##f.close()
##f = open(oppath + 'miriadextract_' + ano + "_lsthour.dat",'w')
##for l in lst:
	##f.write("%s\n"%l)
##f.close()


####calibrate################
##print FILENAME + " MSG: starting calibration."
Ejemplo n.º 2
0
        calibrators[pol].removeDegeneracy = removedegen
        calibrators[pol].convergePercent = converge_percent
        calibrators[pol].maxIteration = max_iter
        calibrators[pol].stepSize = step_size


        timer = time.time()
        calibrators[pol].compute_redundantinfo(badAntenna = badAntenna, badUBLpair = badUBLpair, antennaLocationTolerance = redundancy_tol)
        print "Redundant info on %s computed in %f minutes."%(pol, (time.time() - timer)/60.)
        info = calibrators[pol].Info.get_info()

        ###prepare rawCalpar for each calibrator and consider, if needed, raw calibration################
        if need_crude_cal:
            initant, solution_path, additional_solution_path, degen, _ = omni.find_solution_path(info, tol = calibrators[pol].antennaLocationTolerance, verbose = False)
            crude_calpar[pol] = np.array([omni.raw_calibrate(rawdata[p, 0, f], info, initant, solution_path, additional_solution_path, degen) for f in range(calibrators[pol].nFrequency)])
            data[p] = omni.apply_calpar(rawdata[p], crude_calpar[pol], calibrators[pol].totalVisibilityId)
        else:
            data[p] = rawdata[p]
        #calibrators[pol].rawCalpar = np.zeros((calibrators[pol].nTime, calibrators[pol].nFrequency, 3 + 2 * (calibrators[pol].Info.nAntenna + calibrators[pol].Info.nUBL)),dtype='float32')

        ################################
        ########calibrate################
        ################################

        ################first round of calibration  #########################
        print FILENAME + " MSG: starting calibration on %s. nTime = %i, nFrequency = %i ..."%(pol, calibrators[pol].nTime, calibrators[pol].nFrequency),
        sys.stdout.flush()
        timer = time.time()
        additivein = np.zeros_like(data[p])
        calibrators[pol].logcal(data[p], additivein, verbose=True)
        additiveout = calibrators[pol].lincal(data[p], additivein, verbose=True)
Ejemplo n.º 3
0
                                           badAntenna=badAntenna,
                                           badUBLpair=badUBLpair)
    info = calibrators[key].Info.get_info()
    calibrators[key].nTime = nt
    calibrators[key].nFrequency = nfreq

    ###prepare rawCalpar for each calibrator and consider, if needed, raw calibration################
    if need_crude_cal:
        initant, solution_path, additional_solution_path, degen, _ = omni.find_solution_path(
            info)
        crude_calpar[key] = np.array([
            omni.raw_calibrate(data[p, 0, f], info, initant, solution_path,
                               additional_solution_path, degen)
            for f in range(calibrators[key].nFrequency)
        ])
        data[p] = omni.apply_calpar(data[p], crude_calpar[key],
                                    calibrators[key].totalVisibilityId)

    calibrators[key].rawCalpar = np.zeros(
        (calibrators[key].nTime, calibrators[key].nFrequency, 3 + 2 *
         (calibrators[key].Info.nAntenna + calibrators[key].Info.nUBL)),
        dtype='float32')

    ####calibrate################
    calibrators[key].removeDegeneracy = removedegen
    calibrators[key].convergePercent = converge_percent
    calibrators[key].maxIteration = max_iter
    calibrators[key].stepSize = step_size

    ################first round of calibration  #########################
    print FILENAME + " MSG: starting calibration on %s %s. nTime = %i, nFrequency = %i ..." % (
        dataano, key, calibrators[key].nTime, calibrators[key].nFrequency),
Ejemplo n.º 4
0
    if input_type == 'uv':
        calibrators[pol] = omni.RedundantCalibrator_PAPER(aa)
    elif input_type == 'odf':
        calibrators[pol] = omni.RedundantCalibrator_X5(header_antloc)
    calibrators[pol].read_redundantinfo(infopaths[pol], verbose=False)
    info = calibrators[pol].Info.get_info()
    calibrators[pol].nTime = len(timing)
    calibrators[pol].nFrequency = nfreq

    ####consolidate 3D flags from uv files into per time/freq 2D flags
    uvflags[pol] = np.any(rawflag[p,:,:,calibrators[pol].subsetbl[calibrators[pol].crossindex]], axis = 0)# aweird transpose happens when slicing

    ###apply, if needed, raw calibration################
    if needrawcal:
        original_data = np.copy(data[p])
        data[p] = omni.apply_calpar(data[p], crude_calpar[pol], calibrators[pol].totalVisibilityId)

    ####calibrate################
    calibrators[pol].removeDegeneracy = removedegen
    calibrators[pol].convergePercent = converge_percent
    calibrators[pol].maxIteration = max_iter
    calibrators[pol].stepSize = step_size

    ################first round of calibration  #########################
    print FILENAME + " MSG: starting calibration on %s %s. nTime = %i, nFrequency = %i ..."%(dataano, pol, calibrators[pol].nTime, calibrators[pol].nFrequency),
    sys.stdout.flush()
    timer = time.time()
    additivein = np.zeros_like(data[p])

    #######################Logcal###############################
    calibrators[pol].logcal(data[p], additivein, nthread = nthread, verbose=True)
Ejemplo n.º 5
0
omnigains = {}
adds = {}
cdata = [None,None]
for p, key in zip(range(len(data)), wantpols.keys()):

    calibrators[key] = RedundantCalibrator_CHIME(len(data_pack['feed_positions']))
    calibrators[key].compute_redundantinfo(data_pack, badAntenna = badAntenna, badUBLpair = badUBLpair)
    info = calibrators[key].Info.get_info()
    calibrators[key].nTime = nt
    calibrators[key].nFrequency = nfreq

    ###prepare rawCalpar for each calibrator and consider, if needed, raw calibration################
    if need_crude_cal:
        initant, solution_path, additional_solution_path, degen, _ = omni.find_solution_path(info)
        crude_calpar[key] = np.array([omni.raw_calibrate(data[p, 0, f], info, initant, solution_path, additional_solution_path, degen) for f in range(calibrators[key].nFrequency)])
        data[p] = omni.apply_calpar(data[p], crude_calpar[key], calibrators[key].totalVisibilityId)

    calibrators[key].rawCalpar = np.zeros((calibrators[key].nTime, calibrators[key].nFrequency, 3 + 2 * (calibrators[key].Info.nAntenna + calibrators[key].Info.nUBL)),dtype='float32')

    ####calibrate################
    calibrators[key].removeDegeneracy = removedegen
    calibrators[key].convergePercent = converge_percent
    calibrators[key].maxIteration = max_iter
    calibrators[key].stepSize = step_size

    ################first round of calibration  #########################
    print FILENAME + " MSG: starting calibration on %s %s. nTime = %i, nFrequency = %i ..."%(dataano, key, calibrators[key].nTime, calibrators[key].nFrequency),
    sys.stdout.flush()
    timer = time.time()
    additivein = np.zeros_like(data[p])
    calibrators[key].logcal(data[p], additivein, verbose=True)
Ejemplo n.º 6
0
    def test_all(self):
        ##FILENAME = "test.py"
        ######################################################################
        ##############Config parameters###################################
        ######################################################################
        ano = 'test'##This is the file name difference for final calibration parameter result file. Result will be saved in miriadextract_xx_ano.omnical
        uvfiles = [os.path.dirname(os.path.realpath(__file__)) + '/test.uv']
        wantpols = {'xx':-5}#, 'yy':-6}

        #infopaths = {'xx':os.path.dirname(os.path.realpath(__file__)) + '/../doc/redundantinfo_PSA32.txt', 'yy':os.path.dirname(os.path.realpath(__file__)) + '/../doc/redundantinfo_PSA32.txt'}
        arrayinfos = {'xx':os.path.dirname(os.path.realpath(__file__)) + '/../doc/arrayinfo_apprx_PAPER32_badUBLpair.txt', 'yy':os.path.dirname(os.path.realpath(__file__)) + '/../doc/arrayinfo_apprx_PAPER32_badUBLpair.txt'}

        oppath = os.path.dirname(os.path.realpath(__file__)) + '/results/'
        if not os.path.isdir(oppath):
            os.makedirs(oppath)
        removedegen = True
        removeadditive = False

        needrawcal = True #if true, (generally true for raw data) you need to take care of having raw calibration parameters in float32 binary format freq x nant
        rawpaths = {'xx':os.path.dirname(os.path.realpath(__file__)) + "/testrawphasecalparrad_xx", 'yy':os.path.dirname(os.path.realpath(__file__)) + "/testrawphasecalparrad_yy"}



        keep_binary_data = True
        keep_binary_calpar = True


        converge_percent = 0.000001
        max_iter = 1000
        step_size = .3

        ######################################################################
        ######################################################################
        ######################################################################

        ########Massage user parameters###################################
        oppath += '/'


        ####get some info from the first uvfile   ################
        uv=ap.miriad.UV(uvfiles[0])
        nfreq = uv.nchan;
        nant = uv['nants'] / 2 # 'nants' counting ant-pols, so divide 2
        startfreq = uv['sfreq']
        dfreq = uv['sdf']
        del(uv)

        ####create redundant calibrators################
        #calibrators = [omni.RedundantCalibrator(nant, info = infopaths[key]) for key in wantpols.keys()]
        calibrators = [omni.RedundantCalibrator(nant) for key in wantpols.keys()]
        for calibrator, key in zip(calibrators, wantpols.keys()):
            calibrator.compute_redundantinfo(arrayinfoPath = arrayinfos[key])
            #calibrator.write_redundantinfo(infoPath = './redundantinfo_test_' + key + '.txt', overwrite = True)
        ###start reading miriads################
        ##print FILENAME + " MSG:",  len(uvfiles), "uv files to be processed for " + ano
        data, t, timing, lst = omni.importuvs(uvfiles, calibrators[0].totalVisibilityId, wantpols, nTotalAntenna = 32,timingTolerance = 2*math.pi, init_mem = 5.e7)
        ##print FILENAME + " MSG:",  len(t), "slices read."

        ###raw calibration################
        if needrawcal:
            for p, key in zip(range(len(wantpols)), wantpols.keys()):
                rawcalpar = np.fromfile(rawpaths[key], dtype="complex64").reshape(nfreq, nant)
                data[p] = omni.apply_calpar(data[p], rawcalpar, calibrators[p].totalVisibilityId)

        #####Save various files read################
        ###np.savetxt('miriadextract_' + ano + "_sunpos.dat", sunpos[:len(t)], fmt='%8.5f')
        ##f = open(oppath + 'miriadextract_' + ano + "_localtime.dat",'w')
        ##for time in timing:
            ##f.write("%s\n"%time)
        ##f.close()
        ##f = open(oppath + 'miriadextract_' + ano + "_lsthour.dat",'w')
        ##for l in lst:
            ##f.write("%s\n"%l)
        ##f.close()


        ####calibrate################
        ##print FILENAME + " MSG: starting calibration."
        for p, calibrator in zip(range(len(wantpols)), calibrators):
            calibrator.removeDegeneracy = removedegen
            calibrator.removeAdditive = removeadditive
            calibrator.keepData = keep_binary_data
            calibrator.keepCalpar = keep_binary_calpar
            calibrator.convergePercent = converge_percent
            calibrator.maxIteration = max_iter
            calibrator.stepSize = step_size
            calibrator.computeUBLFit = False


            calibrator.logcal(data[p], np.zeros_like(data[p]), verbose=True)
            calibrator.lincal(data[p], np.zeros_like(data[p]), verbose=True)

            calibrator.utctimes = timing
            calibrator.get_calibrated_data(data[p])
            calibrator.get_omnichisq()
            calibrator.get_omnigain()
            calibrator.get_omnifit()

        #########Test results############
        correctresult = np.fromfile(os.path.dirname(os.path.realpath(__file__)) + '/test.omnical', dtype = 'float32').reshape(14,203,165)[:,:,:3+2*nant]
        nanmask = ~np.isnan(np.sum(correctresult,axis=2))#mask the last dimension because when data contains some 0 and some -0, C++ code return various phasecalibration parameters on different systems, when all other numbers are nan. I do the summation to avoid it failing the euqality check when the input is trivially 0s.

        newresult = calibrators[-1].rawCalpar[:,:,:3+2*nant]#[:,:,3:]

        #calibrators[-1].rawCalpar.tofile(os.path.dirname(os.path.realpath(__file__)) + '/test.omnical')
        np.testing.assert_almost_equal(correctresult[nanmask], newresult[nanmask])#decimal=
Ejemplo n.º 7
0
            pol, (time.time() - timer) / 60.)
        info = calibrators[pol].Info.get_info()

        ###prepare rawCalpar for each calibrator and consider, if needed, raw calibration################
        if need_crude_cal:
            initant, solution_path, additional_solution_path, degen, _ = omni.find_solution_path(
                info,
                tol=calibrators[pol].antennaLocationTolerance,
                verbose=False)
            crude_calpar[pol] = np.array([
                omni.raw_calibrate(rawdata[p, 0,
                                           f], info, initant, solution_path,
                                   additional_solution_path, degen)
                for f in range(calibrators[pol].nFrequency)
            ])
            data[p] = omni.apply_calpar(rawdata[p], crude_calpar[pol],
                                        calibrators[pol].totalVisibilityId)
        else:
            data[p] = rawdata[p]
        #calibrators[pol].rawCalpar = np.zeros((calibrators[pol].nTime, calibrators[pol].nFrequency, 3 + 2 * (calibrators[pol].Info.nAntenna + calibrators[pol].Info.nUBL)),dtype='float32')

        ################################
        ########calibrate################
        ################################

        ################first round of calibration  #########################
        print FILENAME + " MSG: starting calibration on %s. nTime = %i, nFrequency = %i ..." % (
            pol, calibrators[pol].nTime, calibrators[pol].nFrequency),
        sys.stdout.flush()
        timer = time.time()
        additivein = np.zeros_like(data[p])
        calibrators[pol].logcal(data[p], additivein, verbose=True)
Ejemplo n.º 8
0
    def test_all(self):
        ##FILENAME = "test.py"
        ######################################################################
        ##############Config parameters###################################
        ######################################################################
        ano = 'test'  ##This is the file name difference for final calibration parameter result file. Result will be saved in miriadextract_xx_ano.omnical
        uvfiles = [os.path.dirname(os.path.realpath(__file__)) + '/test.uv']
        wantpols = {'xx': -5}  #, 'yy':-6}

        #infopaths = {'xx':os.path.dirname(os.path.realpath(__file__)) + '/../doc/redundantinfo_PSA32.txt', 'yy':os.path.dirname(os.path.realpath(__file__)) + '/../doc/redundantinfo_PSA32.txt'}
        arrayinfos = {
            'xx':
            os.path.dirname(os.path.realpath(__file__)) +
            '/../doc/arrayinfo_apprx_PAPER32_badUBLpair.txt',
            'yy':
            os.path.dirname(os.path.realpath(__file__)) +
            '/../doc/arrayinfo_apprx_PAPER32_badUBLpair.txt'
        }

        oppath = os.path.dirname(os.path.realpath(__file__)) + '/results/'
        if not os.path.isdir(oppath):
            os.makedirs(oppath)
        removedegen = True
        removeadditive = False

        needrawcal = True  #if true, (generally true for raw data) you need to take care of having raw calibration parameters in float32 binary format freq x nant
        rawpaths = {
            'xx':
            os.path.dirname(os.path.realpath(__file__)) +
            "/testrawphasecalparrad_xx",
            'yy':
            os.path.dirname(os.path.realpath(__file__)) +
            "/testrawphasecalparrad_yy"
        }

        keep_binary_data = True
        keep_binary_calpar = True

        converge_percent = 0.000001
        max_iter = 1000
        step_size = .3

        ######################################################################
        ######################################################################
        ######################################################################

        ########Massage user parameters###################################
        oppath += '/'

        ####get some info from the first uvfile   ################
        uv = ap.miriad.UV(uvfiles[0])
        nfreq = uv.nchan
        nant = uv['nants'] / 2  # 'nants' counting ant-pols, so divide 2
        startfreq = uv['sfreq']
        dfreq = uv['sdf']
        del (uv)

        ####create redundant calibrators################
        #calibrators = [omni.RedundantCalibrator(nant, info = infopaths[key]) for key in wantpols.keys()]
        calibrators = [
            omni.RedundantCalibrator(nant) for key in wantpols.keys()
        ]
        for calibrator, key in zip(calibrators, wantpols.keys()):
            calibrator.compute_redundantinfo(arrayinfoPath=arrayinfos[key])
            #calibrator.write_redundantinfo(infoPath = './redundantinfo_test_' + key + '.txt', overwrite = True)
        ###start reading miriads################
        ##print FILENAME + " MSG:",  len(uvfiles), "uv files to be processed for " + ano
        data, t, timing, lst = omni.importuvs(uvfiles,
                                              calibrators[0].totalVisibilityId,
                                              wantpols,
                                              nTotalAntenna=32,
                                              timingTolerance=2 * math.pi,
                                              init_mem=5.e7)
        ##print FILENAME + " MSG:",  len(t), "slices read."

        ###raw calibration################
        if needrawcal:
            for p, key in zip(range(len(wantpols)), wantpols.keys()):
                rawcalpar = np.fromfile(rawpaths[key],
                                        dtype="complex64").reshape(
                                            nfreq, nant)
                data[p] = omni.apply_calpar(data[p], rawcalpar,
                                            calibrators[p].totalVisibilityId)

        #####Save various files read################
        ###np.savetxt('miriadextract_' + ano + "_sunpos.dat", sunpos[:len(t)], fmt='%8.5f')
        ##f = open(oppath + 'miriadextract_' + ano + "_localtime.dat",'w')
        ##for time in timing:
        ##f.write("%s\n"%time)
        ##f.close()
        ##f = open(oppath + 'miriadextract_' + ano + "_lsthour.dat",'w')
        ##for l in lst:
        ##f.write("%s\n"%l)
        ##f.close()

        ####calibrate################
        ##print FILENAME + " MSG: starting calibration."
        for p, calibrator in zip(range(len(wantpols)), calibrators):
            calibrator.removeDegeneracy = removedegen
            calibrator.removeAdditive = removeadditive
            calibrator.keepData = keep_binary_data
            calibrator.keepCalpar = keep_binary_calpar
            calibrator.convergePercent = converge_percent
            calibrator.maxIteration = max_iter
            calibrator.stepSize = step_size
            calibrator.computeUBLFit = False

            calibrator.logcal(data[p], np.zeros_like(data[p]), verbose=False)
            additiveout = calibrator.lincal(data[p],
                                            np.zeros_like(data[p]),
                                            verbose=False)

            calibrator.utctimes = timing
            calibrator.get_calibrated_data(data[p])
            calibrator.get_omnichisq()
            calibrator.get_omnigain()
            calibrator.get_omnifit()

        #########Test results############
        correctresult = np.fromfile(
            os.path.dirname(os.path.realpath(__file__)) + '/test.omnical',
            dtype='float32').reshape(14, 203, 165)[:, :, :]
        nanmask = ~np.isnan(
            np.sum(correctresult, axis=2)
        )  #mask the last dimension because when data contains some 0 and some -0, C++ code return various phasecalibration parameters on different systems, when all other numbers are nan. I do the summation to avoid it failing the euqality check when the input is trivially 0s.
        calibrators[-1].rawCalpar.tofile(
            os.path.dirname(os.path.realpath(__file__)) +
            '/results/new_result.omnical')
        omni.write_redundantinfo(calibrators[-1].Info.get_info(),
                                 os.path.dirname(os.path.realpath(__file__)) +
                                 '/results/new_info.bin',
                                 overwrite=True)
        newresult = calibrators[-1].rawCalpar[:, :, :]
        np.testing.assert_almost_equal(correctresult[:, :, 1:3][nanmask],
                                       newresult[:, :, 1:3][nanmask],
                                       decimal=5)
        np.testing.assert_almost_equal(np.sum(
            np.abs(data[-1] - calibrators[-1].get_modeled_data())
            [:, :,
             calibrators[-1].Info.subsetbl[calibrators[-1].Info.crossindex]]**
            2,
            axis=2)[nanmask],
                                       newresult[:, :, 2][nanmask],
                                       decimal=5)
        np.testing.assert_almost_equal(np.sum(
            np.abs(additiveout)[:, :, calibrators[-1].Info.crossindex]**2,
            axis=2)[nanmask],
                                       newresult[:, :, 2][nanmask],
                                       decimal=5)
        np.testing.assert_almost_equal(correctresult[:, :, 3:67][nanmask],
                                       newresult[:, :, 3:67][nanmask],
                                       decimal=5)
        np.testing.assert_almost_equal(
            np.sort(np.abs(correctresult[:, :, 67:][nanmask])),
            np.sort(np.abs(newresult[:, :, 67:][nanmask])),
            decimal=5)
Ejemplo n.º 9
0
        calibrators[pol] = omni.RedundantCalibrator_X5(header_antloc)
    calibrators[pol].read_redundantinfo(infopaths[pol], verbose=False)
    info = calibrators[pol].Info.get_info()
    calibrators[pol].nTime = len(timing)
    calibrators[pol].nFrequency = nfreq

    ####consolidate 3D flags from uv files into per time/freq 2D flags
    uvflags[pol] = np.any(
        rawflag[p, :, :,
                calibrators[pol].subsetbl[calibrators[pol].crossindex]],
        axis=0)  # aweird transpose happens when slicing

    ###apply, if needed, raw calibration################
    if needrawcal:
        original_data = np.copy(data[p])
        data[p] = omni.apply_calpar(data[p], crude_calpar[pol],
                                    calibrators[pol].totalVisibilityId)

    ####calibrate################
    calibrators[pol].removeDegeneracy = removedegen
    calibrators[pol].convergePercent = converge_percent
    calibrators[pol].maxIteration = max_iter
    calibrators[pol].stepSize = step_size

    ################first round of calibration  #########################
    print FILENAME + " MSG: starting calibration on %s %s. nTime = %i, nFrequency = %i ..." % (
        dataano, pol, calibrators[pol].nTime, calibrators[pol].nFrequency),
    sys.stdout.flush()
    timer = time.time()
    additivein = np.zeros_like(data[p])

    calibrators[pol].logcal(data[p], additivein, nthread=nthread, verbose=True)