Esempio n. 1
0
def main(options):
    ms = options.ms
    if ms == '':
            logging.error('You have to specify an input MS, use -h for help')
            return
    cols = options.cols
    incol = options.incol
    
    t = pt.table(ms, readonly=False, ack=False)

    for col in cols.split(','):
        if col not in t.colnames():
            logging.info('Adding the output column '+col+' to '+ms+'.')
            if incol == '':
                # prepare col metadata
                cd = t.getcoldesc('DATA')
                coldmi = t.getdminfo('DATA')
                if options.dysco:
                    cd['dataManagerType'] = 'DyscoStMan'
                    cd['dataManagerGroup'] = 'DyscoData'
                    coldmi = {'NAME': col,'SEQNR': 3,'SPEC': {'dataBitCount': 10,'distribution': 'TruncatedGaussian','distributionTruncation': 2.5,'normalization': 'AF','studentTNu': 0.0,'weightBitCount': 12},'TYPE': 'DyscoStMan'}
                # not as performing as standard DATA
                else:
                    coldmi["NAME"] = col
                #    cd['dataManagerType'] = 'StandardStMan'
                #    cd['dataManagerGroup'] = 'SSMVar'
                #    coldmi = {'NAME': col,'SEQNR': 0,'SPEC': {'ActualCacheSize': 2,'BUCKETSIZE': 32768,'IndexLength': 799832,'PERSCACHESIZE': 2},'TYPE': 'StandardStMan'}

                cd['comment'] = 'Added by addcol2ms'
                t.addcols(pt.makecoldesc(col, cd), coldmi)

                # if non dysco is done by default
                if options.dysco:
                    logging.warning('Setting '+col+' = 0')
                    pt.taql("update $t set "+col+"=0")

            else:
                # prepare col metadata
                coldmi = t.getdminfo(incol)
                coldmi['NAME'] = col
                cd = t.getcoldesc(incol)

                cd['comment'] = 'Added by addcol2ms'
                t.addcols(pt.makecoldesc(col, cd), coldmi)

                logging.warning('Setting '+col+' = '+incol)
                pt.taql("update $t set "+col+"="+incol)

        else:
            logging.warning('Column '+col+' already exists.')

    t.close()
Esempio n. 2
0
    def iter_antenna(self, antennas=None):
        """
        Iterator to get all visibilities of each antenna
        For GFLAG, GDATA and GWEIGHTS it returns arrays of Ntimes x Nbl x Nfreqs x Npol
        """
        ms = self.ms # to use the "$" in taql

        for ant_id, ant_name in enumerate(self.get_antennas()):
            if antennas is not None and ant_name not in antennas: continue
            logging.info('Workign on antenna: %s', ant_name)
            yield ant_id, ant_name, taql('select TIME, GAGGR(FLAG) as GFLAG, ANTENNA1, ANTENNA2, GAGGR(%s) as GDATA, GAGGR(%s) as GWEIGHT \
                                          from $ms \
                                          where (ANTENNA1=%i or ANTENNA2=%i) and (ANTENNA1 != ANTENNA2) \
                                          groupby TIME' \
                    % (self.dcolname, self.wcolname, ant_id, ant_id) )
Esempio n. 3
0
def run( soltab, ms, inverse=False, useElementResponse=True, useArrayFactor=True, useChanFreq=True ):
    """
    Generic unspecified step for easy expansion.

    Parameters
    ----------
    opt1 : float
        Is a mandatory parameter.

    """
    # load specific libs
    import numpy as np
    import casacore.tables as pt
    from lofar.stationresponse import stationresponse

    sr = stationresponse(ms, inverse, useElementResponse, useArrayFactor, useChanFreq)

    numants = pt.taql('select gcount(*) as numants from '+ms+'::ANTENNA').getcol('numants')[0]
    times = soltab.getAxisValues('time')

    for vals, coord, selection in soltab.getValuesIter(returnAxes=['ant','time','pol','freq'], weight=False):
        vals = reorderAxes( vals, soltab.getAxesNames(), ['ant','time','freq','pol'] )

        for stationnum in range(numants):
            logging.debug('Working on station number %i' % stationnum)
            for itime, time in enumerate(times):
                beam = sr.evaluateStation(time=time, station=stationnum)
                # Reshape from [nfreq, 2, 2] to [nfreq, 4]
                beam = beam.reshape(beam.shape[0], 4)

                if soltab.getAxisLen('pol') == 2:
                    beam = beam[:,[0,3]] # get only XX and YY

                if soltab.getType() == 'amplitude':
                    vals[stationnum, itime, :, :] = np.abs(beam)
                elif soltab.getType() == 'phase':
                    vals[stationnum, itime, :, :] = np.angle(beam)
                else:
                    logging.error('Beam prediction work only for amplitude/phase solution tables.')
                    return 1

        vals = reorderAxes( vals, ['ant','time','freq','pol'], [ax for ax in soltab.getAxesNames() if ax in ['ant','time','freq','pol']] )
        soltab.setValues(vals, selection)

    return 0
Esempio n. 4
0
 def test_subset(self):
     """Create a subset."""
     c1 = makescacoldesc("coli", 0)
     c2 = makescacoldesc("cold", 0.)
     c3 = makescacoldesc("cols", "")
     c4 = makescacoldesc("colb", True)
     c5 = makescacoldesc("colc", 0. + 0j)
     c6 = makearrcoldesc("colarr", 0.)
     t = table("ttable.py_tmp.tab1", maketabdesc((c1, c2, c3, c4, c5,
                                                  c6)), ack=False)
     t1 = t.query('coli >0', sortlist='coli desc', columns='coli,cold')
     querycols = t1.colnames()
     t1 = taql('select coli,cold from $t where coli>0 order by coli desc')
     taqlcol = t1.colnames()
     self.assertEqual(querycols, taqlcol)
     t1.close()
     t.close()
     tabledelete("ttable.py_tmp.tab1")
Esempio n. 5
0
 def get_antennas(self):
     return taql('select NAME from %s/ANTENNA' % (self.ms_files[0]) ).getcol('NAME')
Esempio n. 6
0
# Get the azimuth and elevation of Jupiter from Dwingeloo at a given time.
# Usage: python azel-taql.py

from casacore.tables import taql

print taql('calc meas.azel("Jupiter",2016-06-28 19:54,"DWL") deg')
Esempio n. 7
0
    sys.exit(0)
msfile = msfile[0]
if not os.path.exists(msfile):
    logging.error("Cannot find MS file {}.".format(msfile))
    sys.exit(1)
# open input/output MS
ms = pt.table(msfile, readonly=False, ack=False)

with pt.table(msfile + '::SPECTRAL_WINDOW', ack=False) as freqtab:
    freq = freqtab.getcol('REF_FREQUENCY')[0]
    freqpersample = np.mean(freqtab.getcol('RESOLUTION'))
    timepersample = ms.getcell('INTERVAL', 0)

# get info on all baselines
with pt.taql(
        "SELECT ANTENNA1,ANTENNA2,sqrt(sumsqr(UVW)),GCOUNT() FROM $ms GROUPBY ANTENNA1,ANTENNA2"
) as BL:
    ants1, ants2 = BL.getcol('ANTENNA1'), BL.getcol('ANTENNA2')
    dists = BL.getcol('Col_3') / 1e3  # baseleline length in km
    n_t = BL.getcol('Col_4')[0]  # number of timesteps
    n_bl = len(ants1)

# check if ms is time-ordered
times = ms.getcol('TIME_CENTROID')
if not all(np.diff(times) >= 0):
    logging.critical('This code cannot handle MS that are not time-sorted.')
    sys.exit(1)
del times

# create column to smooth
addcol(ms, options.incol, options.outcol)
Esempio n. 8
0
def read_main_table(infile, subsel=0, ignore=None, chunks=(400, 200, 100, 2)):
    if ignore is None: ignore = []

    # select just the specified ddi
    tb_tool = tables.taql('select * from %s where DATA_DESC_ID = %i' %
                          (infile, subsel))
    if tb_tool.nrows() == 0:
        tb_tool.close()
        return xarray.Dataset()

    # main table uses time x (antenna1,antenna2)
    ant1, ant2 = tb_tool.getcol('ANTENNA1', 0,
                                -1), tb_tool.getcol('ANTENNA2', 0, -1)
    baselines = np.array([
        str(ll[0]).zfill(3) + '_' + str(ll[1]).zfill(3)
        for ll in np.unique(np.hstack([ant1[:, None], ant2[:, None]]), axis=0)
    ])

    dims, cols = ['time', 'baseline', 'chan', 'pol'], tb_tool.colnames()
    tvars, mcoords, xds = {}, {}, xarray.Dataset()

    cshapes = [
        np.array(tb_tool.getcell(col, 0)).shape for col in cols
        if tb_tool.iscelldefined(col, 0)
    ]
    chan_cnt, pol_cnt = [(cc[0], cc[1]) for cc in cshapes if len(cc) == 2][0]

    ts_tb = tables.taql(
        'select DISTINCT TIME from %s where DATA_DESC_ID = %i' %
        (infile, subsel))
    utimes = np.unique(ts_tb.getcol('TIME', 0, -1))
    tol = np.diff(
        utimes).min() / 4  # add a tol around the time ranges returned by taql
    ts_tb.close()

    # loop over time chunks
    for tc in range(0, len(utimes), chunks[0]):
        times = (utimes[tc] - tol,
                 utimes[min(len(utimes) - 1, tc + chunks[0] - 1)] + tol)
        ctlen = min(len(utimes), tc + chunks[0]) - tc  # chunk time length
        bvars = {}

        # loop over baseline chunks
        for bc in range(0, len(baselines), chunks[1]):
            blines = (baselines[bc],
                      baselines[min(len(baselines) - 1, bc + chunks[1] - 1)])
            cblen = min(len(baselines) - bc, chunks[1])

            # read the specified chunk of data
            #def read_chunk(infile, subsel, times, blines, chans, pols):
            dtql = 'DATA_DESC_ID = %i' % subsel
            ttql = 'TIME BETWEEN %f and %f' % times
            atql = 'ANTENNA1 BETWEEN %i and %i' % (int(
                blines[0].split('_')[0]), int(blines[1].split('_')[0]))
            ts_taql = 'select * from %s where %s AND %s AND %s' % (
                infile, dtql, ttql, atql)
            ts_tb = tables.taql(ts_taql)

            tidxs = np.searchsorted(utimes, ts_tb.getcol('TIME', 0, -1)) - tc
            ts_ant1, ts_ant2 = ts_tb.getcol('ANTENNA1', 0, -1), ts_tb.getcol(
                'ANTENNA2', 0, -1)
            ts_bases = [
                str(ll[0]).zfill(3) + '_' + str(ll[1]).zfill(3)
                for ll in np.hstack([ts_ant1[:, None], ts_ant2[:, None]])
            ]
            bidxs = np.searchsorted(baselines, ts_bases) - bc

            # some antenna 2's will be out of bounds for this chunk, store rows that are in bounds
            didxs = np.where((bidxs >= 0)
                             & (bidxs < min(chunks[1],
                                            len(baselines) - bc)))[0]

            # loop over each column and create delayed dask arrays
            for col in cols:
                if (col in ignore +
                    ['TIME']) or (not tb_tool.iscelldefined(col, 0)):
                    continue
                if col not in bvars: bvars[col] = []

                cdata = tb_tool.getcol(col, 0, 1)[0]
                if len(cdata.shape) == 0:
                    delayed_array = dask.delayed(read_col_chunk)(
                        ts_taql, col, (ctlen, cblen), tidxs, bidxs, didxs,
                        None, None)
                    bvars[col] += [
                        dask.array.from_delayed(delayed_array, (ctlen, cblen),
                                                cdata.dtype)
                    ]

                elif col == 'UVW':
                    delayed_array = dask.delayed(read_col_chunk)(
                        ts_taql, col, (ctlen, cblen, 3), tidxs, bidxs, didxs,
                        None, None)
                    bvars[col] += [
                        dask.array.from_delayed(delayed_array,
                                                (ctlen, cblen, 3), cdata.dtype)
                    ]

                elif len(cdata.shape) == 1:
                    pol_list = []
                    dd = 2 if cdata.shape == chan_cnt else 3
                    for pc in range(0, cdata.shape[0], chunks[dd]):
                        pols = (pc, min(cdata.shape[0], pc + chunks[dd]) - 1)
                        cshape = (
                            ctlen,
                            cblen,
                        ) + (pols[1] - pols[0] + 1, )
                        delayed_array = dask.delayed(read_col_chunk)(
                            ts_taql, col, cshape, tidxs, bidxs, didxs, pols,
                            None)
                        pol_list += [
                            dask.array.from_delayed(delayed_array, cshape,
                                                    cdata.dtype)
                        ]
                    bvars[col] += [dask.array.concatenate(pol_list, axis=2)]

                elif len(cdata.shape) == 2:
                    chan_list = []
                    for cc in range(0, cdata.shape[0], chunks[2]):
                        chans = (cc, min(cdata.shape[0], cc + chunks[2]) - 1)
                        pol_list = []
                        for pc in range(0, cdata.shape[1], chunks[3]):
                            pols = (pc,
                                    min(cdata.shape[1], pc + chunks[3]) - 1)
                            cshape = (
                                ctlen,
                                cblen,
                            ) + (chans[1] - chans[0] + 1,
                                 pols[1] - pols[0] + 1)
                            delayed_array = dask.delayed(read_col_chunk)(
                                ts_taql, col, cshape, tidxs, bidxs, didxs,
                                chans, pols)
                            pol_list += [
                                dask.array.from_delayed(
                                    delayed_array, cshape, cdata.dtype)
                            ]
                        chan_list += [dask.array.concatenate(pol_list, axis=3)]
                    bvars[col] += [dask.array.concatenate(chan_list, axis=2)]
            ts_tb.close()

        # now concat all the dask chunks from each baseline
        for kk in bvars.keys():
            if len(bvars[kk]) == 0: continue
            if kk not in tvars: tvars[kk] = []
            tvars[kk] += [dask.array.concatenate(bvars[kk], axis=1)]

    # now concat all the dask chunks from each time to make the xds
    mvars = {}
    for kk in tvars.keys():
        if kk == 'UVW':
            mvars[kk.upper()] = xarray.DataArray(dask.array.concatenate(
                tvars[kk], axis=0),
                                                 dims=dims[:2] + ['uvw_index'])
        elif len(tvars[kk][0].shape) == 3 and (tvars[kk][0].shape[-1]
                                               == pol_cnt):
            mvars[kk.upper()] = xarray.DataArray(dask.array.concatenate(
                tvars[kk], axis=0),
                                                 dims=dims[:2] + ['pol'])
        elif len(tvars[kk][0].shape) == 3 and (tvars[kk][0].shape[-1]
                                               == chan_cnt):
            mvars[kk.upper()] = xarray.DataArray(dask.array.concatenate(
                tvars[kk], axis=0),
                                                 dims=dims[:2] + ['chan'])
        else:
            mvars[kk.upper()] = xarray.DataArray(
                dask.array.concatenate(tvars[kk], axis=0),
                dims=dims[:len(tvars[kk][0].shape)])

    mcoords['time'] = xarray.DataArray(convert_time(utimes), dims=['time'])
    mcoords['baseline'] = xarray.DataArray(np.arange(len(baselines)),
                                           dims=['baseline'])
    xds = xarray.Dataset(mvars, coords=mcoords)
    tb_tool.close()

    return xds
Esempio n. 9
0
#!/usr/bin/python
# usage: plot_weights.py xxx.MS

import os, sys
#sys.path = ['/home/dijkema/opt/python-casacore/lib/python2.7/site-packages/']
import numpy as np
import matplotlib.cm as cm
import matplotlib.pyplot as plt
from casacore.tables import taql

ms = sys.argv[1]

print "selecting on chan"
t=taql('select ANTENNA, gmeans(DATA[FLAG]) as DATA, gmeans(WEIGHT_SPECTRUM[FLAG]) as WEIGHT from [[ SELECT ANTENNA1 AS ANTENNA, DATA, WEIGHT_SPECTRUM, FLAG from '+ms+'], [SELECT ANTENNA2 AS ANTENNA, DATA, WEIGHT_SPECTRUM, FLAG from '+ms+']] group by ANTENNA')
#weights = np.average(np.average(t.getcol('WEIGHT'),axis=0),axis=1)
weights = t.getcol('WEIGHT')[:,:,0]
data = np.abs(t.getcol('DATA')[:,:,0])

fig, ax1 = plt.subplots()
colors = iter(cm.rainbow(np.linspace(0, 1, len(weights))))
#ax1.set_yscale('log')
for w, c in zip(weights, colors):
    ax1.scatter(xrange(len(w)),w,color=c,marker='.')
ax1.set_xlim(xmin=0,xmax=len(w))
ax1.set_ylim(ymin=np.min(weights),ymax=np.max(weights))
ax1.set_xlabel('Channel')
ax1.set_ylabel('Weights')
plt.legend()
plt.savefig('weightVSchan.png', bbox_inches='tight')

fig, ax1 = plt.subplots()
Esempio n. 10
0
def reweight(MSh, mode):

    # get data per antenna
    var_antenna = {}
    med_antenna = {}
    for ant_id, ant_name, ms_ant in MSh.iter_antenna():

        with Timer('Get data'):
            data = ms_ant.getcol('GDATA') # axes: time, ant, freq, pol
            flags = ms_ant.getcol('GFLAG') # axes: time, ant, freq, pol

            # put flagged data to NaNs
            data[flags] = np.nan

            # if completely flagged set variance to 1 and continue
            if np.all(flags):
                var_antenna[ant_id] = None
                med_antenna[ant_id] = None
                continue

        with Timer('Prepare data'):

            # data column is updated subtracting adjacent channels
            if mode == 'subchan':
                data_shifted_l = np.roll(data, -1, axis=2)
                data_shifted_r = np.roll(data, +1, axis=2)
                # if only 2 freq it's aleady ok, subtracting one from the other
                if data.shape[2] > 2:
                    data_shifted_l[:,:,-1,:] = data_shifted_l[:,:,-3,:] # last chan uses the one but last
                    data_shifted_r[:,:,0,:] = data_shifted_r[:,:,2,:] # first chan uses third
                # get the "best" shift, either on the right or left. This is to avoid propagating bad channels (e.g. with RFI)
                ratio_l = np.nanvar(data_shifted_l, axis=(0,1,3))/np.nanmean(data_shifted_l, axis=(0,1,3))
                ratio_l[ np.isnan(ratio_l) ] = np.inf
                ratio_r = np.nanvar(data_shifted_r, axis=(0,1,3))/np.nanmean(data_shifted_r, axis=(0,1,3))
                ratio_r[ np.isnan(ratio_r) ] = np.inf
                data = np.where( ( ratio_l < ratio_r )[np.newaxis,np.newaxis,:,np.newaxis], data - data_shifted_l, data - data_shifted_r)

            # data column is updated subtracting adjacent times
            if mode == 'subtime':
                data_shifted_l = np.roll(data, -1, axis=0)
                data_shifted_r = np.roll(data, +1, axis=0)
                # if only 2 freq it's aleady ok, subtracting one from the other
                if data.shape[0] > 2:
                    data_shifted_l[-1,:,:,:] = data_shifted_l[-3,:,:,:] # last timeslot uses the one but last
                    data_shifted_r[0,:,:,:] = data_shifted_r[2,:,:,:] # first timeslot uses third
                # get the "best" shift, either on the right or left. This is to avoid propagating bad channels (e.g. with RFI)
                ratio_l = np.nanvar(data_shifted_l, axis=(1,2,3))/np.nanmean(data_shifted_l, axis=(1,2,3))
                ratio_l[ np.isnan(ratio_l) ] = np.inf
                ratio_r = np.nanvar(data_shifted_r, axis=(1,2,3))/np.nanmean(data_shifted_r, axis=(1,2,3))
                ratio_r[ np.isnan(ratio_r) ] = np.inf
                data = np.where( ( ratio_l < ratio_r )[:,np.newaxis,np.newaxis,np.newaxis], data - data_shifted_l, data - data_shifted_r)

            # use residual data, nothing to do here
            elif mode == 'residual':
                pass

        with Timer('Calc variances'):
            # find mean/variance per time/freq for each antenna

            med_freqs = np.abs( np.nanmean( data, axis=(1,2) )**2 ) # time x pol
            med_times = np.abs( np.nanmean( data, axis=(0,1) )**2 ) # freq x pol
            med_antenna[ant_id] = med_freqs[:, np.newaxis]+med_times # sum of the time/freq mean - axes: time,freq,pol

            var_freqs = np.nanvar( data, axis=(1,2) ) # time x pol
            var_times = np.nanvar( data, axis=(0,1) ) # freq x pol
            var_antenna[ant_id] = var_freqs[:, np.newaxis]+var_times # sum of the time/freq variances - axes: time,freq,pol

    # reconstruct BL weights from antenna variance
    for ms_bl in MSh.ms.iter(["ANTENNA1","ANTENNA2"]):
        ant_id1 = ms_bl.getcol('ANTENNA1')[0]
        ant_id2 = ms_bl.getcol('ANTENNA2')[0]

        if var_antenna[ant_id1] is None or var_antenna[ant_id2] is None: continue

#        print '### BL: %i - %i' % (ant_id1, ant_id2)
#        print var_antenna[ant_id1]*med_antenna[ant_id2]
#        print ''
#        print var_antenna[ant_id2]*med_antenna[ant_id1]
#        print ''
#        print var_antenna[ant_id1]*var_antenna[ant_id2]
        w = 1./( var_antenna[ant_id1]*med_antenna[ant_id2] + var_antenna[ant_id2]*med_antenna[ant_id1] \
               + var_antenna[ant_id1]*var_antenna[ant_id2] )

        w -= np.nanmedian(w) # TEST: REMOVE MEDIAN?

        f = ms_bl.getcol('FLAG')
        # find how many unflagged weights are nans
        ntoflag = np.count_nonzero(np.isnan(w[~f]))
        logging.debug( 'BL: %i - %i: created %i new flags (%f%%)' % ( ant_id1, ant_id2, ntoflag, (100.*ntoflag)/np.size(w) ) )
        ms_bl.putcol(MSh.wcolname, w)
        ms_bl.flush()
        # flag weights that are nans
        taql('update $ms_bl set FLAG[isnan(WEIGHT_SPECTRUM)]=True, WEIGHT_SPECTRUM[isnan(WEIGHT_SPECTRUM)]=0')
        ms_bl.flush()
Esempio n. 11
0
 def get_elev(self):
     # TODO: fix if multiple time MSs passed
     ms_avgbl = taql('SELECT TIME, MEANS(GAGGR(MSCAL.AZEL1()[1]), 0) AS ELEV FROM %s GROUPBY TIME' %(self.ms_files[0]))
     return ms_avgbl.getcol('ELEV')
Esempio n. 12
0
            st = ss.getSoltab('phase000')
        ax_freq_temp = st.getAxisValues('freq')
        if len(ax_freq_temp) > len_freq_old:
            # Longer freq axis meas a shorter solution interval was used.
            ax_freq = ax_freq_temp
            len_freq_old = len(ax_freq)
            name_freq = ih5
        fh5.close()
    print('Fastest frequency axis taken from {:s} with a solution interval of {:f} Hz.'.format(name_freq, ax_freq[1]-ax_freq[0]))
    if 'pol' in axes_new:
        phases = np.zeros((len(polarizations), 1, len(antennas), len(ax_freq), len(ax_time)))
    else:
        phases = np.zeros((1, len(antennas), len(ax_freq), len(ax_time)))
elif convert_tec and 'tec' in args.soltab2merge:
    print('Determining frequency grid...')
    ff = ct.taql('SELECT CHAN_FREQ, CHAN_WIDTH FROM ' + ms_first + '::SPECTRAL_WINDOW')
    freq_first = ff.getcol('CHAN_FREQ')[0][0]
    freq_spacing = ff.getcol('CHAN_WIDTH')[0][0]
    ff.close()

    fl = ct.taql('SELECT CHAN_FREQ, CHAN_WIDTH FROM ' + ms_last + '::SPECTRAL_WINDOW')
    freq_last = fl.getcol('CHAN_FREQ')[0][0]
    print(freq_first, freq_last, freq_spacing)
    ax_freq = np.arange(freq_first, freq_last + freq_spacing, freq_spacing)
    phases = np.zeros((1, 1, len(antennas), len(ax_freq), len(ax_time)))
    print('Frequency axis taken Measurement Sets with a solution interval of {:f} Hz.'.format(ax_freq[1]-ax_freq[0]))
elif not convert_tec:
    phases = np.zeros(vals_reordered.shape)

h5out = h5parm.h5parm(args.h5out, readonly=False)
if args.append_to_solset:
Esempio n. 13
0
#############################
# 3. Set-up directory structure ans split

MSs = lib_ms.AllMSs(glob.glob(data_dir + '/*MS'), s)
numberOfMSs = len(MSs.getListObj())

if not os.path.exists('cals'): os.mkdir('cals')
if not os.path.exists('tgts'): os.mkdir('tgts')

for msID, MS in enumerate(MSs.getListObj()):
    pathMS = MS.pathMS

    logger.info("- Split-up of original MS '%s' (msID: %02i) -" %
                (pathMS, msID))

    scanIDs = tables.taql("select distinct SCAN_NUMBER from $pathMS").getcol(
        "SCAN_NUMBER")
    numberOfScans = len(scanIDs)

    # Create temporary paths for the sub-MSs.
    for i, scanID in enumerate(scanIDs):
        pathMSscan = "msID%02i_scanID%03i.MS" % (msID, scanID)
        if os.path.exists(pathMSscan): continue
        logger.info("MS %02i/%02i: %03i/%03i - Creating MS: %s" %
                    (msID + 1, numberOfMSs, i + 1, numberOfScans, pathMSscan))
        tables.taql(
            "SELECT from $pathMS where SCAN_NUMBER = $scanID giving $pathMSscan as plain"
        )

#############################
# 4. Fix MS
MSs = lib_ms.AllMSs(glob.glob('msID*_scanID*MS'), s)
Esempio n. 14
0
    def scan_ms(self):
        """
        Scans input MS and stores info
        """
        # Get time info
        tab = pt.table(self.ms_filename, ack=False)
        if self.starttime is None:
            self.starttime = np.min(tab.getcol('TIME'))
        else:
            valid_times = np.where(tab.getcol('TIME') >= self.starttime)[0]
            if len(valid_times) == 0:
                self.log.critical('Start time of {0} is greater than the last time in the MS! '
                                  'Exiting!'.format(self.starttime))
                sys.exit(1)
            self.starttime = tab.getcol('TIME')[valid_times[0]]

        # DPPP takes ceil(startTimeParset - startTimeMS), so ensure that our start time is
        # slightly less than the true one (if it's slightly larger, DPPP sets the first
        # time to the next time, skipping the first time slot)
        self.starttime -= 0.1
        if self.starttime > np.min(tab.getcol('TIME')):
            self.startsat_startofms = False
        else:
            self.startsat_startofms = True
        if self.endtime is None:
            self.endtime = np.max(tab.getcol('TIME'))
        else:
            valid_times = np.where(tab.getcol('TIME') <= self.endtime)[0]
            if len(valid_times) == 0:
                self.log.critical('End time of {0} is less than the first time in the MS! '
                                  'Exiting!'.format(self.endtime))
                sys.exit(1)
            self.endtime = tab.getcol('TIME')[valid_times[-1]]
        if self.endtime < np.max(tab.getcol('TIME')):
            self.goesto_endofms = False
        else:
            self.goesto_endofms = True
        self.timepersample = tab.getcell('EXPOSURE', 0)
        self.numsamples = int(np.ceil((self.endtime - self.starttime) / self.timepersample))
        tab.close()

        # Get frequency info
        sw = pt.table(self.ms_filename+'::SPECTRAL_WINDOW', ack=False)
        self.referencefreq = sw.col('REF_FREQUENCY')[0]
        self.startfreq = np.min(sw.col('CHAN_FREQ')[0])
        self.endfreq = np.max(sw.col('CHAN_FREQ')[0])
        self.numchannels = sw.col('NUM_CHAN')[0]
        self.channelwidth = sw.col('CHAN_WIDTH')[0][0]
        sw.close()

        # Get pointing info
        obs = pt.table(self.ms_filename+'::FIELD', ack=False)
        self.ra = np.degrees(float(obs.col('REFERENCE_DIR')[0][0][0]))
        if self.ra < 0.:
            self.ra = 360.0 + (self.ra)
        self.dec = np.degrees(float(obs.col('REFERENCE_DIR')[0][0][1]))
        obs.close()

        # Get station names and diameter
        ant = pt.table(self.ms_filename+'::ANTENNA', ack=False)
        self.stations = ant.col('NAME')[:]
        self.diam = float(ant.col('DISH_DIAMETER')[0])
        if 'HBA' in self.stations[0]:
            self.antenna = 'HBA'
        elif 'LBA' in self.stations[0]:
            self.antenna = 'LBA'
        else:
            self.log.warning('Antenna type not recognized (only LBA and HBA data '
                             'are supported at this time)')
        ant.close()

        # Find mean elevation and FOV
        el_values = pt.taql("SELECT mscal.azel1()[1] AS el from "
                            + self.ms_filename + " limit ::10000").getcol("el")
        self.mean_el_rad = np.mean(el_values)
Esempio n. 15
0
        parameter_set['RA_DEC_Flux_idx'] = [1, 3, 7]
    else:
        RuntimeError('Unknown catalog: {}'.format(catalog))

    data_root_path = os.environ['DATA_ROOT_PATH']
    basefile_name = 'RX42_SB100-109.2ch10s'
    # basefile_name = 'BOOTES24_SB180-189.2ch8s_SIM'
    catalog_basefile_name = parameter_set['catalog_basefile_name']

    ms_file_name = data_root_path + basefile_name + '.ms'
    catalog_file = data_root_path + catalog_basefile_name + file_ext
    npz_catalog_name = data_root_path + catalog_basefile_name + '.npz'

    # pointing direction in radian
    pointingDirection = casa_tables.taql(
        'select REFERENCE_DIR from {msFile}::FIELD'.format(msFile=ms_file_name)
    ).getcol('REFERENCE_DIR').squeeze()
    M = J2000_to_UVW_operator(*pointingDirection)

    # index in the fits table where RA, DEC and Flux are
    RA_DEC_Flux_idx = parameter_set['RA_DEC_Flux_idx']

    with fits.open(catalog_file) as handle:
        # print out a few information about the HDU file and the data column description
        handle.info()
        data_description = handle[1].header
        data_description
        # the actual data
        data = handle[1].data
        # extract data <- verify with the data_description that the correct data is extracted
        # RA and DEC are in J2000 coordinate (in degrees)
    ax = fig.add_subplot(111)
    im = ax.imshow(screen[..., -1], origin='lower', extent=[ra_max, ra_min, dec_min, dec_max])
    ax.scatter(RA, DEC, marker='x', color='r')
    ax.set_xlabel('Right ascension'); ax.set_ylabel('Declination')
    fig.colorbar(im)
    fig.savefig(antname + '_{:03d}.png'.format(itstep))
    del fig
    show()
    '''
    return names.index(antname), screen


ms = sys.argv[1]
h5p = sys.argv[2]

t = ct.taql('SELECT NAME FROM {ms:s}::ANTENNA'.format(ms=ms))
names = t.getcol('NAME')
t.close()
t = ct.taql('SELECT REFERENCE_DIR FROM {ms:s}::FIELD'.format(ms=ms))
phasecenter = t.getcol('REFERENCE_DIR').squeeze()
print(phasecenter)
# Avoid negative CRVAL1
if np.rad2deg(phasecenter[0]) < 0:
    phasecenter[0] = phasecenter[0] + (2. * np.pi)

t.close()
h5 = h5parm.h5parm(h5p)
ss = h5.getSolset('sol000')
st = ss.getSoltab('amplitude000')
time = st.getAxisValues('time')
stime = time[0]
Esempio n. 17
0
 def get_freqs(self):
     # TODO: check if there is a smarter way to do it with concat.MS
     freqs = []
     for ms_file in self.ms_files:
         freqs += list( taql('SELECT CHAN_FREQ FROM %s/SPECTRAL_WINDOW' % ms_file)[0]['CHAN_FREQ'] * 1e-6 ) # in MHz
     return freqs
Esempio n. 18
0
    def test_msutil(self):
        """Testing msutil."""
        datacoldesc = makearrcoldesc("DATA", 0., ndim=2, shape=[20, 4])
        ms = default_ms("tabtemp", maketabdesc((datacoldesc)))
        ms.close()

        spw = table("tabtemp/SPECTRAL_WINDOW", readonly=False)
        spw.addrows()
        spw.putcell('NUM_CHAN', 0, 20)
        t = table("tabtemp", readonly=False)
        print(t.colnames())
        addImagingColumns("tabtemp")

        self.assertIn('MODEL_DATA', t.colnames())
        self.assertIn('CORRECTED_DATA', t.colnames())
        self.assertIn('IMAGING_WEIGHT', t.colnames())

        removeImagingColumns("tabtemp")

        self.assertNotIn('MODEL_DATA', t.colnames())
        self.assertNotIn('CORRECTED_DATA', t.colnames())
        self.assertNotIn('IMAGING_WEIGHT', t.colnames())

        addDerivedMSCal("tabtemp")

        self.assertIn('PA1', t.colnames())
        self.assertIn('PA2', t.colnames())
        self.assertIn('LAST', t.colnames())
        self.assertIn('AZEL2', t.colnames())
        self.assertIn('AZEL1', t.colnames())
        self.assertIn('UVW_J2000', t.colnames())
        self.assertIn('LAST1', t.colnames())
        self.assertIn('LAST2', t.colnames())
        self.assertIn('HA1', t.colnames())
        self.assertIn('HA2', t.colnames())
        self.assertIn('HA', t.colnames())

        removeDerivedMSCal("tabtemp")

        self.assertNotIn('PA1', t.colnames())
        self.assertNotIn('PA2', t.colnames())
        self.assertNotIn('LAST', t.colnames())
        self.assertNotIn('AZEL2', t.colnames())
        self.assertNotIn('AZEL1', t.colnames())
        self.assertNotIn('UVW_J2000', t.colnames())
        self.assertNotIn('LAST1', t.colnames())
        self.assertNotIn('LAST2', t.colnames())
        self.assertNotIn('HA1', t.colnames())
        self.assertNotIn('HA2', t.colnames())
        self.assertNotIn('HA', t.colnames())
        self.assertNotIn('HA', t.colnames())
        self.assertNotIn('HA', t.colnames())

        taql("SELECT FROM tabtemp where TIME in (SELECT DISTINCT TIME" +
             " FROM tabtemp LIMIT 10) GIVING first10.MS AS PLAIN")
        taql("SELECT FROM tabtemp where TIME in (SELECT DISTINCT TIME" +
             " FROM tabtemp LIMIT 10 OFFSET 10) GIVING second10.MS AS PLAIN")
        msconcat(["first10.MS", "second10.MS"], "combined.MS", concatTime=True)
        spw.close()
        t.close()
        tabledelete("tabtemp")
Esempio n. 19
0
 def get_time(self):
     # TODO: fix if multiple time MSs passed
     ms_avgbl = taql('SELECT TIME FROM %s GROUPBY TIME' %(self.ms_files[0]))
     return ms_avgbl.getcol('TIME')
Esempio n. 20
0
 def get_flags_aggr(self):
     """
     """
     ms = self.ms_file
     return taql('select NTRUES(GAGGR(FLAG), [0, 2]) as FLAG, SHAPE(GAGGR(FLAG)) as N from $ms where ANTENNA1 != ANTENNA2 groupby TIME')
Esempio n. 21
0
def read_pointing_table(infile, chunks=(100, 100, 20, 20)):
    tb_tool = tables.taql('select * from %s' % infile)
    if tb_tool.nrows() == 0:
        tb_tool.close()
        return xarray.Dataset()

    # pointing table uses time x antenna_id
    baselines = np.unique(tb_tool.getcol('ANTENNA_ID', 0, -1))

    dims, cols = ['time', 'antenna_id', 'd2', 'd3'], tb_tool.colnames()
    tvars, mcoords, xds = {}, {}, xarray.Dataset()

    ts_tb = tables.taql('select DISTINCT TIME from %s' % infile)
    utimes = np.unique(ts_tb.getcol('TIME', 0, -1))
    tol = np.diff(
        utimes).min() / 4  # add a tol around the time ranges returned by taql
    ts_tb.close()

    # loop over time chunks
    for tc in range(0, len(utimes), chunks[0]):
        times = (utimes[tc] - tol,
                 utimes[min(len(utimes) - 1, tc + chunks[0] - 1)] + tol)
        ctlen = min(len(utimes), tc + chunks[0]) - tc  # chunk time length
        bvars = {}

        # loop over antenna_id chunks
        for bc in range(0, len(baselines), chunks[1]):
            blines = (baselines[bc],
                      baselines[min(len(baselines) - 1, bc + chunks[1] - 1)])
            cblen = min(len(baselines) - bc, chunks[1])

            # read the specified chunk of data
            ttql = 'TIME BETWEEN %f and %f' % times
            atql = 'ANTENNA_ID BETWEEN %i and %i' % blines
            ts_taql = 'select * from %s where %s AND %s' % (infile, ttql, atql)
            ts_tb = tables.taql(ts_taql)

            tidxs = np.searchsorted(utimes, ts_tb.getcol('TIME', 0, -1)) - tc
            bidxs = np.searchsorted(baselines, ts_tb.getcol(
                'ANTENNA_ID', 0, -1)) - bc
            didxs = np.arange(len(bidxs))

            # loop over each column and create delayed dask arrays
            for col in cols:
                if (col in ['TIME', 'ANTENNA_ID'
                            ]) or (not tb_tool.iscelldefined(col, 0)):
                    continue
                if col not in bvars: bvars[col] = []

                cdata = tb_tool.getcol(col, 0, 1)[0]
                if isinstance(cdata, str): cdata = np.array(cdata)
                if len(cdata.shape) == 0:
                    delayed_array = dask.delayed(read_col_chunk)(
                        ts_taql, col, (ctlen, cblen), tidxs, bidxs, didxs,
                        None, None)
                    bvars[col] += [
                        dask.array.from_delayed(delayed_array, (ctlen, cblen),
                                                cdata.dtype)
                    ]

                elif len(cdata.shape) == 2:
                    d1_list = []
                    for cc in range(0, cdata.shape[0], chunks[2]):
                        d1s = (cc, min(cdata.shape[0], cc + chunks[2]) - 1)
                        d2_list = []
                        for pc in range(0, cdata.shape[1], chunks[3]):
                            d2s = (pc, min(cdata.shape[1], pc + chunks[3]) - 1)
                            cshape = (
                                ctlen,
                                cblen,
                            ) + (d1s[1] - d1s[0] + 1, d2s[1] - d2s[0] + 1)
                            delayed_array = dask.delayed(read_col_chunk)(
                                ts_taql, col, cshape, tidxs, bidxs, didxs, d1s,
                                d2s)
                            d2_list += [
                                dask.array.from_delayed(
                                    delayed_array, cshape, cdata.dtype)
                            ]
                        d1_list += [dask.array.concatenate(d2_list, axis=3)]
                    bvars[col] += [dask.array.concatenate(d1_list, axis=2)]
            ts_tb.close()

        # now concat all the dask chunks from each baseline
        for kk in bvars.keys():
            if len(bvars[kk]) == 0: continue
            if kk not in tvars: tvars[kk] = []
            tvars[kk] += [dask.array.concatenate(bvars[kk], axis=1)]

    # now concat all the dask chunks from each time to make the xds
    mvars = {}
    for kk in tvars.keys():
        mvars[kk.upper()] = xarray.DataArray(
            dask.array.concatenate(tvars[kk], axis=0),
            dims=dims[:len(tvars[kk][0].shape)])

    mcoords['time'] = xarray.DataArray(convert_time(utimes), dims=['time'])
    mcoords['antenna_id'] = xarray.DataArray(np.arange(len(baselines)),
                                             dims=['antenna_id'])
    xds = xarray.Dataset(mvars, coords=mcoords)
    tb_tool.close()

    return xds
Esempio n. 22
0
    def do_execute(self, code, silent, store_history=True, user_expressions=None,
                   allow_stdin=False):
        ashtml=False
        if not silent:
            output=""
            try:
                code=re.sub(ur"([0-9\s\.\)\"])\xb5([A-Za-z])",ur"\1u\2",code) # Tolerate µ as prefix for a unit, substitute it with u
                code=re.sub(ur"\u2245",ur"~=",code) # Tolerate ≅ for approximately equal
                code=str(code) # Code seems to be unicode, convert to string here

                # match the first operation keyword, so that only "select * from (update ..." will yield rows
                m=re.match(".*?((?:select)|(?:update)|(?:insert)|(?:delete)|(?:count)|(?:calc)|(?:create table)|(?:insert)|(?:alter table)|(?:show)|(?:help))",code.lower())
                if m:
                    operation=m.group(1)
                else:
                    operation="select"
                    code="select "+code

                t=pt.taql(code)

                # Don't display output if code is 'SELECT FROM'
                printrows=False
                if operation=="select":
                    # first select has something between "select" and "from"
                    match = re.match('^.*?select(.*?)from',code, re.IGNORECASE)
                    if not(match and match.group(1).isspace()):
                        printrows=True

                printcount=True
                # Don't display row count in simple calc-like expressions
                if operation=="show" or operation=="help" or operation=="calc" or (operation=="select" and not('from' in code.lower())):
                    printcount=False

                if printcount:
                    ashtml=True

                output=self.format_output(t,printrows,printcount,operation,ashtml)

            except UnicodeEncodeError as e:
                output+="Error: unicode is not supported"
            except RuntimeError as e:
                myerror=str(e).split('\n')
                output=""
                m = re.search('parse error at or near position (\d+) ', myerror[1])
                if (m):
                    ashtml=True
                    pos=int(m.group(1))+22
                    output+='<pre>'+myerror[0][0:pos]
                    output+='<span style="background:red;color:white">'+(myerror[0]+' ')[pos]+'</span>'
                    output+=myerror[0][pos+1:]+'\n'
                    output+="\n".join(myerror[1:])
                    #output+=myerror[1]+":"+" "*(21+pos-len(myerror[1]))+"^\n"
                    output+='</pre>'
                else:
                    output+=myerror[0]+"\n"
                    output+="\n".join(myerror[1:])

            if ashtml:
                stream_content={'source': 'TaQL kernel', 'data': {'text/html':output}, 'metadata': {}}
                self.send_response(self.iopub_socket, 'display_data', stream_content)
            else:
                stream_content = {'name': 'stdout', 'text': output}
                self.send_response(self.iopub_socket, 'stream', stream_content)

        return {'status': 'ok',
                # The base class increments the execution count
                'execution_count': self.execution_count,
                'payload': [],
                'user_expressions': {},
               }
Esempio n. 23
0
def describe_ms(infile):
    """
    Summarize the contents of an MS directory in casacore table format

    Parameters
    ----------
    infile : str
        input filename of MS

    Returns
    -------
    pandas.core.frame.DataFrame
        Summary information
    """
    import os
    import pandas as pd
    import numpy as np
    import cngi._utils._table_conversion2 as tblconv
    from casacore import tables

    infile = os.path.expanduser(infile)  # does nothing if $HOME is unknown
    if not infile.endswith('/'): infile = infile + '/'

    # as part of MSv3 conversion, these columns in the main table are no longer needed
    ignorecols = [
        'FLAG_CATEGORY', 'FLAG_ROW', 'SIGMA', 'WEIGHT_SPECTRUM', 'DATA_DESC_ID'
    ]

    # figure out characteristics of main table from select subtables (must all be present)
    spw_xds = tblconv.read_simple_table(infile,
                                        subtable='SPECTRAL_WINDOW',
                                        ignore=ignorecols,
                                        add_row_id=True)
    pol_xds = tblconv.read_simple_table(infile,
                                        subtable='POLARIZATION',
                                        ignore=ignorecols)
    ddi_xds = tblconv.read_simple_table(infile,
                                        subtable='DATA_DESCRIPTION',
                                        ignore=ignorecols)
    ddis = list(ddi_xds['d0'].values)

    summary = pd.DataFrame([])
    spw_ids = ddi_xds.spectral_window_id.values
    pol_ids = ddi_xds.polarization_id.values
    chans = spw_xds.NUM_CHAN.values
    pols = pol_xds.NUM_CORR.values

    for ddi in ddis:
        print('processing ddi %i of %i' % (ddi + 1, len(ddis)), end='\r')
        sorted_table = tables.taql('select * from %s where DATA_DESC_ID = %i' %
                                   (infile, ddi))
        sdf = {
            'ddi':
            ddi,
            'spw_id':
            spw_ids[ddi],
            'pol_id':
            pol_ids[ddi],
            'rows':
            sorted_table.nrows(),
            'times':
            len(np.unique(sorted_table.getcol('TIME'))),
            'baselines':
            len(
                np.unique(np.hstack([
                    sorted_table.getcol(rr)[:, None]
                    for rr in ['ANTENNA1', 'ANTENNA2']
                ]),
                          axis=0)),
            'chans':
            chans[spw_ids[ddi]],
            'pols':
            pols[pol_ids[ddi]]
        }
        sdf['size_MB'] = np.ceil(
            (sdf['times'] * sdf['baselines'] * sdf['chans'] * sdf['pols'] * 9)
            / 1024**2).astype(int)
        summary = pd.concat(
            [summary, pd.DataFrame(sdf, index=[str(ddi)])], axis=0, sort=False)
        sorted_table.close()
    print(' ' * 50, end='\r')

    return summary.set_index('ddi').sort_index()
Esempio n. 24
0
def convert_expanded_table(infile,
                           outfile,
                           subsel=None,
                           ignore=None,
                           compressor=None,
                           chunks=(100, 100, 20, 2),
                           nofile=False):
    if ignore is None: ignore = []
    if compressor is None:
        compressor = Blosc(cname='zstd', clevel=2, shuffle=0)

    # select just the specified ddi
    subseltql = '' if subsel is None else ' where DATA_DESC_ID = %s' % str(
        subsel)
    tb_tool = tables.taql('select * from %s%s' % (infile, subseltql))

    # handle no chunking of first axis
    chunks = [
        chunks[di] if di < len(chunks) else chunks[-1] for di in range(4)
    ]
    if nofile or (chunks[0] == -1):
        chunks[0] = tb_tool.nrows()

    # pointing table will use dims of time by antenna_id
    # main table uses time x (antenna1,antenna2)
    if infile.endswith('POINTING'):
        baselines = np.unique(tb_tool.getcol('ANTENNA_ID', 0, -1))
        mcoords = dict([('time', []),
                        ('antenna_id',
                         xarray.DataArray(baselines, dims=['antenna_id']))])
    else:
        ant1, ant2 = tb_tool.getcol('ANTENNA1', 0,
                                    -1), tb_tool.getcol('ANTENNA2', 0, -1)
        baselines = np.array([
            str(ll[0]).zfill(3) + '_' + str(ll[1]).zfill(3)
            for ll in np.unique(np.hstack([ant1[:, None], ant2[:, None]]),
                                axis=0)
        ])
        mcoords = dict([
            ('time', []),
            ('baseline',
             xarray.DataArray(np.arange(len(baselines)).astype('int64'),
                              dims=['baseline']))
        ])

    dims = [
        'time', 'antenna_id', 'd2', 'd3'
    ] if infile.endswith('POINTING') else ['time', 'baseline', 'chan', 'pol']
    cols = tb_tool.colnames()
    times, mvars, mchunks, xds = [], {}, {}, xarray.Dataset()

    ts_tb = tables.taql('select DISTINCT TIME from %s%s' % (infile, subseltql))
    utimes = np.unique(ts_tb.getcol('TIME', 0, -1))
    ts_tb.close()
    subseltql = ' where ' if subsel is None else ' where DATA_DESC_ID = %s AND ' % str(
        subsel)

    # for each batch of times in a chunk
    for ts in range(0, len(utimes), chunks[0]):
        ts_tb = tables.taql('select * from %s%sTIME BETWEEN %f and %f' %
                            (infile, subseltql, utimes[ts], utimes[min(
                                len(utimes) - 1, ts + chunks[0] - 1)]))
        raw_times = ts_tb.getcol('TIME', 0, -1)
        times = np.unique(raw_times)
        tidxs = np.searchsorted(utimes, raw_times) - ts

        # compute the indices of all the times and baselines at this timestep
        if infile.endswith('POINTING'):
            ts_bases = ts_tb.getcol('ANTENNA_ID', 0, -1)
        else:
            ts_ant1, ts_ant2 = ts_tb.getcol('ANTENNA1', 0, -1), ts_tb.getcol(
                'ANTENNA2', 0, -1)
            ts_bases = np.array([
                str(ll[0]).zfill(3) + '_' + str(ll[1]).zfill(3)
                for ll in np.hstack([ts_ant1[:, None], ts_ant2[:, None]])
            ])
        bidxs = np.searchsorted(baselines, ts_bases)

        # extract data for each col
        for col in cols:
            if (col in ignore + ['TIME']) or (not ts_tb.iscelldefined(col, 0)):
                continue
            data = np.array(ts_tb.getcol(col, 0, -1))
            if data.size == 0: continue
            fulldata = np.full((
                len(times),
                len(baselines),
            ) + data.shape[1:],
                               np.nan,
                               dtype=data.dtype)
            fulldata[tidxs, bidxs] = data
            mchunks = dict(
                zip(dims[:len(fulldata.shape)], [
                    chunks[ii] if chunks[ii] > 0 else fulldata.shape[ii]
                    for ii in range(len(fulldata.shape))
                ]))
            mvars[col.upper()] = xarray.DataArray(
                fulldata, dims=dims[:len(fulldata.shape)]).chunk(mchunks)

        mcoords.update({
            'time':
            xarray.DataArray(convert_time(np.array(times)), dims=['time'])
        })
        xds = xarray.Dataset(mvars, coords=mcoords)
        if (not nofile) and (ts == 0):
            encoding = dict(
                zip(list(xds.data_vars), cycle([{
                    'compressor': compressor
                }])))
            xds.to_zarr(outfile,
                        mode='w',
                        encoding=encoding,
                        consolidated=True)
        elif not nofile:
            xds.to_zarr(outfile,
                        mode='a',
                        append_dim='time',
                        compute=True,
                        consolidated=True)
        print('processed %s time steps' % str(ts + 1), end='\r')
        ts_tb.close()

    if (not nofile) and (tb_tool.nrows() > 0):
        xds = xarray.open_zarr(outfile)
    tb_tool.close()

    return xds
Esempio n. 25
0
 def get_flags(self):
     """
     """
     ms = self.ms_file
     return taql('select FLAG from $ms where ANTENNA1 != ANTENNA2')
Esempio n. 26
0
def create_STATION_UVW_process(timeIndex, freqIndex, args):
    """
    Inner-loop of create_STATION_UVW().
    This function should be placed in create_STATION_UVW(), but due to limitations of joblib, it has been placed here.

    :param timeIndex: time index
    :param freqIndex: frequency index
    :param args: output of parseArgs()
    :return: tuple (timeIndex, freqIndex, STATION_INFO_UVW)
    """
    import casacore.measures as cm
    import casacore.quanta as cq

    def ITRF_to_J2000(time, x, y, z):
        dm = cm.measures()
        dm.do_frame(dm.epoch('UTC', cq.quantity(time, 's')))

        ITRF_position = dm.position(rf='ITRF',
                                    v0=cq.quantity(x, 'm'),
                                    v1=cq.quantity(y, 'm'),
                                    v2=cq.quantity(z, 'm'))
        dm.do_frame(ITRF_position)

        ITRFLL_position = dm.measure(ITRF_position, 'ITRFLL')
        height = ITRFLL_position['m2']

        ITRFLL_direction = dm.direction('ITRFLL',
                                        v0=ITRFLL_position['m0'],
                                        v1=ITRFLL_position['m1'])

        J2000_direction = dm.measure(ITRFLL_direction, 'J2000')
        J2000_position = dm.position(rf='ITRF',
                                     v0=J2000_direction['m0'],
                                     v1=J2000_direction['m1'],
                                     v2=height)

        (az, el,
         r) = (J2000_position['m0']['value'], J2000_position['m1']['value'],
               J2000_position['m2']['value'])
        return (az, el, r)

    ITRF_to_J2000_vec = np.vectorize(ITRF_to_J2000)

    store = openHDF5(args)
    time = store['TIME_MAP'].loc[timeIndex].values
    wavelength = sc.speed_of_light / store['FREQ_MAP'].loc[freqIndex].values
    STATION_INFO_CART = store['STATION_INFO']
    store.close()

    (az, el, r) = ITRF_to_J2000_vec(time, STATION_INFO_CART['x'].values,
                                    STATION_INFO_CART['y'].values,
                                    STATION_INFO_CART['z'].values)
    (x, y, z) = cc.sph2cart(az, el, r)

    # pointing direction in radian (RA, DEC)
    pointingDirection = ct.taql(
        'select REFERENCE_DIR from {msFile}::FIELD'.format(
            msFile=args['msFile'])).getcol('REFERENCE_DIR').squeeze()
    M = cc.J2000_to_UVW_operator(*pointingDirection)
    # TODO: different normalization for other datasets. Add a switch to control this.
    (u, v, w) = M.dot(np.vstack((x, y, z))) / wavelength

    STATION_INFO_UVW = pd.DataFrame({
        'stationID': STATION_INFO_CART['stationID'],
        'antennaID': STATION_INFO_CART['antennaID'],
        'u': u,
        'v': v,
        'w': w
    })
    STATION_INFO_UVW = STATION_INFO_UVW[[
        'stationID', 'antennaID', 'u', 'v', 'w'
    ]]
    return timeIndex, freqIndex, STATION_INFO_UVW
Esempio n. 27
0
'''
print '[MaSER] Forming baselines...'
ANTENNA1 = msfile.getcol('ANTENNA1')
ANTENNA2 = msfile.getcol('ANTENNA2')

ANTENNAS = set(ANTENNA1).union(set(ANTENNA2))
baselines = form_baselines(ANTENNAS)

print len(ANTENNA1)
print len(ANTENNA2)
print len(baselines)

print 'Formed %d baselines.' % len(baselines)

# Determine the number of correlations present.
polarizations = ct.taql('SELECT FROM %s::POLARIZATION' % (filename))
correlations = polarizations.getcol('NUM_CORR')
print 'Found %d correlations.' % correlations[0]
'''
Extract the visibilities per baseline. The measurement set is accessed using TaQL extracting
the columns into numpy arrays.
The final data is stored in regular text files, with each baseline and correlation having its
own file.
The columns will have the uvw coordinates, channel frequency, real and imaginary parts of the
visibilities and their corresponding errors.
'''
print '[MaSER] Extracting visibilities per baseline...'
try:
    os.mkdir('visibilities')
except:
    pass
Esempio n. 28
0
    die('DDE solutions not found!')
else:
    LOGGER.info('Creating final 1'' image with DD solutions.')
    CMD = 'DDF.py --Output-Name=image_dd --Data-MS={:s} --Deconv-PeakFactor 0.050000 --Data-ColName {ic:s} --Data-ChunkHours 4 --Parallel-NCPU=32 --Beam-CenterNorm=1 --Deconv-CycleFactor=0 --Deconv-MaxMinorIter=10000 --Deconv-MaxMajorIter=6 --Deconv-Mode SSD --Beam-Model=LOFAR --Beam-LOFARBeamMode=A --Weight-Mode Natural  --Image-NPix=25000 --CF-wmax 50000 --CF-Nw 100 --Output-Also onNeds --Image-Cell {cell:f} --Facets-NFacets=7 --SSDClean-NEnlargeData 0 --Freq-NDegridBand 1 --Beam-NBand 1 --Facets-DiamMax 0.5 --Facets-DiamMin 0.1 --Deconv-RMSFactor=3.000000 --SSDClean-ConvFFTSwitch 10000 --Data-Sort 1 --Cache-Dir=. --Log-Memory 1 --GAClean-RMSFactorInitHMP 1.000000 --GAClean-MaxMinorIterInitHMP 10000.000000 --DDESolutions-SolsDir=SOLSDIR --Cache-Weight=reset --Output-Mode=Clean --Output-RestoringBeam {beam:s} --Weight-ColName="IMAGING_WEIGHT" --Freq-NBand=2 --RIME-DecorrMode=FT --SSDClean-SSDSolvePars [S,Alpha] --SSDClean-BICFactor 0 --Mask-Auto=1 --Mask-SigTh=10.00 --Selection-UVRangeKm=[5.0,2000.000000] --GAClean-MinSizeInit=10 --Mask-External=image_dirin_SSD_init_natural_m.app.restored.fits.mask.fits --DDESolutions-DDSols={ddsols:s}'.format(CONFIG['data']['mslist'], ic=CONFIG['image']['data_column'], cell=float(CONFIG['image']['cellsize_full'], beam=DDF_RESTORING_BEAM), ddsols=CONFIG['solutions']['ddsols_h5'])
    LOGGER.info(CMD)
    subprocess.call(CMD, shell=True)

LOGGER.info('Making PyBDSF catalogue of DD calibrated 1'' map.')

run_pybdsf(fitsname='image_dirin_SSD_init_natural_m.int.restored.fits', detectimage='image_dirin_SSD_init_natural_m.app.restored.fits', outcat='catalogue_1asec_final_dd')

IN_CATALGOUE = 'catalogue_1asec_final_dd.csv'
subprocess.call("sed '/^[[:space:]]*$/d' {:s} > {:s}".format(IN_CATALOGUE, 'catalogue_1asec_final_dd.sedded.csv'), shell=True)

if not CONFIG['mosaic'].getboolean('do_mosaic'):
    LOGGER.info('Pipeline finished successfully.')
    sys.exit(0)

LOGGER.info('Creating directions for full FoV 0.3'' imaging.')
tab = ct.taql('SELECT REFERENCE_DIR FROM {:s}::FIELD'.format(MSES[0]))
PHASECENTER = tab.getcol('REFERENCE_DIR').squeeze()
make_tiles(*PHASECENTER)

# Make phaseshifted copies of each tile and image.
DPPP_PARSETS = sorted(glob.glob('shift_to_facet_*.parset'))
for i, p in enumerate(DPPP_PARSETS):
    for ms in MSES:
        subprocess.call('DPPP {:s} msin={:s} msout={:s}'.format(p, ms, ms[:-3] + '.facet_{:02d}'.format(i), shell=True)
    # Image here.
    # WSClean with IDG on GPU, or DDFacet?