Example #1
0
def run( step, parset, H ):
    """
    Copy values from a table to another (of the same kind)
    If tables have different sampling, then resample the values
    """
    import numpy as np
    from losoto.h5parm import solFetcher, solWriter
    
    inTable = parset.getString('.'.join(["LoSoTo.Steps", step, "InTable"]), '' ) # complete solset/soltab
    outTable = parset.getString('.'.join(["LoSoTo.Steps", step, "OutTable"]), '' ) # complete solset/soltab or ''

    if inTable == '':
        logging.error('InTable is undefined.')
        return 1

    if outTable == '':
        outSolsetName = inTable.split('/')[0]
        outTableName = None
    else:
        outSolsetName = outTable.split('/')[0]
        outTableName = outTable.split('/')[1]

    ss, st = inTable.split('/')
    sf = solFetcher(H.getSoltab(ss, st))

    t = H.makeSoltab(solset = outSolsetName, soltype = sf.getType(), soltab = outTableName, axesNames=sf.getAxesNames(), \
        axesVals=[sf.getAxisValues(axisName) for axisName in sf.getAxesNames()], \
        vals=sf.getValues(retAxesVals = False), weights=sf.getValues(weight = True, retAxesVals = False), parmdbType=sf.t._v_attrs['parmdb_type'])

    sw = solWriter(t)
    sw.addHistory('DUPLICATE (from table %s)' % (inTable))
    return 0
Example #2
0
def run(step, parset, H):
    """
    Copy values from a table to another (of the same kind)
    If tables have different sampling, then resample the values
    """
    import numpy as np
    from losoto.h5parm import solFetcher, solWriter

    inTable = parset.getString('.'.join(["LoSoTo.Steps", step, "InTable"]),
                               '')  # complete solset/soltab
    outTable = parset.getString('.'.join(["LoSoTo.Steps", step, "OutTable"]),
                                '')  # complete solset/soltab or ''

    if inTable == '':
        logging.error('InTable is undefined.')
        return 1

    if outTable == '':
        outSolsetName = inTable.split('/')[0]
        outTableName = None
    else:
        outSolsetName = outTable.split('/')[0]
        outTableName = outTable.split('/')[1]

    ss, st = inTable.split('/')
    sf = solFetcher(H.getSoltab(ss, st))

    t = H.makeSoltab(solset = outSolsetName, soltype = sf.getType(), soltab = outTableName, axesNames=sf.getAxesNames(), \
        axesVals=[sf.getAxisValues(axisName) for axisName in sf.getAxesNames()], \
        vals=sf.getValues(retAxesVals = False), weights=sf.getValues(weight = True, retAxesVals = False), parmdbType=sf.t._v_attrs['parmdb_type'])

    sw = solWriter(t)
    sw.addHistory('DUPLICATE (from table %s)' % (inTable))
    return 0
Example #3
0
def run(step, parset, H):

    import numpy as np
    from losoto.h5parm import solFetcher, solWriter

    soltabs = getParSoltabs(step, parset, H)

    for soltab in openSoltabs(H, soltabs):

        logging.info("Taking ABSolute value of soltab: " + soltab._v_name)

        sf = solFetcher(soltab)
        sw = solWriter(soltab)

        # axis selection
        userSel = {}
        for axis in sf.getAxesNames():
            userSel[axis] = getParAxis(step, parset, H, axis)
        sf.setSelection(**userSel)

        vals = sf.getValues(retAxesVals=False)
        count = np.count_nonzero(vals < 0)

        logging.info('Abs: %i points initially negative (%f %%)' %
                     (count, 100 * float(count) / np.count_nonzero(vals)))

        # writing back the solutions
        sw.setValues(np.abs(vals))

        sw.addHistory('ABSolute value taken')

    return 0
Example #4
0
def run( step, parset, H ):

    import numpy as np
    from losoto.h5parm import solFetcher, solWriter

    soltabs = getParSoltabs( step, parset, H )

    for soltab in openSoltabs( H, soltabs ):

        logging.info("Taking ABSolute value of soltab: "+soltab._v_name)

        sf = solFetcher(soltab)
        sw = solWriter(soltab)

        # axis selection
        userSel = {}
        for axis in sf.getAxesNames():
            userSel[axis] = getParAxis( step, parset, H, axis )
        sf.setSelection(**userSel)

        vals = sf.getValues(retAxesVals = False)
        count = np.count_nonzero(vals<0)

        logging.info('Abs: %i points initially negative (%f %%)' % (count,100*float(count)/np.count_nonzero(vals)))

        # writing back the solutions
        sw.setValues(np.abs(vals))

        sw.addHistory('ABSolute value taken')
        
    return 0
Example #5
0
def run(step, parset, H):

    import scipy.ndimage.filters
    import numpy as np
    from losoto.h5parm import solFetcher, solWriter

    soltabs = getParSoltabs(step, parset, H)

    s = parset.getIntVector('.'.join(["LoSoTo.Steps", step, "smooth"]), 10)

    for soltab in openSoltabs(H, soltabs):

        logging.info("Smoothing soltab: " + soltab._v_name)

        sf = solFetcher(soltab)
        sw = solWriter(soltab)

        if sf.getType() != 'clock':
            logging.error(
                'Only clock-type solutions can be run in this operation.')
            return 1

        # axis selection
        userSel = {}
        for axis in sf.getAxesNames():
            userSel[axis] = getParAxis(step, parset, H, axis)
        sf.setSelection(**userSel)

        for vals, weights, coord, selection in sf.getValuesIter(
                returnAxes='time', weight=True):

            x = coord['time'][weights != 0]
            y = vals[weights != 0]
            weights = weights[weights != 0]
            spline = scipy.interpolate.UnivariateSpline(x,
                                                        y,
                                                        weights,
                                                        k=1,
                                                        s=1e-15)

            plot = True
            if plot:
                import matplotlib as mpl
                mpl.use("Agg")
                import matplotlib.pyplot as plt
                plt.plot(x, y, 'k.')
                plt.plot(x, spline(x), 'r-')
                plt.savefig('test.png')
                sys.exit(1)

            sw.selection = selection
            sw.setValues(spline(coord['time']))

        sw.addHistory('Smoothed with SMOOTHCLOCK.')
        del sf
        del sw
    return 0
Example #6
0
def run(step, parset, H):

    from losoto.h5parm import solFetcher, solWriter

    soltabs = getParSoltabs(step, parset, H)

    axesToExt = parset.getStringVector(
        '.'.join(["LoSoTo.Steps", step, "Axes"]), ['freq', 'time'])
    size = parset.getIntVector('.'.join(["LoSoTo.Steps", step, "Size"]),
                               [11, 11])
    percent = parset.getFloat('.'.join(["LoSoTo.Steps", step, "Percent"]), 50)
    cycles = parset.getInt('.'.join(["LoSoTo.Steps", step, "Cycles"]), 3)
    ncpu = parset.getInt('.'.join(["LoSoTo.Ncpu"]), 1)

    if axesToExt == []:
        logging.error("Please specify at least one axis to extend flag.")
        return 1

    # start processes for multi-thread
    mpm = multiprocManager(ncpu, flag)

    for soltab in openSoltabs(H, soltabs):

        logging.info("Extending flag on soltab: " + soltab._v_name)

        sf = solFetcher(soltab)
        sw = solWriter(soltab)

        # axis selection
        userSel = {}
        for axis in sf.getAxesNames():
            userSel[axis] = getParAxis(step, parset, H, axis)
        sf.setSelection(**userSel)

        for axisToExt in axesToExt:
            if axisToExt not in sf.getAxesNames():
                logging.error('Axis \"' + axisToExt + '\" not found.')
                return 1

        # fill the queue (note that sf and sw cannot be put into a queue since they have file references)
        for vals, weights, coord, selection in sf.getValuesIter(
                returnAxes=axesToExt, weight=True):
            mpm.put(
                [weights, coord, axesToExt, selection, percent, size, cycles])

        mpm.wait()

        logging.info('Writing solutions')
        for w, sel in mpm.get():
            sw.selection = sel
            sw.setValues(w, weight=True)  # convert back to np.float16

        sw.addHistory('FLAG EXTENDED (over %s)' % (str(axesToExt)))
        del sf
        del sw
    return 0
Example #7
0
def run( step, parset, H ):

    from losoto.h5parm import solFetcher, solWriter

    soltabs = getParSoltabs( step, parset, H )

    axesToExt = parset.getStringVector('.'.join(["LoSoTo.Steps", step, "Axes"]), ['freq','time'] )
    size = parset.getIntVector('.'.join(["LoSoTo.Steps", step, "Size"]), [11,11] )
    percent = parset.getFloat('.'.join(["LoSoTo.Steps", step, "Percent"]), 50 )
    cycles = parset.getInt('.'.join(["LoSoTo.Steps", step, "Cycles"]), 3 )
    ncpu = parset.getInt('.'.join(["LoSoTo.Ncpu"]), 0 )
    if ncpu == 0:
        import multiprocessing
        ncpu = multiprocessing.cpu_count()
    
    if axesToExt == []:
        logging.error("Please specify at least one axis to extend flag.")
        return 1

    for soltab in openSoltabs( H, soltabs ):

        # start processes for multi-thread
        mpm = multiprocManager(ncpu, flag)

        logging.info("Extending flag on soltab: "+soltab._v_name)

        sf = solFetcher(soltab)
        sw = solWriter(soltab)

        # axis selection
        userSel = {}
        for axis in sf.getAxesNames():
            userSel[axis] = getParAxis( step, parset, H, axis )
        sf.setSelection(**userSel)

        for axisToExt in axesToExt:
            if axisToExt not in sf.getAxesNames():
                logging.error('Axis \"'+axisToExt+'\" not found.')
                mpm.wait()
                return 1

        # fill the queue (note that sf and sw cannot be put into a queue since they have file references)
        for vals, weights, coord, selection in sf.getValuesIter(returnAxes=axesToExt, weight=True):
            mpm.put([weights, coord, axesToExt, selection, percent, size, cycles])

        mpm.wait()

        logging.info('Writing solutions')
        for w,sel in mpm.get():
            sw.selection = sel
            sw.setValues(w, weight=True) # convert back to np.float16

        sw.addHistory('FLAG EXTENDED (over %s)' % (str(axesToExt)))
        del sf
        del sw
    return 0
Example #8
0
def run(step, parset, H):
    """
    Normalize the solutions to a given value
    """
    import numpy as np
    from losoto.h5parm import solFetcher, solWriter

    soltabs = getParSoltabs(step, parset, H)
    solTypes = getParSolTypes(step, parset, H)

    normVal = parset.getFloat('.'.join(["LoSoTo.Steps", step, "NormVal"]), 1.)
    normAxes = parset.getStringVector(
        '.'.join(["LoSoTo.Steps", step, "NormAxes"]), ['time'])

    for soltab in openSoltabs(H, soltabs):

        logging.info("Normalizing soltab: " + soltab._v_name)

        tr = solFetcher(soltab)
        tw = solWriter(soltab, useCache=True)  # remember to flush!

        axesNames = tr.getAxesNames()
        for normAxis in normAxes:
            if normAxis not in axesNames:
                logging.error('Normalization axis ' + normAxis + ' not found.')
                return 1

        # axis selection
        userSel = {}
        for axis in tr.getAxesNames():
            userSel[axis] = getParAxis(step, parset, H, axis)
        tr.setSelection(**userSel)

        for vals, weights, coord, selection in tr.getValuesIter(
                returnAxes=normAxes, weight=True):

            # rescale solutions
            if np.sum(weights) == 0: continue  # skip flagged antenna
            valsMean = np.average(vals, weights=weights)
            valsNew = normVal * vals / valsMean
            logging.debug(str(coord))
            logging.debug("Rescaling by: " + str(normVal / valsMean))

            # writing back the solutions
            tw.selection = selection
            tw.setValues(valsNew)

        tw.flush()
        tw.addHistory('NORM (on axis %s)' % (normAxes))

    return 0
Example #9
0
def run( step, parset, H ):

    import scipy.ndimage.filters
    import numpy as np
    from losoto.h5parm import solFetcher, solWriter

    soltabs = getParSoltabs( step, parset, H )

    s = parset.getIntVector('.'.join(["LoSoTo.Steps", step, "smooth"]), 10 )

    for soltab in openSoltabs( H, soltabs ):

        logging.info("Smoothing soltab: "+soltab._v_name)

        sf = solFetcher(soltab)
        sw = solWriter(soltab)

        if sf.getType() != 'clock':
            logging.error('Only clock-type solutions can be run in this operation.')
            return 1

        # axis selection
        userSel = {}
        for axis in sf.getAxesNames():
            userSel[axis] = getParAxis( step, parset, H, axis )
        sf.setSelection(**userSel)

        for vals, weights, coord, selection in sf.getValuesIter(returnAxes='time', weight=True):

            x=coord['time'][weights != 0]
            y=vals[weights != 0]
            weights = weights[weights != 0]
            spline = scipy.interpolate.UnivariateSpline(x, y, weights, k=1, s=1e-15)

            plot = True
            if plot:
                import matplotlib as mpl
                mpl.use("Agg")
                import matplotlib.pyplot as plt
                plt.plot(x, y, 'k.')
                plt.plot(x, spline(x), 'r-')
                plt.savefig('test.png')
                sys.exit(1)

            sw.selection = selection
            sw.setValues(spline(coord['time']))

        sw.addHistory('Smoothed with SMOOTHCLOCK.')
        del sf
        del sw
    return 0
Example #10
0
def run( step, parset, H ):
    """
    Normalize the solutions to a given value
    """
    import numpy as np
    from losoto.h5parm import solFetcher, solWriter
    
    soltabs = getParSoltabs( step, parset, H )
    solTypes = getParSolTypes( step, parset, H )

    normVal = parset.getFloat('.'.join(["LoSoTo.Steps", step, "NormVal"]), 1. )
    normAxes = parset.getStringVector('.'.join(["LoSoTo.Steps", step, "NormAxes"]), ['time'] )

    for soltab in openSoltabs( H, soltabs ):

        logging.info("Normalizing soltab: "+soltab._v_name)

        tr = solFetcher(soltab)
        tw = solWriter(soltab, useCache = True) # remember to flush!

        axesNames = tr.getAxesNames()
        for normAxis in normAxes:
            if normAxis not in axesNames:
                logging.error('Normalization axis '+normAxis+' not found.')
                return 1

        # axis selection
        userSel = {}
        for axis in tr.getAxesNames():
            userSel[axis] = getParAxis( step, parset, H, axis )
        tr.setSelection(**userSel)

        for vals, weights, coord, selection in tr.getValuesIter(returnAxes=normAxes, weight = True):

            # rescale solutions
            if np.sum(weights) == 0: continue # skip flagged selections
            valsMean = np.average(vals, weights=weights)
            vals[weights != 0] *= normVal/valsMean
            logging.debug(str(coord))
            logging.debug("Rescaling by: "+str(normVal/valsMean))

            # writing back the solutions
            tw.selection = selection
            tw.setValues(vals)

        tw.flush()
        tw.addHistory('NORM (on axis %s)' % (normAxes))

    return 0
Example #11
0
def run( step, parset, H ):

    import scipy.ndimage.filters
    import numpy as np
    from losoto.h5parm import solFetcher, solWriter
    from scipy.optimize import minimize
    import itertools
    from scipy.interpolate import griddata
    import scipy.cluster.vq as vq

    def robust_std(data, sigma=3):
        """
        Calculate standard deviation excluding outliers
        ok with masked arrays
        """
        return np.std(data[np.where(np.abs(data) < sigma * np.std(data))])

    def mask_interp(vals, mask, method='nearest'):
        """
        return interpolated values for masked elements
        """
        this_vals = vals.copy()
        #this_vals[mask] = np.interp(np.where(mask)[0], np.where(~mask)[0], vals[~mask])
        #this_vals[mask] = griddata(np.where(~mask)[0], vals[~mask], np.where(mask)[0], method)

        # griddata has nan bug with nearest, I need to use vq
        code, dist = vq.vq(np.where(mask)[0], np.where(~mask)[0])
        this_vals[ np.where(mask)[0] ] = this_vals[code]
        
        return this_vals

    tec_jump_val = 0.019628
    maxsize = 300
    clip = 10 # TECs over these amount of jumps are flagged

    soltabs = getParSoltabs( step, parset, H )

    for soltab in openSoltabs( H, soltabs ):

        logging.info("Removing TEC jumps from soltab: "+soltab._v_name)

        sf = solFetcher(soltab)
        sw = solWriter(soltab) # remember to flush!

        # TODO: check if it's a Tec table

        # axis selection
        userSel = {}
        for axis in sf.getAxesNames():
            userSel[axis] = getParAxis( step, parset, H, axis )
        sf.setSelection(**userSel)

        for vals, weights, coord, selection in sf.getValuesIter(returnAxes='time', weight=True):

            # skip all flagged
            if (weights == 0).all(): continue
            # skip reference
            if (np.diff(vals[(weights == 1)]) == 0).all(): continue

            # kill large values
#            weights[abs(vals/tec_jump_val)>clip] = 0

            # interpolate flagged values to get resonable distances
#            vals = mask_interp(vals, mask=(weights == 0))/tec_jump_val
            # add edges to allow intervals to the borders
#            vals = np.insert(vals, 0, vals[0])
#            vals = np.insert(vals, len(vals), vals[-1])

            vals = np.fmod(vals,tec_jump_val)

#            def find_jumps(d_vals):
#                # jump poistion finder
#                d_smooth = scipy.ndimage.filters.median_filter( mask_interp(d_vals, mask=(abs(d_vals)>0.8)), 21 )
#                d_vals -= d_smooth
#                jumps = list(np.where(np.abs(d_vals) > 1.)[0])
#                return [0]+jumps+[len(d_vals)-1] # add edges
#
#            class Jump(object):
#                def __init__(self, jumps_idx, med):
#                    self.idx_left = jumps_idx[0]
#                    self.idx_right = jumps_idx[1]
#                    self.jump_left = np.rint(d_vals[self.idx_left])
#                    self.jump_right = np.rint(d_vals[self.idx_right])
#                    self.size = self.idx_right-self.idx_left
#                    self.hight = np.median(vals[self.idx_left+1:self.idx_right+1]-med)
#                    if abs((self.hight-self.jump_left)-med) > abs((self.hight-self.jump_right)-med):
#                        self.closejump = self.jump_right
#                    else:
#                        self.closejump = self.jump_left
#    
#            i = 0
#            while i<len(coord['time']):
#                # get tec[i] - tec[i+1], i.e. the derivative assuming constant timesteps
#                # this is in units of tec_jump_val!
#                d_vals = np.diff(vals)
#                # get jumps idx, idx=n means a jump beteen val n and n+1
#                jumps_idx = find_jumps(d_vals)
#
#                # get regions
#                med = np.median(vals)
#                jumps = [Jump(jump_idx, med) for jump_idx in zip( jumps_idx[:-1], jumps_idx[1:] )]
#                jumps = [jump for jump in jumps if jump.closejump != 0]
#                jumps = [jump for jump in jumps if jump.size != 0] # prevent bug on edges
#                jumps = [jump for jump in jumps if jump.size < maxsize]
#
#                jumps.sort(key=lambda x: (np.abs(x.size), x.hight), reverse=False) #smallest first
#                #print [(j.hight, j.closejump) for j in jumps]
#
#                plot = False
#                if plot:
#                    import matplotlib.pyplot as plt
#                    fig, ((ax1, ax2, ax3)) = plt.subplots(3, 1, sharex=True)
#                    fig.subplots_adjust(hspace=0)
#                    d_smooth = scipy.ndimage.filters.median_filter( mask_interp(d_vals, mask=(abs(d_vals)>0.8)), 31 )
#                    ax1.plot(d_vals,'k-')
#                    ax2.plot(d_smooth,'k-')
#                    ax3.plot(vals, 'k-')
#                    [ax3.axvline(jump_idx+0.5, color='r', ls=':') for jump_idx in jumps_idx]
#                    ax1.set_ylabel('d_vals')
#                    ax2.set_ylabel('d_vals - smooth')
#                    ax3.set_ylabel('TEC/jump')
#                    ax3.set_xlabel('timestep')
#                    ax1.set_xlim(xmin=-10, xmax=len(d_smooth)+10)
#                    fig.savefig('plots/%stecjump_debug_%03i' % (coord['ant'], i))
#                i+=1
#
#                if len(jumps) == 0: 
#                    break
#
#                # move down the highest to the side closest to the median
#                j = jumps[0]
#                #print j.idx_left, j.idx_right, j.jump_left, j.jump_right, j.hight, j.closejump
#
#                vals[j.idx_left+1:j.idx_right+1] -= j.closejump
#                logging.debug("%s: Number of jumps left: %i - Removing jump: %i - Size %i" % (coord['ant'], len(jumps_idx)-2, j.closejump, j.size))
                
            # re-create proper vals
#            vals = vals[1:-1]*tec_jump_val
            # set back to 0 the values for flagged data
            vals[weights == 0] = 0

            sw.selection = selection
            sw.setValues(vals)
            sw.setValues(weights, weight=True)

        sw.addHistory('TECJUMP')
        del sf
        del sw
    return 0
Example #12
0
axesVals = [['a','b','c','d'], np.arange(10), np.arange(100)]
vals = np.arange(4*10*100).reshape(4,10,100)
logging.info("Create soltab")
H5.makeSoltab(ss, 'amplitude', 'stTest', axesNames=['axis1','axis2','axis3'], axesVals=axesVals, vals=vals, weights=vals)
logging.info("Create soltab (using same name)")
H5.makeSoltab(ss, 'amplitude', 'stTest', axesNames=['axis1','axis2','axis3'], axesVals=axesVals, vals=vals, weights=vals)
logging.info("Create soltab (using default name)")
H5.makeSoltab(ss, 'amplitude', axesNames=['axis1','axis2','axis3'], axesVals=axesVals, vals=vals, weights=vals)
logging.info('Get a soltab object')
st=H5.getSoltab(ss,'stTest')
logging.info('Get all soltabs:')
print H5.getSoltabs(ss)

print "###########################################"
logging.info('### solFetcher/solWriter - General')
Hsf = solFetcher(st)
Hsw = solWriter(st)
logging.info('Get solution Type (exp: amplitude)')
print Hsf.getType()
logging.info('Get Axes Names')
print Hsf.getAxesNames()
logging.info('Get Axis1 Len (exp: 4)')
print Hsf.getAxisLen('axis1')
logging.info('Get Axis1 Type (exp: str)')
print Hsf.getAxisType('axis1')
logging.info('Get Axis2 Type (exp: float)')
print Hsf.getAxisType('axis2')
logging.info('Get Axis1 Values (exp: a,b,c,d)')
print Hsf.getAxisValues('axis1')
logging.info('Set new axes values')
Hsw.setAxisValues('axis1',['e','f','g','h'])
Example #13
0
def run( step, parset, H ):

    import os
    import numpy as np
    from losoto.h5parm import solFetcher, solHandler
    # avoids error if re-setting "agg" a second run of plot
    if not 'matplotlib' in sys.modules:
        import matplotlib as mpl
        mpl.rc('font',size =8 )
        mpl.rc('figure.subplot',left=0.05, bottom=0.05, right=0.95, top=0.95,wspace=0.22, hspace=0.22 )
        mpl.use("Agg")
    import matplotlib.pyplot as plt # after setting "Agg" to speed up

    soltabs = getParSoltabs( step, parset, H )

    minZ, maxZ = parset.getDoubleVector('.'.join(["LoSoTo.Steps", step, "MinMax"]), [0,0] )
    prefix = parset.getString('.'.join(["LoSoTo.Steps", step, "Prefix"]), '' )

    # Plot various TEC-screen properties

    for st_scr in openSoltabs(H, soltabs):

        # Check if soltab is a tecscreen table
        full_name = st_scr._v_parent._v_name + '/' + st_scr._v_name
        if st_scr._v_title != 'tecscreen':
            logging.warning('Solution table {0} is not a tecscreen solution '
                'table. Skipping.'.format(full_name))
            continue
        logging.info('Using input solution table: {0}'.format(full_name))

        # Plot TEC screens as images
        solset = st_scr._v_parent
        sf_scr = solFetcher(st_scr)
        r, axis_vals = sf_scr.getValues()
        source_names = axis_vals['dir']
        station_names = axis_vals['ant']
        station_dict = H.getAnt(solset)
        station_positions = []
        for station in station_names:
            station_positions.append(station_dict[station])
        times = axis_vals['time']

        tec_screen, axis_vals = sf_scr.getValues()
        times = axis_vals['time']
        residuals = sf_scr.getValues(weight=True, retAxesVals=False)
        height = st_scr._v_attrs['height']
        order = st_scr._v_attrs['order']
        beta_val = st_scr._v_attrs['beta']
        r_0 = st_scr._v_attrs['r_0']
        pp = sf_scr.t.piercepoint

        if (minZ == 0 and maxZ == 0):
            min_tec = None
            max_tec = None
        else:
            min_tec = minZ
            max_tec = maxZ

        make_tec_screen_plots(pp, tec_screen, residuals,
            np.array(station_positions), np.array(source_names), times,
            height, order, beta_val, r_0, prefix=prefix,
            remove_gradient=True, show_source_names=False, min_tec=min_tec,
            max_tec=max_tec)

    return 0
Example #14
0
def run(step, parset, H):

    from losoto.h5parm import solFetcher, solWriter

    soltabs = getParSoltabs(step, parset, H)

    #check_parset('Axes','MaxCycles','MaxRms','Order','Replace','PreFlagZeros')
    axesToFlag = parset.getStringVector(
        '.'.join(["LoSoTo.Steps", step, "Axes"]), 'time')
    maxCycles = parset.getInt('.'.join(["LoSoTo.Steps", step, "MaxCycles"]), 5)
    maxRms = parset.getFloat('.'.join(["LoSoTo.Steps", step, "MaxRms"]), 5.)
    fixRms = parset.getFloat('.'.join(["LoSoTo.Steps", step, "FixRms"]), 0)
    order = parset.getIntVector('.'.join(["LoSoTo.Steps", step, "Order"]), 3)
    replace = parset.getBool('.'.join(["LoSoTo.Steps", step, "Replace"]),
                             False)
    preflagzeros = parset.getBool(
        '.'.join(["LoSoTo.Steps", step, "PreFlagZeros"]), False)
    mode = parset.getString('.'.join(["LoSoTo.Steps", step, "Mode"]), 'smooth')
    ref = parset.getString('.'.join(["LoSoTo.Steps", step, "Reference"]), '')
    ncpu = parset.getInt('.'.join(["LoSoTo.Ncpu"]), 1)

    if ref == '': ref = None

    if axesToFlag == []:
        logging.error("Please specify axis to flag. It must be a single one.")
        return 1

    if len(axesToFlag) != len(order):
        logging.error("AxesToFlag and order must be both 1 or 2 values.")
        return 1

    if len(order) == 1: order = order[0]
    elif len(order) == 2: order = tuple(order)

    mode = mode.lower()
    if mode != 'smooth' and mode != 'poly' and mode != 'spline':
        logging.error('Mode must be smooth, poly or spline')
        return 1

    # start processes for multi-thread
    mpm = multiprocManager(ncpu, flag)

    for soltab in openSoltabs(H, soltabs):

        logging.info("Flagging soltab: " + soltab._v_name)

        sf = solFetcher(soltab)
        sw = solWriter(soltab, useCache=True)  # remember to flush!

        # axis selection
        userSel = {}
        for axis in sf.getAxesNames():
            userSel[axis] = getParAxis(step, parset, H, axis)
        sf.setSelection(**userSel)

        for axisToFlag in axesToFlag:
            if axisToFlag not in sf.getAxesNames():
                logging.error('Axis \"' + axis + '\" not found.')
                return 1

        # reorder axesToFlag as axes in the table
        axesToFlag_orig = axesToFlag
        axesToFlag = [
            coord for coord in sf.getAxesNames() if coord in axesToFlag
        ]
        if type(order) is int: order = [order]
        if axesToFlag_orig != axesToFlag:
            order = order[::-1]  # reverse order if we changed axesToFlag

        solType = sf.getType()

        # fill the queue (note that sf and sw cannot be put into a queue since they have file references)
        for vals, weights, coord, selection in sf.getValuesIter(
                returnAxes=axesToFlag, weight=True, reference=ref):
            mpm.put([
                vals, weights, coord, solType, order, mode, preflagzeros,
                maxCycles, fixRms, maxRms, replace, axesToFlag, selection
            ])
            #v, w, sel = flag(vals, weights, coord, solType, order, mode, preflagzeros, maxCycles, fixRms, maxRms, replace, axesToFlag, selection)

        mpm.wait()

        for v, w, sel in mpm.get():
            sw.selection = sel
            if replace:
                # rewrite solutions (flagged values are overwritten)
                sw.setValues(v, weight=False)
            else:
                sw.setValues(w, weight=True)

        sw.flush()
        sw.addHistory('FLAG (over %s with %s sigma cut)' %
                      (axesToFlag, maxRms))

        del sw
        del sf
        del soltab

    return 0
def makeTECparmdb(H, solset, TECsolTab, timewidths, freq, freqwidth):
    """Returns TEC screen parmdb parameters

    H - H5parm object
    solset - solution set with TEC screen parameters
    TECsolTab = solution table with tecscreen values
    timewidths - time widths of output parmdb
    freq - frequency of output parmdb
    freqwidth - frequency width of output parmdb
    """
    from pylab import pinv
    global ipbar, pbar

    station_dict = H.getAnt(solset)
    station_names = station_dict.keys()
    station_positions = station_dict.values()
    source_dict = H.getSou(solset)
    source_names = source_dict.keys()
    source_positions = source_dict.values()

    tec_sf = solFetcher(TECsolTab)
    tec_screen, axis_vals = tec_sf.getValues()
    times = axis_vals['time']
    beta = TECsolTab._v_attrs['beta']
    r_0 = TECsolTab._v_attrs['r_0']
    height = TECsolTab._v_attrs['height']
    order = TECsolTab._v_attrs['order']
    pp = tec_sf.t.piercepoint

    N_sources = len(source_names)
    N_times = len(times)
    N_freqs = 1
    N_stations = len(station_names)
    N_piercepoints = N_sources * N_stations

    freqs = freq
    freqwidths = freqwidth
    parms = {}
    v = {}
    v['times'] = times
    v['timewidths'] = timewidths
    v['freqs'] = freqs
    v['freqwidths'] = freqwidths

    for station_name in station_names:
        for source_name in source_names:

            v['values'] = np.zeros((N_times, N_freqs), dtype=np.double)
            parmname = 'Piercepoint:X:%s:%s' % (station_name, source_name)
            parms[parmname] = v.copy()

            v['values'] = np.zeros((N_times, N_freqs), dtype=np.double)
            parmname = 'Piercepoint:Y:%s:%s' % (station_name, source_name)
            parms[parmname] = v.copy()

            v['values'] = np.zeros((N_times, N_freqs), dtype=np.double)
            parmname = 'Piercepoint:Z:%s:%s' % (station_name, source_name)
            parms[parmname] = v.copy()

            v['values'] = np.zeros((N_times, N_freqs), dtype=np.double)
            parmname = 'TECfit_white:%s:%s' % (station_name, source_name)
            parms[parmname] = v.copy()

            v['values'] = np.zeros((N_times, N_freqs), dtype=np.double)
            parmname = 'TECfit_white:0:%s:%s' % (station_name, source_name)
            parms[parmname] = v.copy()

            v['values'] = np.zeros((N_times, N_freqs), dtype=np.double)
            parmname = 'TECfit_white:1:%s:%s' % (station_name, source_name)
            parms[parmname] = v.copy()

    for k in range(N_times):
        D = np.resize(pp[k, :, :], (N_piercepoints, N_piercepoints, 3))
        D = np.transpose(D, (1, 0, 2)) - D
        D2 = np.sum(D**2, axis=2)
        C = -(D2 / (r_0**2))**(beta / 2.0) / 2.0
        tec_fit_white = np.dot(pinv(C),
                               tec_screen[:, k, :].reshape(N_piercepoints))
        pp_idx = 0
        for src, source_name in enumerate(source_names):
            for sta, station_name in enumerate(station_names):

                parmname = 'Piercepoint:X:%s:%s' % (station_name, source_name)
                parms[parmname]['values'][k, 0] = pp[k, pp_idx, 0]

                parmname = 'Piercepoint:Y:%s:%s' % (station_name, source_name)
                parms[parmname]['values'][k, 0] = pp[k, pp_idx, 1]

                parmname = 'Piercepoint:Z:%s:%s' % (station_name, source_name)
                parms[parmname]['values'][k, 0] = pp[k, pp_idx, 2]

                parmname = 'TECfit_white:%s:%s' % (station_name, source_name)
                parms[parmname]['values'][k, 0] = tec_fit_white[pp_idx]

                parmname = 'TECfit_white:0:%s:%s' % (station_name, source_name)
                parms[parmname]['values'][k, 0] = tec_fit_white[pp_idx]

                parmname = 'TECfit_white:1:%s:%s' % (station_name, source_name)
                parms[parmname]['values'][k, 0] = tec_fit_white[pp_idx]

                pp_idx += 1
        pbar.update(ipbar)
        ipbar += 1

    time_start = times[0] - timewidths[0] / 2
    time_end = times[-1] + timewidths[-1] / 2

    v['times'] = np.array([(time_start + time_end) / 2])
    v['timewidths'] = np.array([time_end - time_start])

    v_r0 = v.copy()
    v_r0['values'] = np.array(r_0, dtype=np.double, ndmin=2)
    parms['r_0'] = v_r0

    v_beta = v.copy()
    v_beta['values'] = np.array(beta, dtype=np.double, ndmin=2)
    parms['beta'] = v_beta

    v_height = v.copy()
    v_height['values'] = np.array(height, dtype=np.double, ndmin=2)
    parms['height'] = v_height

    return parms
Example #16
0
def run( step, parset, H ):
   """
   Generic unspecified step for easy expansion.
   """
   import numpy as np
   from losoto.h5parm import solFetcher, solWriter
   # all the following are LoSoTo function to extract information from the parset

   # get involved solsets using local step values or global values or all
   solsets = getParSolsets( step, parset, H )
   logging.info('Solset: '+str(solsets))
   # get involved soltabs using local step values or global values or all
   soltabs = getParSoltabs( step, parset, H )
   logging.info('Soltab: '+str(soltabs))
   # get list of SolTypes using local step values or global values or all
   solTypes = getParSolTypes( step, parset, H )
   logging.info('SolType: '+str(solTypes))


   # do something on every soltab (use the openSoltab LoSoTo function)
   for soltab in openSoltabs( H, soltabs ):
        logging.info("--> Working on soltab: "+soltab._v_name)
        # use the solFetcher from the H5parm lib
        t = solFetcher(soltab)
        tw = solWriter(soltab)

        axisNames = t.getAxesNames()
        logging.info("Axis names are: "+str(axisNames))

        solType = t.getType()
        logging.info("Soltab type is: "+solType)

        # this will make a selection for the getValues() and getValuesIter()
        # interpret every entry in the parset which has an axis name as a selector
        userSel = {}
        for axis in t.getAxesNames():
            userSel[axis] = getParAxis( step, parset, H, axis )
        t.setSelection(**userSel)

        t.setSelection(ant=ants, pol=pols, dir=dirs)
        logging.info("Selection is: "+str(t.selection))

        # find axis values
        logging.info("Antennas (no selection) are: "+str(t.getAxisValues('ant', ignoreSelection=True)))
        logging.info("Antennas (with selection) are: "+str(t.getAxisValues('ant')))
        # but one can also use (selection is active here!)
        logging.info("Antennas (other method) are: "+str(t.ant))
        logging.info("Frequencies are: "+str(t.freq))
        logging.info("Directions are: "+str(t.dir))
        logging.info("Polarizations are: "+str(t.pol))
        # try to access a non-existent axis
        t.getAxisValues('nonexistantaxis')

        # now get all values given this selection
        logging.info("Get data using t.val")
        val = t.val
        logging.debug('shape of val: '+str(t.val.shape))
        logging.info("$ val is "+str(val[0,0,0,0,100]))
        weight = t.weight
        time = t.time
        thisTime = t.time[100]

        # another way to get the data is using the getValues()
        logging.info("Get data using getValues()")
        grid, axes = t.getValues()
        # axis names
        logging.info("Axes: "+str(t.getAxesNames()))
        # axis shape
        print axes
        print [t.getAxisLen(axis) for axis in axes] # not ordered, is a dict!
Example #17
0
def run( step, parset, H ):

    import numpy as np
    from losoto.h5parm import solFetcher, solWriter

    soltabs = getParSoltabs( step, parset, H )

    axesToClip = parset.getStringVector('.'.join(["LoSoTo.Steps", step, "Axes"]), [] )
    clipLevel = parset.getFloat('.'.join(["LoSoTo.Steps", step, "ClipLevel"]), 0. )
    log = parset.getBool('.'.join(["LoSoTo.Steps", step, "Log"]), True )
    
    if len(axesToClip) < 1:
        logging.error("Please specify axes to clip.")
        return 1
    if clipLevel == 0.:
        logging.error("Please specify factor above/below median at which to clip.")
        return 1

    for soltab in openSoltabs( H, soltabs ):

        logging.info("Clipping soltab: "+soltab._v_name)

        sf = solFetcher(soltab)

        # axis selection
        userSel = {}
        for axis in sf.getAxesNames():
            userSel[axis] = getParAxis( step, parset, H, axis )
        sf.setSelection(**userSel)

        # some checks
        for i, axis in enumerate(axesToClip[:]):
            if axis not in sf.getAxesNames():
                del axesToClip[i]
                logging.warning('Axis \"'+axis+'\" not found. Ignoring.')

        if sf.getType() != 'amplitude':
            logging.error('CLIP is for "amplitude" tables, not %s.' % sf.getType())
            continue

        sw = solWriter(soltab, useCache=True) # remember to flush()

        before_count=0
        after_count=0
        total=0
        for vals, weights, coord, selection in sf.getValuesIter(returnAxes=axesToClip, weight = True):

            total+=len(vals)
            before_count+=(len(weights)-np.count_nonzero(weights))

            # first find the median and standard deviation
            if (weights == 0).all():
                valmedian = 0
            else:
                if log:
                    valmedian = np.median(np.log10(vals[(weights != 0)]))
                    rms = np.std(np.log10(vals[(weights != 0)]))
                    np.putmask(weights, np.abs(np.log10(vals)-valmedian) > rms * clipLevel, 0)
                else:
                    valmedian = np.median(vals[(weights != 0)])
                    rms = np.std(vals[(weights != 0)])
                    np.putmask(weights, np.abs(vals-valmedian) > rms * clipLevel, 0)
        
            after_count+=(len(weights)-np.count_nonzero(weights))

            # writing back the solutions
            sw.selection = selection
            sw.setValues(weights, weight=True)

        sw.addHistory('CLIP (over %s with %s sigma cut)' % (axesToClip, clipLevel))
        logging.info('Clip, flagged data: %f %% -> %f %%' \
                % (100.*before_count/total, 100.*after_count/total))

        sw.flush()
        
    return 0
Example #18
0
        if st._v_title == 'tecscreen':
            st_tec = st
    if st_tec is not None:
        solTypes.append('TECScreen')
    solTypes = list(set(solTypes))
    logging.info('Found solution types in input parmdb and H5parm: ' +
                 ', '.join(solTypes))

    # For each solType, select appropriate solutions and construct
    # the dictionary to pass to pdb.addValues()
    len_sol = {}
    for solType in solTypes:
        if solType != 'TECScreen':
            len_sol[solType] = len(pdb.getNames(solType + ':*'))
        else:
            tec_sf = solFetcher(st_tec)
            N_times = tec_sf.getAxisLen(axis='time')
            len_sol[solType] = N_times

    cachedSolTabs = {}
    for instrumentdbFile in instrumentdbFiles:
        out_instrumentdbFile = out_globaldbFile + '/' + outroot + '_' + instrumentdbFile.split(
            '/')[-1]
        logging.info('Filling ' + out_instrumentdbFile + ':')

        # Remove existing instrumentdb (if clobber) and create new one
        if os.path.exists(out_instrumentdbFile):
            if options.clobber:
                shutil.rmtree(out_instrumentdbFile)
            else:
                logging.critical('Output instrumentdb file exists and '
Example #19
0
opt = optparse.OptionParser(usage='%prog -p H5parm [-s solset] -g parmdb [-n 100]\n'\
                +_author, version='%prog '+_version.__version__)
opt.add_option('-p', '--h5parm', help='H5parm name', type='string', default='')
opt.add_option('-g', '--parmdb', help='Parmdb name', type='string', default='')
opt.add_option('-s', '--solset', help='Solution-set name (default=sol000)', type='string', default='sol000')
opt.add_option('-n', '--numiter', help='Number of iterations (default=100)', type=int, default=100)
(options, args) = opt.parse_args()

_logging.setLevel('debug')

n = options.numiter

solset = options.solset
h5parmFile = options.h5parm
H5 = h5parm(h5parmFile, readonly=False)
H = solFetcher(H5.getSoltab(solset,'rotation000'))
logging.info("H5parm filename = "+h5parmFile)

parmdbFile = options.parmdb
P = lofar.parmdb.parmdb(parmdbFile)
P2 = lofar.parmdb.parmdb('tmp.parmdb', create=True)
logging.info("parmdb filename = "+parmdbFile)

######################################################
logging.info("### Read all frequencies for a pol/dir/station")

start = time.clock()
for i in xrange(n):
    Pfreqs = P.getValuesGrid('RotationAngle:CS001LBA:3C196')['RotationAngle:CS001LBA:3C196']['freqs']
elapsed = (time.clock() - start)
logging.info("PARMDB -- "+str(elapsed)+" s.")
Example #20
0
def run(step, parset, H):
    """
    Fits a screen to TEC values derived by the TECFIT operation.

    The TEC values are read from the specified tec soltab.

    The results of the fit are stored in the specified tecscreen solution table.
    These values are the screen TEC values per station per pierce point per
    solution interval. The pierce point locations are stored in an auxiliary
    array in the output solution table.

    TEC screens can be plotted with the PLOT operation by setting PlotType =
    TECScreen.

    The H5parm_exporter.py tool can be used to export the screen to a parmdb
    that BBS and the AWimager can use. Note, however, that the output screens
    are not normalized properly (any normalization was lost due to the use of
    source-to-source phase gradients in the TECFIT operation). Therefore, a
    direction-independent calibration must be done after exporting the screens
    to a parmdb file, with the following settings in the BBS solve step:

        Model.Ionosphere.Enable = T
        Model.Ionosphere.Type = EXPION
    """
    import numpy as np
    import re
    from losoto.h5parm import solFetcher, solWriter
    # Switch to the Agg backend to prevent problems with pylab imports when
    # DISPLAY env. variable is not set
    import os
    if 'DISPLAY' not in os.environ:
        import matplotlib
        matplotlib.use("Agg")

    soltabs = getParSoltabs(step, parset, H)
    outSoltabs = parset.getStringVector(
        '.'.join(["LoSoTo.Steps", step, "OutSoltab"]), [])
    height = np.array(
        parset.getDoubleVector('.'.join(["LoSoTo.Steps", step, "Height"]),
                               [200e3]))
    order = int(
        parset.getString('.'.join(["LoSoTo.Steps", step, "Order"]), '15'))

    # Load TEC values from TECFIT operation
    indx = 0
    for soltab in openSoltabs(H, soltabs):
        if 'tec' not in soltab._v_title:
            logging.warning(
                'No TECFIT solution tables found for solution table '
                '{0}'.format(soltabs[indx]))
            continue
            indx += 1
        solset = soltabs[indx].split('/')[0]
        logging.info('Using input solution table: {0}'.format(soltabs[indx]))
        logging.info('Using output solution table: {0}'.format(
            outSoltabs[indx]))

        # Collect station and source names and positions and times, making sure
        # that they are ordered correctly.
        t = solFetcher(soltab)
        r, axis_vals = t.getValues()
        source_names = axis_vals['dir']
        source_dict = H.getSou(solset)
        source_positions = []
        for source in source_names:
            source_positions.append(source_dict[source])
        station_names = axis_vals['ant']
        station_dict = H.getAnt(solset)
        station_positions = []
        for station in station_names:
            station_positions.append(station_dict[station])
        times = axis_vals['time']

        # Get sizes
        N_sources = len(source_names)
        N_times = len(times)
        N_stations = len(station_names)
        N_piercepoints = N_sources * N_stations
        rr = np.reshape(r.transpose([0, 2, 1]), [N_piercepoints, N_times])

        heights = list(set(np.linspace(height[0], height[-1], 5)))
        heights.sort()
        if len(heights) > 1:
            logging.info('Trying range of heights: {0} m'.format(heights))
        for i, height in enumerate(heights):
            # Find pierce points and airmass values for given screen height
            logging.info('Using height = {0} m and order = {1}'.format(
                height, order))
            if height < 100e3:
                logging.warning("Height is less than 100e3 m.")
            pp, airmass = calculate_piercepoints(np.array(station_positions),
                                                 np.array(source_positions),
                                                 np.array(times), height)

            # Fit a TEC screen
            r_0 = 10e3
            beta = 5.0 / 3.0
            tec_screen, residual = fit_screen_to_tec(station_names,
                                                     source_names, pp, airmass,
                                                     rr, times, height, order,
                                                     r_0, beta)
            total_resid = np.sum(np.abs(residual))
            if i > 0:
                if total_resid < best_resid:
                    tec_screen_best = tec_screen
                    pp_best = pp
                    height_best = height
                    best_resid = total_resid
            else:
                tec_screen_best = tec_screen
                pp_best = pp
                height_best = height
                best_resid = total_resid
            if len(heights) > 1:
                logging.info(
                    'Total residual for fit: {0}\n'.format(total_resid))

        # Use screen with lowest total residual
        if len(heights) > 1:
            tec_screen = tec_screen_best
            pp = pp_best
            height = height_best
            logging.info(
                'Using height (with lowest total residual) of {0} m'.format(
                    height))

        # Write the results to the output solset
        dirs_out = source_names
        times_out = times
        ants_out = station_names

        # Make output tecscreen table
        outSolset = outSoltabs[indx].split('/')[0]
        outSoltab = outSoltabs[indx].split('/')[1]
        if not outSolset in H.getSolsets().keys():
            solsetTEC = H.makeSolset(outSolset)
            dirs_pos = source_positions
            sourceTable = solsetTEC._f_get_child('source')
            sourceTable.append(zip(*(dirs_out, dirs_pos)))
            ants_pos = station_positions
            antennaTable = solsetTEC._f_get_child('antenna')
            antennaTable.append(zip(*(ants_out, ants_pos)))

        # Store tecscreen values. The residual values are stored in the weights
        # table. Flagged values of the screen have weights set to 0.0.
        vals = tec_screen.transpose([1, 0, 2])
        weights = residual.transpose([1, 0, 2])
        tec_screen_st = H.makeSoltab(outSolset,
                                     'tecscreen',
                                     outSoltab,
                                     axesNames=['dir', 'time', 'ant'],
                                     axesVals=[dirs_out, times_out, ants_out],
                                     vals=vals,
                                     weights=weights)

        # Store beta, r_0, height, and order as attributes of the tecscreen
        # soltab
        tec_screen_st._v_attrs['beta'] = beta
        tec_screen_st._v_attrs['r_0'] = r_0
        tec_screen_st._v_attrs['height'] = height
        tec_screen_st._v_attrs['order'] = order

        # Make output piercepoint table
        tec_screen_solset = tec_screen_st._v_parent._v_name
        H.H.create_carray('/' + tec_screen_solset + '/' +
                          tec_screen_st._v_name,
                          'piercepoint',
                          obj=pp)

        # Add histories
        sw = solWriter(tec_screen_st)
        sw.addHistory('CREATE (by TECSCREEN operation)')
        indx += 1

    return 0
Example #21
0
def run( step, parset, H ):
    """
    Separate phase solutions into FR, Clock and TEC.

    The Clock and TEC values are stored in the specified output soltab with type 'clock', 'tec', 'FR'.
    """
    from losoto.h5parm import solFetcher, solWriter
    import numpy as np
    import scipy.optimize

    rmwavcomplex = lambda RM, wav, y: abs(np.cos(2.*RM[0]*wav*wav)  - np.cos(y)) + abs(np.sin(2.*RM[0]*wav*wav)  - np.sin(y))
    c = 2.99792458e8

    # get involved solsets using local step values or global values or all
    soltabs = getParSoltabs( step, parset, H )

    refAnt = parset.getString('.'.join(["LoSoTo.Steps", step, "RefAnt"]), '' )
    ncpu = parset.getInt('.'.join(["LoSoTo.Ncpu"]), 1 )

    for t, soltab in enumerate(openSoltabs( H, soltabs )):
        logging.info("--> Working on soltab: "+soltab._v_name)
        sf = solFetcher(soltab)

        # times and ants needs to be complete or selection is much slower
        times = sf.getAxisValues('time')
        ants = sf.getAxisValues('ant')

        # this will make a selection for the getValues() and getValuesIter()
        userSel = {}
        for axis in sf.getAxesNames():
            userSel[axis] = getParAxis( step, parset, H, axis )
        sf.setSelection(**userSel)

        # some checks
        solType = sf.getType()
        if solType != 'phase':
           logging.warning("Soltab type of "+soltab._v_name+" is of type "+solType+", should be phase. Ignoring.")
           continue

        if refAnt != '' and not refAnt in ants:
            logging.error('Reference antenna '+refAnt+' not found.')
            return 1
        if refAnt == '': refAnt = ants[0]

        solsetname = soltabs[t].split('/')[0]
        st = H.makeSoltab(solsetname, 'rotationmeasure',
                                 axesNames=['ant','time'], axesVals=[ants, times],
                                 vals=np.zeros((len(ants),len(times))),
                                 weights=np.ones((len(ants),len(times))))
        sw = solWriter(st)
        sw.addHistory('Created by FARADAY operation.')
            
        for vals, weights, coord, selection in sf.getValuesIter(returnAxes=['freq','pol','time'], weight=True, reference = refAnt):

            if len(coord['freq']) < 10:
                logging.error('Faraday rotation estimation needs at least 10 frequency channels, preferably distributed over a wide range.')
                return 1

            fitrm = np.zeros(len(times))
            fitweights = np.ones(len(times))
            fitrmguess = 0 # good guess

            if 'RR' in coord['pol'] and 'LL' in coord['pol']:
                coord_rr = np.where(coord['pol'] == 'RR')[0][0]
                coord_ll = np.where(coord['pol'] == 'LL')[0][0]
            elif 'XX' in coord['pol'] and 'YY' in coord['pol']:
                logging.warning('Linear polarization detected, LoSoTo assumes XX->RR and YY->LL.')
                coord_rr = np.where(coord['pol'] == 'XX')[0][0]
                coord_ll = np.where(coord['pol'] == 'YY')[0][0]
            else:
                logging.error("Cannot proceed with Faraday estimation with polarizations: "+str(coord['pol']))
                return 1

            if not coord['ant'] == refAnt:
                logging.debug('Working on ant: '+coord['ant']+'...')

                for t, time in enumerate(times):

                    # apply flags
                    idx       = ((weights[0,:,t] != 0.) & (weights[1,:,t] != 0.))
                    freq      = np.copy(coord['freq'])[idx]
                    phase_rr  = vals[coord_rr,:,t][idx]
                    phase_ll  = vals[coord_ll,:,t][idx]

                    if (len(weights[0,:,t]) - len(idx))/len(weights[0,:,t]) > 1/4.:
                        logging.debug('High number of filtered out data points for the timeslot '+str(t)+': '+str(len(weights[0,:,t]) - len(idx)))
        
                    if len(freq) < 10:
                        fitweights[t] = 0
                        logging.warning('No valid data found for Faraday fitting for antenna: '+coord['ant'])
                        continue
        
                    # RR-LL to be consistent with BBS/NDPPP
                    phase_diff  = (phase_rr - phase_ll)      # not divide by 2 otherwise jump problem, then later fix this
                    wav = c/freq
    
                    fitresultrm_wav, success = scipy.optimize.leastsq(rmwavcomplex, [fitrmguess], args=(wav, phase_diff))
                    # fractional residual
                    residual = np.mean(np.abs(np.mod((2.*fitresultrm_wav*wav*wav)-phase_diff,2.*np.pi) - np.pi))

#                    print "t:", t, "result:", fitresultrm_wav, "residual:", residual

                    if residual > 0.5:
                        fitrmguess = fitresultrm_wav[0]
                        weight = 1
                    else:       
                        # high residual, flag
                        logging.warning('Bad solution for ant: '+coord['ant']+' (time: '+str(t)+', resdiaul: '+str(residual)+').')
                        weight = 0

                    # Debug plot
                    doplot = False
                    if doplot and coord['ant'] == 'RS310LBA' and t%10==0:
                        print "Plotting"
                        if not 'matplotlib' in sys.modules:
                            import matplotlib as mpl
                            mpl.rc('font',size =8 )
                            mpl.rc('figure.subplot',left=0.05, bottom=0.05, right=0.95, top=0.95,wspace=0.22, hspace=0.22 )
                            mpl.use("Agg")
                        import matplotlib.pyplot as plt

                        fig = plt.figure()
                        fig.subplots_adjust(wspace=0)
                        ax = fig.add_subplot(110)

                        # plot rm fit
                        plotrm = lambda RM, wav: np.mod( (2.*RM*wav*wav) + np.pi, 2.*np.pi) - np.pi # notice the factor of 2
                        ax.plot(freq, plotrm(fitresultrm_wav, c/freq[:]), "-", color='purple')

                        ax.plot(freq, np.mod(phase_rr + np.pi, 2.*np.pi) - np.pi, 'ob' )
                        ax.plot(freq, np.mod(phase_ll + np.pi, 2.*np.pi) - np.pi, 'og' )
                        ax.plot(freq, np.mod(phase_diff + np.pi, 2.*np.pi) - np.pi , '.', color='purple' )                           
     
                        residual = np.mod(plotrm(fitresultrm_wav, c/freq[:])-phase_diff+np.pi,2.*np.pi)-np.pi
                        ax.plot(freq, residual, '.', color='yellow')
        
                        ax.set_xlabel('freq')
                        ax.set_ylabel('phase')
                        ax.set_ylim(ymin=-np.pi, ymax=np.pi)
    
                        logging.warning('Save pic: '+str(t)+'_'+coord['ant']+'.png')
                        plt.savefig(str(t)+'_'+coord['ant']+'.png', bbox_inches='tight')
                        del fig

                    fitrm[t] = fitresultrm_wav[0]
                    fitweights[t] = weight

            sw.setSelection(ant=coord['ant'], time=coord['time'])
            sw.setValues( np.expand_dims(fitrm, axis=1) )
            sw.setValues( np.expand_dims(fitweights, axis=1), weight=True )

        del st
        del sw        
        del sf
    return 0
Example #22
0
def run(step, parset, H):
    """
    Separate phase solutions into Clock and TEC.

    Phase solutions are assumed to be stored in solsets of the H5parm file, one
    solset per field.

    The Clock and TEC values are stored in the specified output soltab with type 'clock' and 'tec'.

    """
    import numpy as np
    from losoto.h5parm import solFetcher, solWriter

    # get involved solsets using local step values or global values or all
    soltabs = getParSoltabs(step, parset, H)

    flagBadChannels = parset.getBool(
        '.'.join(["LoSoTo.Steps", step, "FlagBadChannels"]), True)
    flagCut = parset.getFloat('.'.join(["LoSoTo.Steps", step, "FlagCut"]), 5.)
    chi2cut = parset.getFloat('.'.join(["LoSoTo.Steps", step, "Chi2cut"]),
                              3000.)
    combinePol = parset.getBool('.'.join(["LoSoTo.Steps", step, "CombinePol"]),
                                False)
    #fitOffset = parset.getBool('.'.join(["LoSoTo.Steps", step, "FitOffset"]), False )
    removePhaseWraps = parset.getBool(
        '.'.join(["LoSoTo.Steps", step, "RemovePhaseWraps"]), True)
    fit3rdorder = parset.getBool(
        '.'.join(["LoSoTo.Steps", step, "Fit3rdOrder"]), False)
    circular = parset.getBool('.'.join(["LoSoTo.Steps", step, "Circular"]),
                              False)
    reverse = parset.getBool('.'.join(["LoSoTo.Steps", step, "Reverse"]),
                             False)

    # do something on every soltab (use the openSoltab LoSoTo function)
    #for soltab in openSoltabs( H, soltabs ):
    for soltabname in soltabs:
        solsetname = soltabname.split('/')[0]
        soltab = H.getSoltab(solset=solsetname,
                             soltab=soltabname.split('/')[1])
        logging.info("--> Working on soltab: " + soltab._v_name)
        t = solFetcher(soltab)
        tw = solWriter(soltab)

        # some checks
        solType = t.getType()
        if solType != 'phase':
            logging.warning("Soltab type of " + soltab._v_name + " is: " +
                            solType + " should be phase. Ignoring.")
            continue

        # this will make a selection for the getValues() and getValuesIter()
        userSel = {}
        for axis in t.getAxesNames():
            userSel[axis] = getParAxis(step, parset, H, axis)
        t.setSelection(**userSel)

        # Collect station properties
        station_dict = H.getAnt(solsetname)
        stations = t.getAxisValues('ant')
        station_positions = np.zeros((len(stations), 3), dtype=np.float)
        for i, station_name in enumerate(stations):
            station_positions[i, 0] = station_dict[station_name][0]
            station_positions[i, 1] = station_dict[station_name][1]
            station_positions[i, 2] = station_dict[station_name][2]

        returnAxes = ['ant', 'freq', 'pol', 'time']
        for vals, flags, coord, selection in t.getValuesIter(
                returnAxes=returnAxes, weight=True):

            if len(coord['ant']) < 2:
                logging.error(
                    'Clock/TEC separation needs at least 2 antennas selected.')
                return 1
            if len(coord['freq']) < 10:
                logging.error(
                    'Clock/TEC separation needs at least 10 frequency channels, preferably distributed over a wide range'
                )
                return 1

            freqs = coord['freq']
            stations = coord['ant']
            times = coord['time']

            # get axes index
            axes = [i for i in t.getAxesNames() if i in returnAxes]

            # reverse time axes
            if reverse:
                vals = np.swapaxes(
                    np.swapaxes(vals, 0, axes.index('time'))[::-1], 0,
                    axes.index('time'))
                flags = np.swapaxes(
                    np.swapaxes(flags, 0, axes.index('time'))[::-1], 0,
                    axes.index('time'))

            result=doFit(vals,flags==0,freqs,stations,station_positions,axes,\
                             flagBadChannels=flagBadChannels,flagcut=flagCut,chi2cut=chi2cut,combine_pol=combinePol,removePhaseWraps=removePhaseWraps,fit3rdorder=fit3rdorder,circular=circular)
            if fit3rdorder:
                clock, tec, offset, tec3rd = result
                if reverse:
                    clock = clock[::-1, :]
                    tec = tec[::-1, :]
                    tec3rd = tec3rd[::-1, :]
            else:
                clock, tec, offset = result
                if reverse:
                    clock = clock[::-1, :]
                    tec = tec[::-1, :]

            weights = tec > -5
            tec[np.logical_not(weights)] = 0
            clock[np.logical_not(weights)] = 0
            weights = np.float16(weights)

            if combinePol:
                tf_st = H.makeSoltab(solsetname,
                                     'tec',
                                     axesNames=['time', 'ant'],
                                     axesVals=[times, stations],
                                     vals=tec[:, :, 0],
                                     weights=weights[:, :, 0])
                sw = solWriter(tf_st)
                sw.addHistory('CREATE (by CLOCKTECFIT operation)')
                tf_st = H.makeSoltab(solsetname,
                                     'clock',
                                     axesNames=['time', 'ant'],
                                     axesVals=[times, stations],
                                     vals=clock[:, :, 0] * 1e-9,
                                     weights=weights[:, :, 0])
                sw = solWriter(tf_st)
                sw.addHistory('CREATE (by CLOCKTECFIT operation)')
                tf_st = H.makeSoltab(solsetname,
                                     'phase_offset',
                                     axesNames=['ant'],
                                     axesVals=[stations],
                                     vals=offset[:, 0],
                                     weights=np.ones_like(offset[:, 0],
                                                          dtype=np.float16))
                sw = solWriter(tf_st)
                sw.addHistory('CREATE (by CLOCKTECFIT operation)')
                if fit3rdorder:
                    tf_st = H.makeSoltab(solsetname,
                                         'tec3rd',
                                         axesNames=['time', 'ant'],
                                         axesVals=[times, stations],
                                         vals=tec3rd[:, :, 0],
                                         weights=weights[:, :, 0])
                    sw = solWriter(tf_st)
            else:
                tf_st = H.makeSoltab(solsetname,
                                     'tec',
                                     axesNames=['time', 'ant', 'pol'],
                                     axesVals=[times, stations, ['XX', 'YY']],
                                     vals=tec,
                                     weights=weights)
                sw = solWriter(tf_st)
                sw.addHistory('CREATE (by CLOCKTECFIT operation)')
                tf_st = H.makeSoltab(solsetname,
                                     'clock',
                                     axesNames=['time', 'ant', 'pol'],
                                     axesVals=[times, stations, ['XX', 'YY']],
                                     vals=clock * 1e-9,
                                     weights=weights)
                sw = solWriter(tf_st)
                sw.addHistory('CREATE (by CLOCKTECFIT operation)')
                tf_st = H.makeSoltab(solsetname,
                                     'phase_offset',
                                     axesNames=['ant', 'pol'],
                                     axesVals=[stations, ['XX', 'YY']],
                                     vals=offset,
                                     weights=np.ones_like(offset,
                                                          dtype=np.float16))
                sw = solWriter(tf_st)
                sw.addHistory('CREATE (by CLOCKTECFIT operation)')
                if fit3rdorder:
                    tf_st = H.makeSoltab(
                        solsetname,
                        'tec3rd',
                        axesNames=['time', 'ant', 'pol'],
                        axesVals=[times, stations, ['XX', 'YY']],
                        vals=tec3rd,
                        weights=weights)
                    sw = solWriter(tf_st)
    return 0
Example #23
0
def  makeTECparmdb(H, solset, TECsolTab, PPsolTab, timewidths, freq, freqwidth):
    """Returns TEC screen parmdb parameters

    H - H5parm object
    solset - solution set with TEC screen parameters
    TECsolTab = solution table with tecfitwhite values
    PPsolTab = solution table with piercepoint values
    timewidths - time widths of output parmdb
    freq - frequency of output parmdb
    freqwidth - frequency width of output parmdb
    """
    station_dict = H.getAnt(solset)
    station_names = station_dict.keys()
    station_positions = station_dict.values()
    source_dict = H.getSou(solset)
    source_names = source_dict.keys()
    source_positions = source_dict.values()

    tec_sf = solFetcher(TECsolTab)
    tec_fit_white, axis_vals = tec_sf.getValues()
    times = axis_vals['time']
    beta = TECsolTab._v_attrs['beta']
    r_0 = TECsolTab._v_attrs['r_0']
    height = TECsolTab._v_attrs['height']
    order = TECsolTab._v_attrs['order']
    pp_sf = solFetcher(PPsolTab)
    pp, axis_vals = pp_sf.getValues()

    N_sources = len(source_names)
    N_times = len(times)
    N_freqs = 1
    N_stations = len(station_names)
    N_piercepoints = N_sources * N_stations

    freqs = freq
    freqwidths = freqwidth
    parms = {}
    v = {}
    v['times'] = times
    v['timewidths'] = timewidths
    v['freqs'] = freqs
    v['freqwidths'] = freqwidths

    for station_name in station_names:
        for source_name in source_names:

            v['values'] = np.zeros((N_times, N_freqs), dtype=np.double)
            parmname = 'Piercepoint:X:%s:%s' % (station_name, source_name)
            parms[parmname] = v.copy()

            v['values'] = np.zeros((N_times, N_freqs), dtype=np.double)
            parmname = 'Piercepoint:Y:%s:%s' % (station_name, source_name)
            parms[parmname] = v.copy()

            v['values'] = np.zeros((N_times, N_freqs), dtype=np.double)
            parmname = 'Piercepoint:Z:%s:%s' % (station_name, source_name)
            parms[parmname] = v.copy()

            v['values'] = np.zeros((N_times, N_freqs), dtype=np.double)
            parmname = 'TECfit_white:%s:%s' % (station_name, source_name)
            parms[parmname] = v.copy()

            v['values'] = np.zeros((N_times, N_freqs), dtype=np.double)
            parmname = 'TECfit_white:0:%s:%s' % (station_name, source_name)
            parms[parmname] = v.copy()

            v['values'] = np.zeros((N_times, N_freqs), dtype=np.double)
            parmname = 'TECfit_white:1:%s:%s' % (station_name, source_name)
            parms[parmname] = v.copy()

    for k in range(N_times):
        pp_idx = 0
        for src, source_name in enumerate(source_names):
            for sta, station_name in enumerate(station_names):

                parmname = 'Piercepoint:X:%s:%s' % (station_name, source_name)
                parms[parmname]['values'][k, 0] = pp[k, pp_idx, 0]

                parmname = 'Piercepoint:Y:%s:%s' % (station_name, source_name)
                parms[parmname]['values'][k, 0] = pp[k, pp_idx, 1]

                parmname = 'Piercepoint:Z:%s:%s' % (station_name, source_name)
                parms[parmname]['values'][k, 0] = pp[k, pp_idx, 2]

                parmname = 'TECfit_white:%s:%s' % (station_name, source_name)
                parms[parmname]['values'][k, 0] = tec_fit_white[src, k, sta]

                parmname = 'TECfit_white:0:%s:%s' % (station_name, source_name)
                parms[parmname]['values'][k, 0] = tec_fit_white[src, k, sta]

                parmname = 'TECfit_white:1:%s:%s' % (station_name, source_name)
                parms[parmname]['values'][k, 0] = tec_fit_white[src, k, sta]

                pp_idx += 1


    time_start = times[0] - timewidths[0]/2
    time_end = times[-1] + timewidths[-1]/2

    v['times'] = np.array([(time_start + time_end) / 2])
    v['timewidths'] = np.array([time_end - time_start])

    v_r0 = v.copy()
    v_r0['values'] = np.array(r_0, dtype=np.double, ndmin=2)
    parms['r_0'] = v_r0

    v_beta = v.copy()
    v_beta['values'] = np.array(beta, dtype=np.double, ndmin=2)
    parms['beta'] = v_beta

    v_height = v.copy()
    v_height['values'] = np.array(height, dtype=np.double, ndmin=2)
    parms['height'] = v_height

    return parms
Example #24
0
def run( step, parset, H ):
    """
    Separate phase solutions into Clock and TEC.

    Phase solutions are assumed to be stored in solsets of the H5parm file, one
    solset per field.

    The Clock and TEC values are stored in the specified output soltab with type 'clock' and 'tec'.

    """
    import numpy as np
    from losoto.h5parm import solFetcher, solWriter

    # get involved solsets using local step values or global values or all
    soltabs = getParSoltabs( step, parset, H )

    flagBadChannels = parset.getBool('.'.join(["LoSoTo.Steps", step, "FlagBadChannels"]), True )
    flagCut = parset.getFloat('.'.join(["LoSoTo.Steps", step, "FlagCut"]), 5. )
    chi2cut = parset.getFloat('.'.join(["LoSoTo.Steps", step, "Chi2cut"]), 3000. )
    combinePol = parset.getBool('.'.join(["LoSoTo.Steps", step, "CombinePol"]), False )
    #fitOffset = parset.getBool('.'.join(["LoSoTo.Steps", step, "FitOffset"]), False )
    removePhaseWraps=parset.getBool('.'.join(["LoSoTo.Steps", step, "RemovePhaseWraps"]), True )
    fit3rdorder=parset.getBool('.'.join(["LoSoTo.Steps", step, "Fit3rdOrder"]), False )
    circular=parset.getBool('.'.join(["LoSoTo.Steps", step, "Circular"]), False )
    reverse=parset.getBool('.'.join(["LoSoTo.Steps", step, "Reverse"]), False )

    # do something on every soltab (use the openSoltab LoSoTo function)
    #for soltab in openSoltabs( H, soltabs ):
    for soltabname in soltabs:
        solsetname=soltabname.split('/')[0]
        soltab=H.getSoltab(solset=solsetname, soltab=soltabname.split('/')[1])
        logging.info("--> Working on soltab: "+soltab._v_name)
        t = solFetcher(soltab)
        tw = solWriter(soltab)

        # some checks
        solType = t.getType()
        if solType != 'phase':
           logging.warning("Soltab type of "+soltab._v_name+" is: "+solType+" should be phase. Ignoring.")
           continue

        # this will make a selection for the getValues() and getValuesIter()
        userSel = {}
        for axis in t.getAxesNames():
            userSel[axis] = getParAxis( step, parset, H, axis )
        t.setSelection(**userSel)

        # Collect station properties
        station_dict = H.getAnt(solsetname)
        stations = t.getAxisValues('ant')
        station_positions = np.zeros((len(stations), 3), dtype=np.float)
        for i, station_name in enumerate(stations):
            station_positions[i, 0] = station_dict[station_name][0]
            station_positions[i, 1] = station_dict[station_name][1]
            station_positions[i, 2] = station_dict[station_name][2]
            
        returnAxes=['ant','freq','pol','time']
        for vals, flags, coord, selection in t.getValuesIter(returnAxes=returnAxes,weight=True):

            if len(coord['ant']) < 2:
                logging.error('Clock/TEC separation needs at least 2 antennas selected.')
                return 1
            if len(coord['freq']) < 10:
                logging.error('Clock/TEC separation needs at least 10 frequency channels, preferably distributed over a wide range')
                return 1

            freqs=coord['freq']
            stations=coord['ant']
            times=coord['time']

            # get axes index
            axes=[i for i in t.getAxesNames() if i in returnAxes]

            # reverse time axes
            if reverse: 
                vals = np.swapaxes(np.swapaxes(vals, 0, axes.index('time'))[::-1], 0, axes.index('time'))
                flags = np.swapaxes(np.swapaxes(flags, 0, axes.index('time'))[::-1], 0, axes.index('time'))

            result=doFit(vals,flags==0,freqs,stations,station_positions,axes,\
                             flagBadChannels=flagBadChannels,flagcut=flagCut,chi2cut=chi2cut,combine_pol=combinePol,removePhaseWraps=removePhaseWraps,fit3rdorder=fit3rdorder,circular=circular)
            if fit3rdorder:
                clock,tec,offset,tec3rd=result
                if reverse: 
                    clock = clock[::-1,:]
                    tec = tec[::-1,:]
                    tec3rd = tec3rd[::-1,:]
            else:
                clock,tec,offset=result
                if reverse: 
                    clock = clock[::-1,:]
                    tec = tec[::-1,:]

            weights=tec>-5
            tec[np.logical_not(weights)]=0
            clock[np.logical_not(weights)]=0
            weights=np.float16(weights)

            if combinePol:
                tf_st = H.makeSoltab(solsetname, 'tec',
                                 axesNames=['time', 'ant'], axesVals=[times, stations],
                                 vals=tec[:,:,0],
                                 weights=weights[:,:,0])
                sw = solWriter(tf_st)
                sw.addHistory('CREATE (by CLOCKTECFIT operation)')
                tf_st = H.makeSoltab(solsetname, 'clock',
                                 axesNames=['time', 'ant'], axesVals=[times, stations],
                                 vals=clock[:,:,0]*1e-9,
                                 weights=weights[:,:,0])
                sw = solWriter(tf_st)
                sw.addHistory('CREATE (by CLOCKTECFIT operation)')
                tf_st = H.makeSoltab(solsetname, 'phase_offset',
                                 axesNames=['ant'], axesVals=[stations],
                                 vals=offset[:,0],
                                 weights=np.ones_like(offset[:,0],dtype=np.float16))
                sw = solWriter(tf_st)
                sw.addHistory('CREATE (by CLOCKTECFIT operation)')
                if fit3rdorder:
                    tf_st = H.makeSoltab(solsetname, 'tec3rd',
                                         axesNames=['time', 'ant'], axesVals=[times, stations],
                                         vals=tec3rd[:,:,0],
                                         weights=weights[:,:,0])
                    sw = solWriter(tf_st)
            else:
                tf_st = H.makeSoltab(solsetname, 'tec',
                                 axesNames=['time', 'ant','pol'], axesVals=[times, stations, ['XX','YY']],
                                 vals=tec,
                                 weights=weights)
                sw = solWriter(tf_st)
                sw.addHistory('CREATE (by CLOCKTECFIT operation)')
                tf_st = H.makeSoltab(solsetname, 'clock',
                                 axesNames=['time', 'ant','pol'], axesVals=[times, stations, ['XX','YY']],
                                 vals=clock*1e-9,
                                 weights=weights)
                sw = solWriter(tf_st)
                sw.addHistory('CREATE (by CLOCKTECFIT operation)')
                tf_st = H.makeSoltab(solsetname, 'phase_offset',
                                 axesNames=['ant','pol'], axesVals=[stations, ['XX','YY']],
                                 vals=offset,
                                 weights=np.ones_like(offset,dtype=np.float16))
                sw = solWriter(tf_st)
                sw.addHistory('CREATE (by CLOCKTECFIT operation)')
                if fit3rdorder:
                    tf_st = H.makeSoltab(solsetname, 'tec3rd',
                                         axesNames=['time', 'ant','pol'], axesVals=[times, stations, ['XX','YY']],
                                         vals=tec3rd,
                                         weights=weights)
                    sw = solWriter(tf_st)
    return 0
Example #25
0
def collect_solutions(H, dirs=None, freq_tol=1e6, solsets=None):
    """
    Collects and returns phase solutions, etc. needed for fitting

    Keyword arguments:
    H -- H5parm object
    dirs -- list of directions to use (None => all)
    freq_tol -- tolerance for grouping of phase solutions into bands (Hz)
    solsets -- list of solution sets in H to search (None => all)
    """
    import numpy as np
    from pylab import find
    import re
    try:
        import progressbar
    except ImportError:
        import losoto.progressbar as progressbar
    logging.info("Scanning for solutions needed for TEC fitting...")

    # Determine axis lengths
    sources = []
    freqs = []
    stations = []
    if solsets is None:
        solsets = H.getSolsets().keys()
    N_times_max = 0
    first_solset = None
    for solset in solsets[:]:
        logging.info('  --Scanning solution set {0}...'.format(solset))
        has_phase_st = False
        soltabs = H.getSoltabs(solset)

        for soltab in soltabs:
            # Only use soltabs with phase solutions
            if 'phase' in soltabs[soltab]._v_title:
                logging.info(
                    '    --Scanning solution table {0}...'.format(soltab))
                has_phase_st = True
                solset_sources = H.getSou(solset)
                if len(solset_sources) == 1 and 'pointing' in solset_sources:
                    logging.info('      Found direction-independent solutions')
                    dir_indep = True
                    soln_type_dirindep = soltabs[soltab]._v_title
                    if first_solset is None:
                        first_solset = solset
                else:
                    logging.info('      Found direction-dependent solutions')
                    dir_indep = False
                    if first_solset is None:
                        first_solset = solset
                if not dir_indep:
                    soln_type_dirdep = soltabs[soltab]._v_title
                    logging.info('      Found sources: {0}'.format(
                        H.getSou(solset).keys()))
                    sources += H.getSou(solset).keys()
                    stations += H.getAnt(solset).keys()
                    t = solFetcher(soltabs[soltab])
                    solns, axis_vals = t.getValues()
                    times = axis_vals['time']
                    N_times = len(times)
                    freqs.append(axis_vals['freq'][0])
                    if N_times > N_times_max:
                        N_times_max = N_times
                        times_max = times
        if not has_phase_st:
            logging.info('    --No phase solutions found')
            solsets.remove(solset)

    # Consolidate frequencies into bands (defined by freq_tol parameter)
    has_dup = True
    while has_dup:
        has_dup = False
        for freq in freqs[:]:
            nfreq = len(
                find((np.array(freqs) > (freq - freq_tol))
                     & (np.array(freqs) < (freq + freq_tol))))
            if nfreq > 1:
                has_dup = True
                freqs.remove(freq)
                break
    freqs = np.array(sorted(freqs))
    N_freqs = len(freqs)
    stations_set = set(stations)
    sources_set = set(sources)
    if 'pointing' in sources_set:
        sources_set.remove('pointing')
    if dirs is not None:
        sources_filt = []
        for dir in dirs:
            if dir in sources_set:
                sources_filt.append(dir)
        sources_set = set(sources_filt)
    source_names = np.array(sorted(list(sources_set)))
    N_sources = len(set(source_names))
    N_stations = len(set(stations))
    N_times = N_times_max
    times = times_max
    logging.info('Scanning complete')
    logging.info('  Number of sources: {0}'.format(N_sources))
    logging.info('  Number of stations: {0}'.format(N_stations))
    logging.info('  Number of times: {0}'.format(N_times))
    logging.info('  Number of bands: {0}'.format(N_freqs))
    if N_sources == 0 or N_stations == 0 or N_times == 0 or N_freqs == 0:
        logging.error('No solutions found.')
        return (None, None, None, None, None, None, None, None, None, None,
                None, None)

    # Initialize the arrays
    freqwidths = np.zeros(N_freqs)
    timewidths = np.zeros(N_times)
    m = np.zeros((N_sources, N_freqs))
    phases0 = np.zeros((N_sources, N_stations, N_freqs, N_times))
    phases1 = np.zeros((N_sources, N_stations, N_freqs, N_times))
    flags = np.ones((N_sources, N_stations, N_freqs, N_times))
    source_positions = np.zeros((N_sources, 2))
    solset_array_dir_dep = np.zeros((N_sources, N_freqs), dtype='|S100')
    solset_array_dir_indep = np.zeros(N_freqs, dtype='|S100')

    # Populate the arrays
    for solset in solsets:
        source_dict = H.getSou(solset)
        sources = source_dict.keys()
        soltabs = H.getSoltabs(solset)

        for soltab in soltabs:
            t = solFetcher(soltabs[soltab])
            solns, axes = t.getValues()
            freq = axes['freq'][0]
            freq_indx = find((np.array(freqs) > (freq - freq_tol))
                             & (np.array(freqs) < (freq + freq_tol)))

            for source in sources:
                if source in source_names:
                    source_indx = find(source_names == source)
                    m[source_indx, freq_indx] = 1
                    solset_array_dir_dep[source_indx, freq_indx] = solset
                elif source == 'pointing' and len(sources) == 1:
                    solset_array_dir_indep[freq_indx] = solset

    # Collect station properties and pointing direction
    station_dict = H.getAnt(first_solset)
    station_names = np.array(list(stations_set))
    station_positions = np.zeros((len(station_names), 3), dtype=np.float)
    for i, station_name in enumerate(station_names):
        station_positions[i, 0] = station_dict[station_name][0]
        station_positions[i, 1] = station_dict[station_name][1]
        station_positions[i, 2] = station_dict[station_name][2]

    source_dict = H.getSou(first_solset)
    ra = source_dict['pointing'][0]
    dec = source_dict['pointing'][1]
    pointing = np.array(
        [np.cos(ra) * np.cos(dec),
         np.sin(ra) * np.cos(dec),
         np.sin(dec)])

    # Collect source positions and phase solutions for each band
    logging.info('Collecting phase solutions...')
    pbar = progressbar.ProgressBar(maxval=len(source_names)).start()
    for i, source1 in enumerate(source_names):
        for k in xrange(N_freqs):
            if m[i, k]:
                solset_name = str(solset_array_dir_dep[i, k])
                soltab = H.getSoltab(solset=solset_name,
                                     soltab=soln_type_dirdep + '000')
                solset_dir_indep_name = str(solset_array_dir_indep[k])
                source_dict = H.getSou(solset_name)
                source_positions[i, ] = source_dict[source1]

                if solset_dir_indep_name != '':
                    soltab_dir_indep = H.getSoltab(
                        solset=solset_dir_indep_name,
                        soltab=soln_type_dirindep + '000')
                    sf_dir_indep = solFetcher(soltab_dir_indep)
                else:
                    sf_dir_indep = None
                sf_dir_dep = solFetcher(soltab)

                for l, station in enumerate(station_names):
                    if soln_type_dirdep == 'scalarphase':
                        sf_dir_dep.setSelection(ant=[station], dir=[source1])
                    else:
                        sf_dir_dep.setSelection(ant=[station],
                                                pol=['XX'],
                                                dir=[source1])
                    values_dir_dep = sf_dir_dep.getValues()
                    v1_phase = np.array(values_dir_dep[0]).squeeze()
                    times_dir_dep = values_dir_dep[1]['time']
                    ind = np.where(~np.isnan(v1_phase))
                    v1_phase = v1_phase[ind]
                    times_dir_dep = times_dir_dep[ind]

                    if sf_dir_indep is not None:
                        if soln_type_dirindep == 'scalarphase':
                            sf_dir_indep.setSelection(ant=[station])
                        else:
                            sf_dir_indep.setSelection(ant=[station],
                                                      pol=['XX'])
                        values_dir_indep = sf_dir_indep.getValues()
                        v1_dir_indep = np.array(values_dir_indep[0]).squeeze()
                        times_dir_indep = values_dir_indep[1]['time']
                        ind = np.where(~np.isnan(v1_dir_indep))
                        v1_dir_indep = v1_dir_indep[ind]
                        times_dir_indep = times_dir_indep[ind]
                        v1_dir_indep_interp = interpolate_phase(
                            v1_dir_indep, times_dir_indep, times_dir_dep)
                        v1_phase += v1_dir_indep_interp
                    if len(times_dir_dep) != N_times_max:
                        v1_phase = interpolate_phase(v1_phase, times_dir_dep,
                                                     times_max)
                    phases0[i, l, k, :] = v1_phase

                    if soln_type_dirdep == 'scalarphase':
                        phases1[i, l, k, :] = v1_phase
                    else:
                        sf_dir_dep.setSelection(ant=[station],
                                                pol=['YY'],
                                                dir=[source1])
                        values_dir_dep = sf_dir_dep.getValues()
                        v1_phase = np.array(values_dir_dep[0]).squeeze()
                        times_dir_dep = values_dir_dep[1]['time']
                        ind = np.where(~np.isnan(v1_phase))
                        v1_phase = v1_phase[ind]
                        times_dir_dep = times_dir_dep[ind]

                        if sf_dir_indep is not None:
                            if soln_type_dirindep == 'scalarphase':
                                sf_dir_indep.setSelection(ant=[station])
                            else:
                                sf_dir_indep.setSelection(ant=[station],
                                                          pol=['YY'])
                            values_dir_indep = sf_dir_indep.getValues()
                            v1_dir_indep = np.array(
                                values_dir_indep[0]).squeeze()
                            times_dir_indep = values_dir_indep[1]['time']
                            ind = np.where(~np.isnan(v1_dir_indep))
                            v1_dir_indep = v1_dir_indep[ind]
                            times_dir_indep = times_dir_indep[ind]
                            v1_dir_indep_interp = interpolate_phase(
                                v1_dir_indep, times_dir_indep, times_dir_dep)
                            v1_phase += v1_dir_indep_interp
                        if len(times_dir_dep) != N_times_max:
                            v1_phase = interpolate_phase(
                                v1_phase, times_dir_dep, times_max)
                        phases1[i, l, k, :] = v1_phase

                    if len(times_dir_dep) != N_times_max:
                        # Set all weights to unity if interpolation was required
                        flags[i, l, k, :] = 1.0
                    else:
                        flags[i, l,
                              k, :] = sf_dir_dep.getValues(weight=True,
                                                           retAxesVals=False)

                    if np.all(phases0[i, l, k, :] == 0.0):
                        # Check for flagged stations
                        flags[i, l, k, :] = 0.0
        pbar.update(i)
    pbar.finish()
    logging.info('Collection complete')
    for i, source in enumerate(source_names):
        nbands = len(find(m[i, :]))
        logging.info('  Source {0} has solutions in {1} bands'.format(
            source, nbands))

    # Invert the weights to give flags (0 => unflagged, 1 => flagged)
    zeroflags = np.where(flags == 0.0)
    oneflags = flags.nonzero()
    flags[zeroflags] = 1.0
    flags[oneflags] = 0.0

    return (phases0, phases1, flags, m, station_names, station_positions,
            source_names, source_positions, freqs, times, pointing,
            soln_type_dirdep)
Example #26
0
def run(step, parset, H):
    """
    Interpolate the solutions from one table into a destination table
    """
    import itertools
    import scipy.interpolate
    import numpy as np
    from losoto.h5parm import solFetcher, solWriter

    soltabs = getParSoltabs(step, parset, H)
    solTypes = getParSolTypes(step, parset, H)

    calSoltab = parset.getString('.'.join(["LoSoTo.Steps", step, "CalSoltab"]),
                                 '')
    calDir = parset.getString('.'.join(["LoSoTo.Steps", step, "CalDir"]), '')
    interpAxes = parset.getStringVector(
        '.'.join(["LoSoTo.Steps", step, "InterpAxes"]), ['time', 'freq'])
    interpMethod = parset.getString(
        '.'.join(["LoSoTo.Steps", step, "InterpMethod"]), 'linear')
    medAxis = parset.getString('.'.join(["LoSoTo.Steps", step, "MedAxis"]), '')
    rescale = parset.getBool('.'.join(["LoSoTo.Steps", step, "Rescale"]),
                             False)

    if interpMethod not in ["nearest", "linear", "cubic"]:
        logging.error('Interpolation method must be nearest, linear or cubic.')
        return 1

    if rescale and medAxis == '':
        logging.error('A medAxis is needed for rescaling.')
        return 1

    # open calibration table
    css, cst = calSoltab.split('/')
    cr = solFetcher(H.getSoltab(css, cst))
    cAxesNames = cr.getAxesNames()

    for soltab in openSoltabs(H, soltabs):

        logging.info("Interpolating soltab: " + soltab._v_name)

        tr = solFetcher(soltab)
        tw = solWriter(soltab)

        axesNames = tr.getAxesNames()
        for i, interpAxis in enumerate(interpAxes[:]):
            if interpAxis not in axesNames or interpAxis not in cAxesNames:
                logging.error('Axis ' + interpAxis + ' not found. Ignoring.')
                del interpAxes[i]
        if rescale and (medAxis not in axesNames or medAxis not in cAxesNames):
            logging.error('Axis ' + medAxis + ' not found. Cannot proceed.')
            return 1

        # axis selection
        userSel = {}
        for axis in tr.getAxesNames():
            userSel[axis] = getParAxis(step, parset, H, axis)
        tr.setSelection(**userSel)

        for vals, coord, selection in tr.getValuesIter(returnAxes=interpAxes):

            # construct grid
            coordSel = removeKeys(coord, interpAxes)
            logging.debug("Working on coords:" + str(coordSel))
            # change dir if sepcified
            if calDir != '':
                coordSel['dir'] = calDir
            cr.setSelection(**coordSel)
            calValues, calCoord = cr.getValues()

            # fill medAxis with the median value
            if rescale:
                axis = cAxesNames.index(medAxis)
                calValues = np.repeat(
                    np.expand_dims(np.median(calValues, axis), axis),
                    calValues.shape[axis], axis)

            # create a list of values whose coords are calPoints
            calValues = np.ndarray.flatten(calValues)

            # create calibrator/target coordinates arrays
            calPoints = []
            targetPoints = []
            for interpAxis in interpAxes:
                calPoints.append(calCoord[interpAxis])
                targetPoints.append(coord[interpAxis])
            calPoints = np.array([x for x in itertools.product(*calPoints)])
            targetPoints = np.array(
                [x for x in itertools.product(*targetPoints)])

            # interpolation
            valsNew = scipy.interpolate.griddata(calPoints, calValues,
                                                 targetPoints, interpMethod)

            # fill values outside boudaries with "nearest" solutions
            # NOTE: in 1D is useless due to scipy bug but this is compensated in the next if
            if interpMethod != 'nearest':
                valsNewNearest = scipy.interpolate.griddata(
                    calPoints, calValues, targetPoints, 'nearest')
                # NaN is != from itself
                valsNew[np.isnan(valsNew)] = valsNewNearest[np.isnan(valsNew)]

            # fix bug in Scipy which put NaNs outside the convex hull in 1D for 'nearest'
            if len(np.squeeze(vals).shape) == 1:
                import scipy.cluster.vq as vq
                valsNew = np.squeeze(valsNew)
                code, dist = vq.vq(targetPoints, calPoints)
                # NaN is != from itself
                valsNew[np.isnan(valsNew)] = calValues[code][np.isnan(valsNew)]
                valsNew = valsNew.reshape(vals.shape)

            if rescale:
                # rescale solutions
                axis = interpAxes.index(medAxis)
                valsMed = np.repeat(
                    np.expand_dims(np.median(vals, axis), axis),
                    vals.shape[axis], axis)
                valsNewMed = np.repeat(
                    np.expand_dims(np.median(valsNew, axis), axis),
                    valsNew.shape[axis], axis)
                valsNew = vals * valsNewMed / valsMed
                #print "Rescaling by: ", valsNewMed[:,0]/valsMed[:,0]

            # writing back the solutions
            tw.selection = selection
            tw.setValues(valsNew)

    tw.addHistory('INTERP (from table %s)' % (calSoltab))
    return 0
Example #27
0
    for name, st in solTabs.iteritems():
        if st._v_title == 'tecscreen':
            st_tec = st
    if st_tec is not None:
        solTypes.append('TECScreen')
    solTypes = list(set(solTypes))
    logging.info('Found solution types in input parmdb and H5parm: '+', '.join(solTypes))

    # For each solType, select appropriate solutions and construct
    # the dictionary to pass to pdb.addValues()
    len_sol = {}
    for solType in solTypes:
        if solType != 'TECScreen':
            len_sol[solType] = len(pdb.getNames(solType+':*'))
        else:
            tec_sf = solFetcher(st_tec)
            N_times = tec_sf.getAxisLen(axis='time')
            len_sol[solType] = N_times

    cachedSolTabs = {}
    for instrumentdbFile in instrumentdbFiles:
        out_instrumentdbFile = out_globaldbFile + '/' + outroot + '_' + instrumentdbFile.split('/')[-1]
        logging.info('Filling '+out_instrumentdbFile+':')

        # Remove existing instrumentdb (if clobber) and create new one
        if os.path.exists(out_instrumentdbFile):
            if options.clobber:
                shutil.rmtree(out_instrumentdbFile)
            else:
                logging.critical('Output instrumentdb file exists and '
                    'clobber = False.')
Example #28
0
def run( step, parset, H ):
    """
    subtract a clock and/or tec from a phase.
    """
    import numpy as np
    from losoto.h5parm import solFetcher, solWriter

    soltabs = getParSoltabs( step, parset, H )
    soltabsToSub = parset.getStringVector('.'.join(["LoSoTo.Steps", step, "Sub"]), [] )
    ratio = parset.getBool('.'.join(["LoSoTo.Steps", step, "Ratio"]), False )

    for soltab in openSoltabs( H, soltabs ):
        logging.info("--> Working on soltab: "+soltab._v_name)

        sf = solFetcher(soltab)
        sw = solWriter(soltab, useCache = True)

        sfss = [] # sol fetcher to sub tables
        for soltabToSub in soltabsToSub:
            ss, st = soltabToSub.split('/')
            sfs = solFetcher(H.getSoltab(ss, st))
            if sf.getType() != 'phase' and (sfs.getType() == 'tec' or sfs.getType() == 'clock' or sfs.getType() == 'rotationmeasure' or sfs.getType() == 'tec3rd'):
                logging.warning(soltabToSub+' is of type clock/tec/rm and should be subtracted from a phase. Skipping it.')
                continue
            sfss.append( sfs )
            logging.info('Subtracting table: '+soltabToSub)

            # a major speed up if tables are assumed with same axes, check that (should be the case in almost any case)
            for axisName in sfs.getAxesNames():
                assert all(sfs.getAxisValues(axisName) == sf.getAxisValues(axisName))
        
        if sf.getType() == 'phase' and (sfs.getType() == 'tec' or sfs.getType() == 'clock' or sfs.getType() == 'rotationmeasure' or sfs.getType() == 'tec3rd' ):
            # the only return axes is freq, slower but better code
            for vals, weights, coord, selection in sf.getValuesIter(returnAxes='freq', weight = True):

                for sfs in sfss:

                    # restrict to have the same coordinates of phases
                    for i, axisName in enumerate(sfs.getAxesNames()):
                        sfs.selection[i] = selection[sf.getAxesNames().index(axisName)]

                    valsSub = np.squeeze(sfs.getValues(retAxesVals=False, weight=False))
                    weightsSub = np.squeeze(sfs.getValues(retAxesVals=False, weight=True))

                    if sfs.getType() == 'clock':
                        vals -= 2. * np.pi * valsSub * coord['freq']

                    elif sfs.getType() == 'tec':
                        vals -= -8.44797245e9 * valsSub / coord['freq']

                    elif sfs.getType() == 'tec3rd':
                        vals -= - 1.e21 * valsSub / np.power(coord['freq'],3)

                    elif sfs.getType() == 'rotationmeasure':
                        wav = 2.99792458e8/coord['freq']
                        ph = wav * wav * valsSub
                        if coord['pol'] == 'XX' or coord['pol'] == 'RR':
                            vals -= ph
                        elif coord['pol'] == 'YY' or coord['pol'] == 'LL':
                            vals += ph
                    else:
                        vals -= valsSub

                    # flag data that are contaminated by flagged clock/tec data
                    if weightsSub == 0: weights[:] = 0

                sw.selection = selection
                sw.setValues(vals)
                sw.setValues(weights, weight = True)
        else:
                if ratio: sw.setValues((sf.getValues(retAxesVals=False)-sfs.getValues(retAxesVals=False))/sfs.getValues(retAxesVals=False))
                else: sw.setValues(sf.getValues(retAxesVals=False)-sfs.getValues(retAxesVals=False))
                weight = sf.getValues(retAxesVals=False, weight=True)
                weight[sfs.getValues(retAxesVals=False, weight=True) == 0] = 0
                sw.setValues(weight, weight = True)
            
        sw.addHistory('RESIDUALS by subtracting tables '+' '.join(soltabsToSub))
        sw.flush()
        del sf
        del sw
        
    return 0
Example #29
0
def run( step, parset, H ):
    """
    Fits a screen to TEC values derived by the TECFIT operation.

    The TEC values are read from the specified tec soltab.

    The results of the fit are stored in the specified tecscreen solution table.
    These values are the screen TEC values per station per pierce point per
    solution interval. The pierce point locations are stored in an auxiliary
    array in the output solution table.

    TEC screens can be plotted with the PLOT operation by setting PlotType =
    TECScreen.

    The H5parm_exporter.py tool can be used to export the screen to a parmdb
    that BBS and the AWimager can use. Note, however, that the output screens
    are not normalized properly (any normalization was lost due to the use of
    source-to-source phase gradients in the TECFIT operation). Therefore, a
    direction-independent calibration must be done after exporting the screens
    to a parmdb file, with the following settings in the BBS solve step:

        Model.Ionosphere.Enable = T
        Model.Ionosphere.Type = EXPION
    """
    import numpy as np
    import re
    from losoto.h5parm import solFetcher, solWriter
    # Switch to the Agg backend to prevent problems with pylab imports when
    # DISPLAY env. variable is not set
    import os
    if 'DISPLAY' not in os.environ:
        import matplotlib
        matplotlib.use("Agg")

    soltabs = getParSoltabs( step, parset, H )
    outSoltabs = parset.getStringVector('.'.join(["LoSoTo.Steps", step, "OutSoltab"]), [] )
    height = np.array(parset.getDoubleVector('.'.join(["LoSoTo.Steps", step, "Height"]), [200e3] ))
    order = int(parset.getString('.'.join(["LoSoTo.Steps", step, "Order"]), '15' ))

    # Load TEC values from TECFIT operation
    indx = 0
    for soltab in openSoltabs(H, soltabs):
        if 'tec' not in soltab._v_title:
            logging.warning('No TECFIT solution tables found for solution table '
                '{0}'.format(soltabs[indx]))
            continue
            indx += 1
        solset = soltabs[indx].split('/')[0]
        logging.info('Using input solution table: {0}'.format(soltabs[indx]))
        logging.info('Using output solution table: {0}'.format(outSoltabs[indx]))

        # Collect station and source names and positions and times, making sure
        # that they are ordered correctly.
        t = solFetcher(soltab)
        r, axis_vals = t.getValues()
        source_names = axis_vals['dir']
        source_dict = H.getSou(solset)
        source_positions = []
        for source in source_names:
            source_positions.append(source_dict[source])
        station_names = axis_vals['ant']
        station_dict = H.getAnt(solset)
        station_positions = []
        for station in station_names:
            station_positions.append(station_dict[station])
        times = axis_vals['time']

        # Get sizes
        N_sources = len(source_names)
        N_times = len(times)
        N_stations = len(station_names)
        N_piercepoints = N_sources * N_stations
        rr = np.reshape(r.transpose([0, 2, 1]), [N_piercepoints, N_times])

        heights = list(set(np.linspace(height[0], height[-1], 5)))
        heights.sort()
        if len(heights) > 1:
            logging.info('Trying range of heights: {0} m'.format(heights))
        for i, height in enumerate(heights):
            # Find pierce points and airmass values for given screen height
            logging.info('Using height = {0} m and order = {1}'.format(height, order))
            if height < 100e3:
                logging.warning("Height is less than 100e3 m.")
            pp, airmass = calculate_piercepoints(np.array(station_positions),
                np.array(source_positions), np.array(times), height)

            # Fit a TEC screen
            r_0 = 10e3
            beta = 5.0 / 3.0
            tec_screen, residual = fit_screen_to_tec(station_names, source_names,
                pp, airmass, rr, times, height, order, r_0, beta)
            total_resid = np.sum(np.abs(residual))
            if i > 0:
                if total_resid < best_resid:
                    tec_screen_best = tec_screen
                    pp_best = pp
                    height_best = height
                    best_resid = total_resid
            else:
                tec_screen_best = tec_screen
                pp_best = pp
                height_best = height
                best_resid = total_resid
            if len(heights) > 1:
                logging.info('Total residual for fit: {0}\n'.format(total_resid))

        # Use screen with lowest total residual
        if len(heights) > 1:
            tec_screen = tec_screen_best
            pp = pp_best
            height = height_best
            logging.info('Using height (with lowest total residual) of {0} m'.format(height))

        # Write the results to the output solset
        dirs_out = source_names
        times_out = times
        ants_out = station_names

        # Make output tecscreen table
        outSolset = outSoltabs[indx].split('/')[0]
        outSoltab = outSoltabs[indx].split('/')[1]
        if not outSolset in H.getSolsets().keys():
            solsetTEC = H.makeSolset(outSolset)
            dirs_pos = source_positions
            sourceTable = solsetTEC._f_get_child('source')
            sourceTable.append(zip(*(dirs_out, dirs_pos)))
            ants_pos = station_positions
            antennaTable = solsetTEC._f_get_child('antenna')
            antennaTable.append(zip(*(ants_out, ants_pos)))

        # Store tecscreen values. The residual values are stored in the weights
        # table. Flagged values of the screen have weights set to 0.0.
        vals = tec_screen.transpose([1, 0, 2])
        weights = residual.transpose([1, 0, 2])
        tec_screen_st = H.makeSoltab(outSolset, 'tecscreen', outSoltab,
            axesNames=['dir', 'time', 'ant'], axesVals=[dirs_out, times_out,
            ants_out], vals=vals, weights=weights)

        # Store beta, r_0, height, and order as attributes of the tecscreen
        # soltab
        tec_screen_st._v_attrs['beta'] = beta
        tec_screen_st._v_attrs['r_0'] = r_0
        tec_screen_st._v_attrs['height'] = height
        tec_screen_st._v_attrs['order'] = order

        # Make output piercepoint table
        tec_screen_solset = tec_screen_st._v_parent._v_name
        H.H.create_carray('/'+tec_screen_solset+'/'+tec_screen_st._v_name,
            'piercepoint', obj=pp)

        # Add histories
        sw = solWriter(tec_screen_st)
        sw.addHistory('CREATE (by TECSCREEN operation)')
        indx += 1

    return 0
Example #30
0
def run( step, parset, H ):
    from losoto.h5parm import solFetcher, solWriter
    import numpy as np
    import scipy.optimize

    delaycomplex = lambda d, freq, y: abs(np.cos(d[0]*freq)  - np.cos(y)) + abs(np.sin(d[0]*freq)  - np.sin(y))

    # get involved solsets using local step values or global values or all
    soltabs = getParSoltabs( step, parset, H )

    refAnt = parset.getString('.'.join(["LoSoTo.Steps", step, "RefAnt"]), '' )
    outTab = parset.getString('.'.join(["LoSoTo.Steps", step, "OutTable"]), 'phasediff' )
    maxres = parset.getFloat('.'.join(["LoSoTo.Steps", step, "MaxResidual"]), 1.)

    for t, soltab in enumerate(openSoltabs( H, soltabs )):
        logging.info("--> Working on soltab: "+soltab._v_name)
        sf = solFetcher(soltab)

        # times and ants needs to be complete or selection is much slower
        times = sf.getAxisValues('time')
        ants = sf.getAxisValues('ant')

        # this will make a selection for the getValues() and getValuesIter()
        userSel = {}
        for axis in sf.getAxesNames():
            userSel[axis] = getParAxis( step, parset, H, axis )
        sf.setSelection(**userSel)

        # some checks
        solType = sf.getType()
        if solType != 'phase':
           logging.warning("Soltab type of "+soltab._v_name+" is of type "+solType+", should be phase. Ignoring.")
           continue

        if refAnt != '' and not refAnt in ants:
            logging.error('Reference antenna '+refAnt+' not found.')
            return 1
        if refAnt == '': refAnt = ants[0]

        # create new table
        solsetname = soltabs[t].split('/')[0]
        st = H.makeSoltab(solsetname, soltype = sf.getType(), soltab = outTab, axesNames=sf.getAxesNames(), \
                          axesVals=[sf.getAxisValues(axisName) for axisName in sf.getAxesNames()], \
                          vals=sf.getValues(retAxesVals = False), weights=sf.getValues(weight = True, retAxesVals = False), parmdbType=sf.t._v_attrs['parmdb_type'])
        sw = solWriter(st)
        sw.addHistory('Created by CROSSDELAY operation.')
            
        for vals, weights, coord, selection in sf.getValuesIter(returnAxes=['freq','pol','time'], weight=True, reference = refAnt):

            fitdelayguess = 1.e-10 # good guess, do not use 0 as it seems the minimizer is unstable with that

            if 'RR' in coord['pol'] and 'LL' in coord['pol']:
                coord1 = np.where(coord['pol'] == 'RR')[0][0]
                coord2 = np.where(coord['pol'] == 'LL')[0][0]
            elif 'XX' in coord['pol'] and 'YY' in coord['pol']:
                coord1 = np.where(coord['pol'] == 'XX')[0][0]
                coord2 = np.where(coord['pol'] == 'YY')[0][0]

            if not coord['ant'] == refAnt:
                logging.debug('Working on ant: '+coord['ant']+'...')

                if (weights == 0.).all() == True:
                    logging.warning('Skipping flagged antenna: '+coord['ant'])
                    weights[:] = 0
                else:

                    for t, time in enumerate(times):

                        # apply flags
                        idx       = ((weights[coord1,:,t] != 0.) & (weights[coord2,:,t] != 0.))
                        freq      = np.copy(coord['freq'])[idx]
                        phase1    = vals[coord1,:,t][idx]
                        phase2    = vals[coord2,:,t][idx]
    
                        if len(freq) < 10:
                            vals[:,:,t] = 0.
                            weights[:,:,t] = 0.
                            logging.debug('Not enough unflagged point for the timeslot '+str(t))
                            continue
            
                        if (len(idx) - len(freq))/len(freq) > 1/4.:
                            logging.debug('High number of filtered out data points for the timeslot '+str(t)+': '+str(len(idx) - len(freq)))
            
                        phase_diff  = (phase1 - phase2)
        
                        fitresultdelay, success = scipy.optimize.leastsq(delaycomplex, [fitdelayguess], args=(freq, phase_diff))
                        #if t%100==0: print fitresultdelay
                        # fractional residual
                        residual = np.mean(np.abs(np.mod(fitresultdelay*freq-phase_diff + np.pi, 2.*np.pi) - np.pi))

                        #print "t:", t, "result:", fitresultdelay, "residual:", residual
    
                        if residual < maxres:
                            fitdelayguess = fitresultdelay[0]
                            weight = 1
                        else:       
                            # high residual, flag
                            logging.warning('Bad solution for ant: '+coord['ant']+' (time: '+str(t)+', resdiaul: '+str(residual)+').')
                            weight = 0

                        vals[:,:,t] = 0.
                        vals[coord1,:,t][idx] = fitresultdelay[0]*freq/2.
                        vals[coord2,:,t][idx] = -1.*(fitresultdelay[0]*freq)/2.
                        weights[:,:,t] = 0.
                        weights[coord1,:,t][idx] = weight
                        weights[coord2,:,t][idx] = weight

                        # Debug plot
                        doplot = False
                        if doplot and t%500==0 and coord['ant'] == 'CS004LBA':
                            if not 'matplotlib' in sys.modules:
                                import matplotlib as mpl
                                mpl.rc('font',size =8 )
                                mpl.rc('figure.subplot',left=0.05, bottom=0.05, right=0.95, top=0.95,wspace=0.22, hspace=0.22 )
                                mpl.use("Agg")
                            import matplotlib.pyplot as plt
    
                            fig = plt.figure()
                            fig.subplots_adjust(wspace=0)
                            ax = fig.add_subplot(111)
    
                            # plot rm fit
                            plotdelay = lambda delay, freq: np.mod( delay*freq + np.pi, 2.*np.pi) - np.pi
                            ax.plot(freq, plotdelay(fitresultdelay[0], freq), "-", color='purple')
    
                            ax.plot(freq, np.mod(phase1 + np.pi, 2.*np.pi) - np.pi, 'ob' )
                            ax.plot(freq, np.mod(phase2 + np.pi, 2.*np.pi) - np.pi, 'og' )
                            ax.plot(freq, np.mod(phase_diff + np.pi, 2.*np.pi) - np.pi , '.', color='purple' )                           
         
                            residual = np.mod(plotdelay(fitresultdelay[0], freq)-phase_diff + np.pi,2.*np.pi)-np.pi
                            ax.plot(freq, residual, '.', color='yellow')
            
                            ax.set_xlabel('freq')
                            ax.set_ylabel('phase')
                            ax.set_ylim(ymin=-np.pi, ymax=np.pi)
        
                            logging.warning('Save pic: '+str(t)+'_'+coord['ant']+'.png')
                            plt.savefig(coord['ant']+'_'+str(t)+'.png', bbox_inches='tight')
                            del fig

            sw.setSelection(**coord)
            sw.setValues( vals )
            sw.setValues( weights, weight=True )

        del st
        del sw        
        del sf
    return 0
Example #31
0
def run( step, parset, H ):

    import scipy.ndimage.filters
    import numpy as np
    from losoto.h5parm import solFetcher, solWriter

    soltabs = getParSoltabs( step, parset, H )

    axesToSmooth = parset.getStringVector('.'.join(["LoSoTo.Steps", step, "Axes"]), [] )
    FWHM = parset.getIntVector('.'.join(["LoSoTo.Steps", step, "FWHM"]), [] )
    mode = parset.getString('.'.join(["LoSoTo.Steps", step, "Mode"]), "runningmedian" )

    if mode == "runningmedian" and len(axesToSmooth) != len(FWHM):
        logging.error("Axes and FWHM lenghts must be equal.")
        return 1

    if mode == "runningmedian":
        logging.warning('Flagged data are still taken into account!')

    if FWHM != [] and mode != "runningmedian":
        logging.warning("FWHM makes sense only with runningmedian mode, ignoring it.")

    for soltab in openSoltabs( H, soltabs ):

        logging.info("Smoothing soltab: "+soltab._v_name)

        sf = solFetcher(soltab)
        sw = solWriter(soltab, useCache = True) # remember to flush!

        # axis selection
        userSel = {}
        for axis in sf.getAxesNames():
            userSel[axis] = getParAxis( step, parset, H, axis )
        sf.setSelection(**userSel)

        for i, axis in enumerate(axesToSmooth[:]):
            if axis not in sf.getAxesNames():
                del axesToSmooth[i]
                del FWHM[i]
                logging.warning('Axis \"'+axis+'\" not found. Ignoring.')

        for vals, weights, coord, selection in sf.getValuesIter(returnAxes=axesToSmooth, weight=True):

            if mode == 'runningmedian':
                valsnew = scipy.ndimage.filters.median_filter(vals, FWHM)
            elif mode == 'median':
                valsnew = np.median( vals[(weights!=0)] )
            elif mode == 'mean':
                valsnew = np.mean( vals[(weights!=0)] )
            else:
                logging.error('Mode must be: runningmedian, median or mean')
                return 1

            sw.selection = selection
            sw.setValues(valsnew)

        sw.flush()
        sw.addHistory('SMOOTH (over %s with mode = %s)' % (axesToSmooth, mode))
        del sf
        del sw
    return 0
        if st._v_title == 'tecscreen':
            st_tec = st
    if st_tec is not None:
        solTypes.append('TECScreen')
    solTypes = list(set(solTypes))
    logging.info('Found solution types in input parmdb and H5parm: ' +
                 ', '.join(solTypes))

    # For each solType, select appropriate solutions and construct
    # the dictionary to pass to pdb.addValues()
    len_sol = {}
    for solType in solTypes:
        if solType != 'TECScreen':
            len_sol[solType] = len(pdb.getNames(solType + ':*'))
        else:
            tec_sf = solFetcher(st_tec)
            N_times = tec_sf.getAxisLen(axis='time')
            len_sol[solType] = N_times

    for instrumentdbFile in instrumentdbFiles:
        out_instrumentdbFile = out_globaldbFile + '/' + outroot + '_' + instrumentdbFile.split(
            '/')[-1]
        logging.info('Filling ' + out_instrumentdbFile + ':')

        # Remove existing instrumentdb (if clobber) and create new one
        if os.path.exists(out_instrumentdbFile):
            if options.clobber:
                shutil.rmtree(out_instrumentdbFile)
            else:
                logging.critical('Output instrumentdb file exists and '
                                 'clobber = False.')
Example #33
0
def makeTECparmdb(H, solset, TECsolTab, timewidths, freq, freqwidth):
    """Returns TEC screen parmdb parameters

    H - H5parm object
    solset - solution set with TEC screen parameters
    TECsolTab = solution table with tecscreen values
    timewidths - time widths of output parmdb
    freq - frequency of output parmdb
    freqwidth - frequency width of output parmdb
    """
    global ipbar, pbar

    station_dict = H.getAnt(solset)
    station_names = station_dict.keys()
    station_positions = station_dict.values()
    source_dict = H.getSou(solset)
    source_names = source_dict.keys()
    source_positions = source_dict.values()

    tec_sf = solFetcher(TECsolTab)
    tec_screen, axis_vals = tec_sf.getValues()
    times = axis_vals['time']
    beta = TECsolTab._v_attrs['beta']
    r_0 = TECsolTab._v_attrs['r_0']
    height = TECsolTab._v_attrs['height']
    order = TECsolTab._v_attrs['order']
    pp = tec_sf.t.piercepoint

    N_sources = len(source_names)
    N_times = len(times)
    N_freqs = 1
    N_stations = len(station_names)
    N_piercepoints = N_sources * N_stations

    freqs = freq
    freqwidths = freqwidth
    parms = {}
    v = {}
    v['times'] = times
    v['timewidths'] = timewidths
    v['freqs'] = freqs
    v['freqwidths'] = freqwidths

    for station_name in station_names:
        for source_name in source_names:

            v['values'] = np.zeros((N_times, N_freqs), dtype=np.double)
            parmname = 'Piercepoint:X:%s:%s' % (station_name, source_name)
            parms[parmname] = v.copy()

            v['values'] = np.zeros((N_times, N_freqs), dtype=np.double)
            parmname = 'Piercepoint:Y:%s:%s' % (station_name, source_name)
            parms[parmname] = v.copy()

            v['values'] = np.zeros((N_times, N_freqs), dtype=np.double)
            parmname = 'Piercepoint:Z:%s:%s' % (station_name, source_name)
            parms[parmname] = v.copy()

            v['values'] = np.zeros((N_times, N_freqs), dtype=np.double)
            parmname = 'TECfit_white:%s:%s' % (station_name, source_name)
            parms[parmname] = v.copy()

            v['values'] = np.zeros((N_times, N_freqs), dtype=np.double)
            parmname = 'TECfit_white:0:%s:%s' % (station_name, source_name)
            parms[parmname] = v.copy()

            v['values'] = np.zeros((N_times, N_freqs), dtype=np.double)
            parmname = 'TECfit_white:1:%s:%s' % (station_name, source_name)
            parms[parmname] = v.copy()

    for k in range(N_times):
        D = np.resize(pp[k, :, :], (N_piercepoints, N_piercepoints, 3))
        D = np.transpose(D, ( 1, 0, 2 )) - D
        D2 = np.sum(D**2, axis=2)
        C = -(D2 / (r_0**2))**(beta / 2.0) / 2.0
        tec_fit_white = np.dot(np.linalg.inv(C),
            tec_screen[:, k, :].reshape(N_piercepoints))
        pp_idx = 0
        for src, source_name in enumerate(source_names):
            for sta, station_name in enumerate(station_names):

                parmname = 'Piercepoint:X:%s:%s' % (station_name, source_name)
                parms[parmname]['values'][k, 0] = pp[k, pp_idx, 0]

                parmname = 'Piercepoint:Y:%s:%s' % (station_name, source_name)
                parms[parmname]['values'][k, 0] = pp[k, pp_idx, 1]

                parmname = 'Piercepoint:Z:%s:%s' % (station_name, source_name)
                parms[parmname]['values'][k, 0] = pp[k, pp_idx, 2]

                parmname = 'TECfit_white:%s:%s' % (station_name, source_name)
                parms[parmname]['values'][k, 0] = tec_fit_white[pp_idx]

                parmname = 'TECfit_white:0:%s:%s' % (station_name, source_name)
                parms[parmname]['values'][k, 0] = tec_fit_white[pp_idx]

                parmname = 'TECfit_white:1:%s:%s' % (station_name, source_name)
                parms[parmname]['values'][k, 0] = tec_fit_white[pp_idx]

                pp_idx += 1
        pbar.update(ipbar)
        ipbar += 1

    time_start = times[0] - timewidths[0]/2
    time_end = times[-1] + timewidths[-1]/2

    v['times'] = np.array([(time_start + time_end) / 2])
    v['timewidths'] = np.array([time_end - time_start])

    v_r0 = v.copy()
    v_r0['values'] = np.array(r_0, dtype=np.double, ndmin=2)
    parms['r_0'] = v_r0

    v_beta = v.copy()
    v_beta['values'] = np.array(beta, dtype=np.double, ndmin=2)
    parms['beta'] = v_beta

    v_height = v.copy()
    v_height['values'] = np.array(height, dtype=np.double, ndmin=2)
    parms['height'] = v_height

    return parms
Example #34
0
def run(step, parset, H):

    import scipy.ndimage.filters
    import numpy as np
    from losoto.h5parm import solFetcher, solWriter

    soltabs = getParSoltabs(step, parset, H)

    axesToSmooth = parset.getStringVector(
        '.'.join(["LoSoTo.Steps", step, "Axes"]), [])
    FWHM = parset.getIntVector('.'.join(["LoSoTo.Steps", step, "FWHM"]), [])
    mode = parset.getString('.'.join(["LoSoTo.Steps", step, "Mode"]),
                            "runningmedian")

    if mode == "runningmedian" and len(axesToSmooth) != len(FWHM):
        logging.error("Axes and FWHM lenghts must be equal.")
        return 1

    if mode == "runningmedian":
        logging.warning('Flagged data are still taken into account!')

    if FWHM != [] and mode != "runningmedian":
        logging.warning(
            "FWHM makes sense only with runningmedian mode, ignoring it.")

    for soltab in openSoltabs(H, soltabs):

        logging.info("Smoothing soltab: " + soltab._v_name)

        sf = solFetcher(soltab)
        sw = solWriter(soltab, useCache=True)  # remember to flush!

        # axis selection
        userSel = {}
        for axis in sf.getAxesNames():
            userSel[axis] = getParAxis(step, parset, H, axis)
        sf.setSelection(**userSel)

        for i, axis in enumerate(axesToSmooth[:]):
            if axis not in sf.getAxesNames():
                del axesToSmooth[i]
                del FWHM[i]
                logging.warning('Axis \"' + axis + '\" not found. Ignoring.')

        for vals, weights, coord, selection in sf.getValuesIter(
                returnAxes=axesToSmooth, weight=True):

            if mode == 'runningmedian':
                valsnew = scipy.ndimage.filters.median_filter(vals, FWHM)
            elif mode == 'median':
                valsnew = np.median(vals[(weights != 0)])
            elif mode == 'mean':
                valsnew = np.mean(vals[(weights != 0)])
            else:
                logging.error('Mode must be: runningmedian, median or mean')
                return 1

            sw.selection = selection
            sw.setValues(valsnew)

        sw.flush()
        sw.addHistory('SMOOTH (over %s with mode = %s)' % (axesToSmooth, mode))
        del sf
        del sw
    return 0
Example #35
0
def run(step, parset, H):

    import os
    import numpy as np
    from losoto.h5parm import solFetcher, solHandler
    # avoids error if re-setting "agg" a second run of plot
    if not 'matplotlib' in sys.modules:
        import matplotlib as mpl
        mpl.rc('font', size=8)
        mpl.rc('figure.subplot',
               left=0.05,
               bottom=0.05,
               right=0.95,
               top=0.95,
               wspace=0.22,
               hspace=0.22)
        mpl.use("Agg")
    import matplotlib.pyplot as plt  # after setting "Agg" to speed up

    soltabs = getParSoltabs(step, parset, H)

    minZ, maxZ = parset.getDoubleVector(
        '.'.join(["LoSoTo.Steps", step, "MinMax"]), [0, 0])
    prefix = parset.getString('.'.join(["LoSoTo.Steps", step, "Prefix"]), '')

    # Plot various TEC-screen properties

    for st_scr in openSoltabs(H, soltabs):

        # Check if soltab is a tecscreen table
        full_name = st_scr._v_parent._v_name + '/' + st_scr._v_name
        if st_scr._v_title != 'tecscreen':
            logging.warning('Solution table {0} is not a tecscreen solution '
                            'table. Skipping.'.format(full_name))
            continue
        logging.info('Using input solution table: {0}'.format(full_name))

        # Plot TEC screens as images
        solset = st_scr._v_parent
        sf_scr = solFetcher(st_scr)
        r, axis_vals = sf_scr.getValues()
        source_names = axis_vals['dir']
        station_names = axis_vals['ant']
        station_dict = H.getAnt(solset)
        station_positions = []
        for station in station_names:
            station_positions.append(station_dict[station])
        times = axis_vals['time']

        tec_screen, axis_vals = sf_scr.getValues()
        times = axis_vals['time']
        residuals = sf_scr.getValues(weight=True, retAxesVals=False)
        height = st_scr._v_attrs['height']
        order = st_scr._v_attrs['order']
        beta_val = st_scr._v_attrs['beta']
        r_0 = st_scr._v_attrs['r_0']
        pp = sf_scr.t.piercepoint

        if (minZ == 0 and maxZ == 0):
            min_tec = None
            max_tec = None
        else:
            min_tec = minZ
            max_tec = maxZ

        make_tec_screen_plots(pp,
                              tec_screen,
                              residuals,
                              np.array(station_positions),
                              np.array(source_names),
                              times,
                              height,
                              order,
                              beta_val,
                              r_0,
                              prefix=prefix,
                              remove_gradient=True,
                              show_source_names=False,
                              min_tec=min_tec,
                              max_tec=max_tec)

    return 0
Example #36
0
              weights=vals)
logging.info("Create soltab (using default name)")
H5.makeSoltab(ss,
              'amplitude',
              axesNames=['axis1', 'axis2', 'axis3'],
              axesVals=axesVals,
              vals=vals,
              weights=vals)
logging.info('Get a soltab object')
st = H5.getSoltab(ss, 'stTest')
logging.info('Get all soltabs:')
print H5.getSoltabs(ss)

print "###########################################"
logging.info('### solFetcher/solWriter - General')
Hsf = solFetcher(st)
Hsw = solWriter(st)
logging.info('Get solution Type (exp: amplitude)')
print Hsf.getType()
logging.info('Get Axes Names')
print Hsf.getAxesNames()
logging.info('Get Axis1 Len (exp: 4)')
print Hsf.getAxisLen('axis1')
logging.info('Get Axis1 Type (exp: str)')
print Hsf.getAxisType('axis1')
logging.info('Get Axis2 Type (exp: float)')
print Hsf.getAxisType('axis2')
logging.info('Get Axis1 Values (exp: a,b,c,d)')
print Hsf.getAxisValues('axis1')
logging.info('Set new axes values')
Hsw.setAxisValues('axis1', ['e', 'f', 'g', 'h'])
Example #37
0
def run(step, parset, H):
    """
   Generic unspecified step for easy expansion.
   """
    import numpy as np
    from losoto.h5parm import solFetcher, solWriter
    # all the following are LoSoTo function to extract information from the parset

    # get involved solsets using local step values or global values or all
    solsets = getParSolsets(step, parset, H)
    logging.info('Solset: ' + str(solsets))
    # get involved soltabs using local step values or global values or all
    soltabs = getParSoltabs(step, parset, H)
    logging.info('Soltab: ' + str(soltabs))
    # get list of SolTypes using local step values or global values or all
    solTypes = getParSolTypes(step, parset, H)
    logging.info('SolType: ' + str(solTypes))

    # do something on every soltab (use the openSoltab LoSoTo function)
    for soltab in openSoltabs(H, soltabs):
        logging.info("--> Working on soltab: " + soltab._v_name)
        # use the solFetcher from the H5parm lib
        t = solFetcher(soltab)
        tw = solWriter(soltab)

        axisNames = t.getAxesNames()
        logging.info("Axis names are: " + str(axisNames))

        solType = t.getType()
        logging.info("Soltab type is: " + solType)

        # this will make a selection for the getValues() and getValuesIter()
        # interpret every entry in the parset which has an axis name as a selector
        userSel = {}
        for axis in t.getAxesNames():
            userSel[axis] = getParAxis(step, parset, H, axis)
        t.setSelection(**userSel)

        t.setSelection(ant=ants, pol=pols, dir=dirs)
        logging.info("Selection is: " + str(t.selection))

        # find axis values
        logging.info("Antennas (no selection) are: " +
                     str(t.getAxisValues('ant', ignoreSelection=True)))
        logging.info("Antennas (with selection) are: " +
                     str(t.getAxisValues('ant')))
        # but one can also use (selection is active here!)
        logging.info("Antennas (other method) are: " + str(t.ant))
        logging.info("Frequencies are: " + str(t.freq))
        logging.info("Directions are: " + str(t.dir))
        logging.info("Polarizations are: " + str(t.pol))
        # try to access a non-existent axis
        t.getAxisValues('nonexistantaxis')

        # now get all values given this selection
        logging.info("Get data using t.val")
        val = t.val
        logging.debug('shape of val: ' + str(t.val.shape))
        logging.info("$ val is " + str(val[0, 0, 0, 0, 100]))
        weight = t.weight
        time = t.time
        thisTime = t.time[100]

        # another way to get the data is using the getValues()
        logging.info("Get data using getValues()")
        grid, axes = t.getValues()
        # axis names
        logging.info("Axes: " + str(t.getAxesNames()))
        # axis shape
        print axes
        print[t.getAxisLen(axis) for axis in axes]  # not ordered, is a dict!
        # data array shape (same of axis shape)
        logging.info("Shape of values: " + str(grid.shape))
        #logging.info("$ val is "+str(grid[0,0,0,0,100]))

        # reset selection
        t.setSelection()
        logging.info('Reset selection to \'\'')
        logging.info("Antennas are: " + str(t.ant))
        logging.info("Frequencies are: " + str(t.freq))
        logging.info("Directions are: " + str(t.dir))
        logging.info("Polarizations are: " + str(t.pol))

        # finally the getValuesIter allaws to iterate across all possible combinations of a set of axes
        logging.info('Iteration on time/freq')
        for vals, coord, selection in t.getValuesIter(
                returnAxes=['time', 'freq']):
            # writing back the solutions
            tw.selection = selection
            tw.setValues(vals)
        logging.info('Iteration on time')
        for vals, coord, selection in t.getValuesIter(returnAxes=['time']):
            # writing back the solutions
            tw.selection = selection
            tw.setValues(vals)
        logging.info('Iteration on dir after selection to 1 dir')
        t.setSelection(dir='pointing')
        for vals, coord, selection in t.getValuesIter(returnAxes=['dir']):
            # writing back the solutions
            tw.selection = selection
            tw.setValues(vals)

    return 0  # if everything went fine, otherwise 1
Example #38
0
    for name, st in solTabs.iteritems():
        if st._v_title == 'tecscreen':
            st_tec = st
    if st_tec is not None:
        solTypes.append('TECScreen')
    solTypes = list(set(solTypes))
    logging.info('Found solution types in input parmdb and H5parm: '+', '.join(solTypes))

    # For each solType, select appropriate solutions and construct
    # the dictionary to pass to pdb.addValues()
    len_sol = {}
    for solType in solTypes:
        if solType != 'TECScreen':
            len_sol[solType] = len(pdb.getNames(solType+':*'))
        else:
            tec_sf = solFetcher(st_tec)
            N_times = tec_sf.getAxisLen(axis='time')
            len_sol[solType] = N_times

    for instrumentdbFile in instrumentdbFiles:
        out_instrumentdbFile = out_globaldbFile + '/' + outroot + '_' + instrumentdbFile.split('/')[-1]
        logging.info('Filling '+out_instrumentdbFile+':')

        # Remove existing instrumentdb (if clobber) and create new one
        if os.path.exists(out_instrumentdbFile):
            if options.clobber:
                shutil.rmtree(out_instrumentdbFile)
            else:
                logging.critical('Output instrumentdb file exists and '
                    'clobber = False.')
                sys.exit(1)
Example #39
0
def run( step, parset, H ):

    import scipy.ndimage.filters
    import numpy as np
    from losoto.h5parm import solFetcher, solWriter
    from scipy.optimize import minimize
    import itertools
    from scipy.interpolate import griddata
    import scipy.cluster.vq as vq

    def robust_std(data, sigma=3):
        """
        Calculate standard deviation excluding outliers
        ok with masked arrays
        """
        return np.std(data[np.where(np.abs(data) < sigma * np.std(data))])

    def mask_interp(vals, mask, method='nearest'):
        """
        return interpolated values for masked elements
        """
        this_vals = vals.copy()
        #this_vals[mask] = np.interp(np.where(mask)[0], np.where(~mask)[0], vals[~mask])
        #this_vals[mask] = griddata(np.where(~mask)[0], vals[~mask], np.where(mask)[0], method)

        # griddata has nan bug with nearest, I need to use vq
        code, dist = vq.vq(np.where(mask)[0], np.where(~mask)[0])
        this_vals[ np.where(mask)[0] ] = this_vals[code]
        
        return this_vals

    tec_jump_val = 0.019628
    maxsize = 300
    clip = 10 # TECs over these amount of jumps are flagged

    soltabs = getParSoltabs( step, parset, H )

    for soltab in openSoltabs( H, soltabs ):

        logging.info("Removing TEC jumps from soltab: "+soltab._v_name)

        sf = solFetcher(soltab)
        sw = solWriter(soltab) # remember to flush!

        # TODO: check if it's a Tec table

        # axis selection
        userSel = {}
        for axis in sf.getAxesNames():
            userSel[axis] = getParAxis( step, parset, H, axis )
        sf.setSelection(**userSel)

        for vals, weights, coord, selection in sf.getValuesIter(returnAxes='time', weight=True):

            # skip all flagged
            if (weights == 0).all(): continue
            # skip reference
            if (np.diff(vals[(weights == 1)]) == 0).all(): continue

            # kill large values
#            weights[abs(vals/tec_jump_val)>clip] = 0

            # interpolate flagged values to get resonable distances
#            vals = mask_interp(vals, mask=(weights == 0))/tec_jump_val
            # add edges to allow intervals to the borders
#            vals = np.insert(vals, 0, vals[0])
#            vals = np.insert(vals, len(vals), vals[-1])

            vals = np.fmod(vals,tec_jump_val)

#            def find_jumps(d_vals):
#                # jump poistion finder
#                d_smooth = scipy.ndimage.filters.median_filter( mask_interp(d_vals, mask=(abs(d_vals)>0.8)), 21 )
#                d_vals -= d_smooth
#                jumps = list(np.where(np.abs(d_vals) > 1.)[0])
#                return [0]+jumps+[len(d_vals)-1] # add edges
#
#            class Jump(object):
#                def __init__(self, jumps_idx, med):
#                    self.idx_left = jumps_idx[0]
#                    self.idx_right = jumps_idx[1]
#                    self.jump_left = np.rint(d_vals[self.idx_left])
#                    self.jump_right = np.rint(d_vals[self.idx_right])
#                    self.size = self.idx_right-self.idx_left
#                    self.hight = np.median(vals[self.idx_left+1:self.idx_right+1]-med)
#                    if abs((self.hight-self.jump_left)-med) > abs((self.hight-self.jump_right)-med):
#                        self.closejump = self.jump_right
#                    else:
#                        self.closejump = self.jump_left
#    
#            i = 0
#            while i<len(coord['time']):
#                # get tec[i] - tec[i+1], i.e. the derivative assuming constant timesteps
#                # this is in units of tec_jump_val!
#                d_vals = np.diff(vals)
#                # get jumps idx, idx=n means a jump beteen val n and n+1
#                jumps_idx = find_jumps(d_vals)
#
#                # get regions
#                med = np.median(vals)
#                jumps = [Jump(jump_idx, med) for jump_idx in zip( jumps_idx[:-1], jumps_idx[1:] )]
#                jumps = [jump for jump in jumps if jump.closejump != 0]
#                jumps = [jump for jump in jumps if jump.size != 0] # prevent bug on edges
#                jumps = [jump for jump in jumps if jump.size < maxsize]
#
#                jumps.sort(key=lambda x: (np.abs(x.size), x.hight), reverse=False) #smallest first
#                #print [(j.hight, j.closejump) for j in jumps]
#
#                plot = False
#                if plot:
#                    import matplotlib.pyplot as plt
#                    fig, ((ax1, ax2, ax3)) = plt.subplots(3, 1, sharex=True)
#                    fig.subplots_adjust(hspace=0)
#                    d_smooth = scipy.ndimage.filters.median_filter( mask_interp(d_vals, mask=(abs(d_vals)>0.8)), 31 )
#                    ax1.plot(d_vals,'k-')
#                    ax2.plot(d_smooth,'k-')
#                    ax3.plot(vals, 'k-')
#                    [ax3.axvline(jump_idx+0.5, color='r', ls=':') for jump_idx in jumps_idx]
#                    ax1.set_ylabel('d_vals')
#                    ax2.set_ylabel('d_vals - smooth')
#                    ax3.set_ylabel('TEC/jump')
#                    ax3.set_xlabel('timestep')
#                    ax1.set_xlim(xmin=-10, xmax=len(d_smooth)+10)
#                    fig.savefig('plots/%stecjump_debug_%03i' % (coord['ant'], i))
#                i+=1
#
#                if len(jumps) == 0: 
#                    break
#
#                # move down the highest to the side closest to the median
#                j = jumps[0]
#                #print j.idx_left, j.idx_right, j.jump_left, j.jump_right, j.hight, j.closejump
#
#                vals[j.idx_left+1:j.idx_right+1] -= j.closejump
#                logging.debug("%s: Number of jumps left: %i - Removing jump: %i - Size %i" % (coord['ant'], len(jumps_idx)-2, j.closejump, j.size))
                
            # re-create proper vals
#            vals = vals[1:-1]*tec_jump_val
            # set back to 0 the values for flagged data
            vals[weights == 0] = 0

            sw.selection = selection
            sw.setValues(vals)
            sw.setValues(weights, weight=True)

        sw.addHistory('TECJUMP')
        del sf
        del sw
    return 0
Example #40
0
opt.add_option('-n',
               '--numiter',
               help='Number of iterations (default=100)',
               type=int,
               default=100)
(options, args) = opt.parse_args()

logger = _logging.Logger('debug')
logging = _logging.logger

n = options.numiter

solset = options.solset
h5parmFile = options.h5parm
H5 = h5parm(h5parmFile, readonly=False)
H = solFetcher(H5.getSoltab(solset, 'rotation000'))
logging.info("H5parm filename = " + h5parmFile)

parmdbFile = options.parmdb
P = lofar.parmdb.parmdb(parmdbFile)
P2 = lofar.parmdb.parmdb('tmp.parmdb', create=True)
logging.info("parmdb filename = " + parmdbFile)

######################################################
logging.info("### Read all frequencies for a pol/dir/station")

start = time.clock()
for i in range(n):
    Pfreqs = P.getValuesGrid('RotationAngle:CS001LBA:3C196'
                             )['RotationAngle:CS001LBA:3C196']['freqs']
elapsed = (time.clock() - start)
Example #41
0
def run( step, parset, H ):
    """
    Interpolate the solutions from one table into a destination table
    """
    import itertools
    import scipy.interpolate
    import numpy as np
    from losoto.h5parm import solFetcher, solWriter

    soltabs = getParSoltabs( step, parset, H )
    solTypes = getParSolTypes( step, parset, H )

    calSoltab = parset.getString('.'.join(["LoSoTo.Steps", step, "CalSoltab"]), '' )
    calDir = parset.getString('.'.join(["LoSoTo.Steps", step, "CalDir"]), '' )
    interpAxes = parset.getStringVector('.'.join(["LoSoTo.Steps", step, "InterpAxes"]), ['time','freq'] )
    interpMethod = parset.getString('.'.join(["LoSoTo.Steps", step, "InterpMethod"]), 'linear' )
    medAxis = parset.getString('.'.join(["LoSoTo.Steps", step, "MedAxis"]), '' )
    rescale = parset.getBool('.'.join(["LoSoTo.Steps", step, "Rescale"]), False )

    if interpMethod not in ["nearest", "linear", "cubic"]:
        logging.error('Interpolation method must be nearest, linear or cubic.')
        return 1

    if rescale and medAxis == '':
        logging.error('A medAxis is needed for rescaling.')
        return 1

    # open calibration table
    css, cst = calSoltab.split('/')
    cr = solFetcher(H.getSoltab(css, cst))
    cAxesNames = cr.getAxesNames()

    for soltab in openSoltabs( H, soltabs ):

        logging.info("Interpolating soltab: "+soltab._v_name)

        tr = solFetcher(soltab)
        tw = solWriter(soltab)

        axesNames = tr.getAxesNames()
        for i, interpAxis in enumerate(interpAxes[:]):
            if interpAxis not in axesNames or interpAxis not in cAxesNames:
                logging.error('Axis '+interpAxis+' not found. Ignoring.')
                del interpAxes[i]
        if rescale and (medAxis not in axesNames or medAxis not in cAxesNames):
            logging.error('Axis '+medAxis+' not found. Cannot proceed.')
            return 1

        # axis selection
        userSel = {}
        for axis in tr.getAxesNames():
            userSel[axis] = getParAxis( step, parset, H, axis )
        tr.setSelection(**userSel)

        for vals, coord, selection in tr.getValuesIter(returnAxes=interpAxes):

            # construct grid
            coordSel = removeKeys(coord, interpAxes)
            logging.debug("Working on coords:"+str(coordSel))
            # change dir if sepcified
            if calDir != '':
                coordSel['dir'] = calDir
            cr.setSelection(**coordSel)
            calValues, calCoord = cr.getValues()

            # fill medAxis with the median value
            if rescale:
                axis = cAxesNames.index(medAxis)
                calValues = np.repeat( np.expand_dims( np.median( calValues, axis ), axis ), calValues.shape[axis], axis )

            # create a list of values whose coords are calPoints
            calValues = np.ndarray.flatten(calValues)

            # create calibrator/target coordinates arrays
            calPoints = []
            targetPoints = []
            for interpAxis in interpAxes:
                calPoints.append(calCoord[interpAxis])
                targetPoints.append(coord[interpAxis])
            calPoints = np.array([x for x in itertools.product(*calPoints)])
            targetPoints = np.array([x for x in itertools.product(*targetPoints)])

            # interpolation
            valsNew = scipy.interpolate.griddata(calPoints, calValues, targetPoints, interpMethod)

            # fill values outside boudaries with "nearest" solutions
            # NOTE: in 1D is useless due to scipy bug but this is compensated in the next if
            if interpMethod != 'nearest':
                valsNewNearest = scipy.interpolate.griddata(calPoints, calValues, targetPoints, 'nearest')
                # NaN is != from itself
                valsNew[ np.isnan(valsNew) ] = valsNewNearest [ np.isnan(valsNew) ]

            # fix bug in Scipy which put NaNs outside the convex hull in 1D for 'nearest'
            if len(np.squeeze(vals).shape) == 1:
                import scipy.cluster.vq as vq
                valsNew = np.squeeze(valsNew)
                code, dist = vq.vq(targetPoints, calPoints)
                # NaN is != from itself
                valsNew[ np.isnan(valsNew) ] = calValues[code][ np.isnan(valsNew) ]
                valsNew = valsNew.reshape(vals.shape)

            if rescale:
                # rescale solutions
                axis = interpAxes.index(medAxis)
                valsMed = np.repeat( np.expand_dims( np.median( vals, axis ), axis ), vals.shape[axis], axis )
                valsNewMed = np.repeat( np.expand_dims( np.median( valsNew, axis ), axis ), valsNew.shape[axis], axis )
                valsNew = vals*valsNewMed/valsMed
                #print "Rescaling by: ", valsNewMed[:,0]/valsMed[:,0]

            # writing back the solutions
            tw.selection = selection
            tw.setValues(valsNew)

    tw.addHistory('INTERP (from table %s)' % (calSoltab))
    return 0
Example #42
0
def collect_solutions(H, dirs=None, freq_tol=1e6, solsets=None):
    """
    Collects and returns phase solutions, etc. needed for fitting

    Keyword arguments:
    H -- H5parm object
    dirs -- list of directions to use (None => all)
    freq_tol -- tolerance for grouping of phase solutions into bands (Hz)
    solsets -- list of solution sets in H to search (None => all)
    """
    import numpy as np
    from pylab import find
    import re
    try:
        import progressbar
    except ImportError:
        import losoto.progressbar as progressbar
    logging.info("Scanning for solutions needed for TEC fitting...")

    # Determine axis lengths
    sources = []
    freqs = []
    stations = []
    if solsets is None:
        solsets = H.getSolsets().keys()
    N_times_max = 0
    first_solset = None
    for solset in solsets[:]:
        logging.info('  --Scanning solution set {0}...'.format(solset))
        has_phase_st = False
        soltabs = H.getSoltabs(solset)

        for soltab in soltabs:
            # Only use soltabs with phase solutions
            if 'phase' in soltabs[soltab]._v_title:
                logging.info('    --Scanning solution table {0}...'.format(soltab))
                has_phase_st = True
                solset_sources = H.getSou(solset)
                if len(solset_sources) == 1 and 'pointing' in solset_sources:
                    logging.info('      Found direction-independent solutions')
                    dir_indep = True
                    soln_type_dirindep = soltabs[soltab]._v_title
                    if first_solset is None:
                        first_solset = solset
                else:
                    logging.info('      Found direction-dependent solutions')
                    dir_indep = False
                    if first_solset is None:
                        first_solset = solset
                if not dir_indep:
                    soln_type_dirdep = soltabs[soltab]._v_title
                    logging.info('      Found sources: {0}'.format(
                        H.getSou(solset).keys()))
                    sources += H.getSou(solset).keys()
                    stations += H.getAnt(solset).keys()
                    t = solFetcher(soltabs[soltab])
                    solns, axis_vals = t.getValues()
                    times = axis_vals['time']
                    N_times = len(times)
                    freqs.append(axis_vals['freq'][0])
                    if N_times > N_times_max:
                        N_times_max = N_times
                        times_max = times
        if not has_phase_st:
            logging.info('    --No phase solutions found')
            solsets.remove(solset)

    # Consolidate frequencies into bands (defined by freq_tol parameter)
    has_dup = True
    while has_dup:
        has_dup = False
        for freq in freqs[:]:
            nfreq = len( find( (np.array(freqs) > (freq-freq_tol)) &
                               (np.array(freqs) < (freq+freq_tol)) ) )
            if nfreq > 1:
                has_dup = True
                freqs.remove(freq)
                break
    freqs = np.array(sorted(freqs))
    N_freqs = len(freqs)
    stations_set = set(stations)
    sources_set = set(sources)
    if 'pointing' in sources_set:
        sources_set.remove('pointing')
    if dirs is not None:
        sources_filt = []
        for dir in dirs:
            if dir in sources_set:
                sources_filt.append(dir)
        sources_set = set(sources_filt)
    source_names = np.array(sorted(list(sources_set)))
    N_sources = len(set(source_names))
    N_stations = len(set(stations))
    N_times = N_times_max
    times = times_max
    logging.info('Scanning complete')
    logging.info('  Number of sources: {0}'.format(N_sources))
    logging.info('  Number of stations: {0}'.format(N_stations))
    logging.info('  Number of times: {0}'.format(N_times))
    logging.info('  Number of bands: {0}'.format(N_freqs))
    if N_sources == 0 or N_stations == 0 or N_times == 0 or N_freqs == 0:
        logging.error('No solutions found.')
        return (None, None, None, None, None, None, None, None, None, None,
            None, None)

    # Initialize the arrays
    freqwidths = np.zeros(N_freqs)
    timewidths = np.zeros(N_times)
    m = np.zeros((N_sources, N_freqs))
    phases0 = np.zeros((N_sources, N_stations, N_freqs, N_times))
    phases1 = np.zeros((N_sources, N_stations, N_freqs, N_times))
    flags = np.ones((N_sources, N_stations, N_freqs, N_times))
    source_positions = np.zeros((N_sources, 2))
    solset_array_dir_dep = np.zeros((N_sources, N_freqs), dtype='|S100')
    solset_array_dir_indep = np.zeros(N_freqs, dtype='|S100')

    # Populate the arrays
    for solset in solsets:
        source_dict = H.getSou(solset)
        sources = source_dict.keys()
        soltabs = H.getSoltabs(solset)

        for soltab in soltabs:
            t = solFetcher(soltabs[soltab])
            solns, axes = t.getValues()
            freq = axes['freq'][0]
            freq_indx = find( (np.array(freqs) > (freq-freq_tol)) &
                (np.array(freqs) < (freq+freq_tol)) )

            for source in sources:
                if source in source_names:
                    source_indx = find(source_names == source)
                    m[source_indx, freq_indx] = 1
                    solset_array_dir_dep[source_indx, freq_indx] = solset
                elif source=='pointing' and len(sources) == 1:
                    solset_array_dir_indep[freq_indx] = solset

    # Collect station properties and pointing direction
    station_dict = H.getAnt(first_solset)
    station_names = np.array(list(stations_set))
    station_positions = np.zeros((len(station_names), 3), dtype=np.float)
    for i, station_name in enumerate(station_names):
        station_positions[i, 0] = station_dict[station_name][0]
        station_positions[i, 1] = station_dict[station_name][1]
        station_positions[i, 2] = station_dict[station_name][2]

    source_dict = H.getSou(first_solset)
    ra = source_dict['pointing'][0]
    dec = source_dict['pointing'][1]
    pointing = np.array([np.cos(ra) * np.cos(dec), np.sin(ra) * np.cos(dec),
        np.sin(dec)])

    # Collect source positions and phase solutions for each band
    logging.info('Collecting phase solutions...')
    pbar = progressbar.ProgressBar(maxval=len(source_names)).start()
    for i, source1 in enumerate(source_names):
        for k in xrange(N_freqs):
            if m[i, k]:
                solset_name = str(solset_array_dir_dep[i, k])
                soltab = H.getSoltab(solset=solset_name, soltab=soln_type_dirdep+'000')
                solset_dir_indep_name = str(solset_array_dir_indep[k])
                source_dict = H.getSou(solset_name)
                source_positions[i, ] = source_dict[source1]

                if solset_dir_indep_name != '':
                    soltab_dir_indep = H.getSoltab(solset=solset_dir_indep_name,
                        soltab=soln_type_dirindep+'000')
                    sf_dir_indep = solFetcher(soltab_dir_indep)
                else:
                    sf_dir_indep = None
                sf_dir_dep = solFetcher(soltab)

                for l, station in enumerate(station_names):
                    if soln_type_dirdep == 'scalarphase':
                        sf_dir_dep.setSelection(ant=[station], dir=[source1])
                    else:
                        sf_dir_dep.setSelection(ant=[station], pol=['XX'], dir=[source1])
                    values_dir_dep = sf_dir_dep.getValues()
                    v1_phase = np.array(values_dir_dep[0]).squeeze()
                    times_dir_dep = values_dir_dep[1]['time']
                    ind = np.where(~np.isnan(v1_phase))
                    v1_phase = v1_phase[ind]
                    times_dir_dep = times_dir_dep[ind]

                    if sf_dir_indep is not None:
                        if soln_type_dirindep == 'scalarphase':
                            sf_dir_indep.setSelection(ant=[station])
                        else:
                            sf_dir_indep.setSelection(ant=[station], pol=['XX'])
                        values_dir_indep = sf_dir_indep.getValues()
                        v1_dir_indep = np.array(values_dir_indep[0]).squeeze()
                        times_dir_indep = values_dir_indep[1]['time']
                        ind = np.where(~np.isnan(v1_dir_indep))
                        v1_dir_indep = v1_dir_indep[ind]
                        times_dir_indep = times_dir_indep[ind]
                        v1_dir_indep_interp = interpolate_phase(v1_dir_indep,
                            times_dir_indep, times_dir_dep)
                        v1_phase += v1_dir_indep_interp
                    if len(times_dir_dep) != N_times_max:
                         v1_phase = interpolate_phase(v1_phase, times_dir_dep,
                            times_max)
                    phases0[i, l, k, :] = v1_phase

                    if soln_type_dirdep == 'scalarphase':
                        phases1[i, l, k, :] = v1_phase
                    else:
                        sf_dir_dep.setSelection(ant=[station], pol=['YY'], dir=[source1])
                        values_dir_dep = sf_dir_dep.getValues()
                        v1_phase = np.array(values_dir_dep[0]).squeeze()
                        times_dir_dep = values_dir_dep[1]['time']
                        ind = np.where(~np.isnan(v1_phase))
                        v1_phase = v1_phase[ind]
                        times_dir_dep = times_dir_dep[ind]

                        if sf_dir_indep is not None:
                            if soln_type_dirindep == 'scalarphase':
                                sf_dir_indep.setSelection(ant=[station])
                            else:
                                sf_dir_indep.setSelection(ant=[station], pol=['YY'])
                            values_dir_indep = sf_dir_indep.getValues()
                            v1_dir_indep = np.array(values_dir_indep[0]).squeeze()
                            times_dir_indep = values_dir_indep[1]['time']
                            ind = np.where(~np.isnan(v1_dir_indep))
                            v1_dir_indep = v1_dir_indep[ind]
                            times_dir_indep = times_dir_indep[ind]
                            v1_dir_indep_interp = interpolate_phase(v1_dir_indep,
                                times_dir_indep, times_dir_dep)
                            v1_phase += v1_dir_indep_interp
                        if len(times_dir_dep) != N_times_max:
                            v1_phase = interpolate_phase(v1_phase, times_dir_dep,
                                times_max)
                        phases1[i, l, k, :] = v1_phase

                    if len(times_dir_dep) != N_times_max:
                        # Set all weights to unity if interpolation was required
                        flags[i, l, k, :] = 1.0
                    else:
                        flags[i, l, k, :] = sf_dir_dep.getValues(weight=True,
                            retAxesVals=False)

                    if np.all(phases0[i, l, k, :] == 0.0):
                        # Check for flagged stations
                        flags[i, l, k, :] = 0.0
        pbar.update(i)
    pbar.finish()
    logging.info('Collection complete')
    for i, source in enumerate(source_names):
        nbands = len(find(m[i, :]))
        logging.info('  Source {0} has solutions in {1} bands'.format(source, nbands))

    # Invert the weights to give flags (0 => unflagged, 1 => flagged)
    zeroflags = np.where(flags == 0.0)
    oneflags = flags.nonzero()
    flags[zeroflags] = 1.0
    flags[oneflags] = 0.0

    return (phases0, phases1, flags, m, station_names, station_positions,
        source_names, source_positions, freqs, times, pointing, soln_type_dirdep)
Example #43
0
def run(step, parset, H):

    import numpy as np
    from losoto.h5parm import solFetcher, solWriter

    soltabs = getParSoltabs(step, parset, H)

    axesToClip = parset.getStringVector(
        '.'.join(["LoSoTo.Steps", step, "Axes"]), [])
    clipLevel = parset.getFloat('.'.join(["LoSoTo.Steps", step, "ClipLevel"]),
                                0.)
    log = parset.getBool('.'.join(["LoSoTo.Steps", step, "Log"]), True)

    if len(axesToClip) < 1:
        logging.error("Please specify axes to clip.")
        return 1
    if clipLevel == 0.:
        logging.error(
            "Please specify factor above/below median at which to clip.")
        return 1

    for soltab in openSoltabs(H, soltabs):

        logging.info("Clipping soltab: " + soltab._v_name)

        sf = solFetcher(soltab)

        # axis selection
        userSel = {}
        for axis in sf.getAxesNames():
            userSel[axis] = getParAxis(step, parset, H, axis)
        sf.setSelection(**userSel)

        # some checks
        for i, axis in enumerate(axesToClip[:]):
            if axis not in sf.getAxesNames():
                del axesToClip[i]
                logging.warning('Axis \"' + axis + '\" not found. Ignoring.')

        if sf.getType() != 'amplitude':
            logging.error('CLIP is for "amplitude" tables, not %s.' %
                          sf.getType())
            continue

        sw = solWriter(soltab, useCache=True)  # remember to flush()

        before_count = 0
        after_count = 0
        total = 0
        for vals, weights, coord, selection in sf.getValuesIter(
                returnAxes=axesToClip, weight=True):

            total += len(vals)
            before_count += (len(weights) - np.count_nonzero(weights))

            # first find the median and standard deviation
            if (weights == 0).all():
                valmedian = 0
            else:
                if log:
                    valmedian = np.median(np.log10(vals[(weights != 0)]))
                    rms = np.std(np.log10(vals[(weights != 0)]))
                    np.putmask(
                        weights,
                        np.abs(np.log10(vals) - valmedian) > rms * clipLevel,
                        0)
                else:
                    valmedian = np.median(vals[(weights != 0)])
                    rms = np.std(vals[(weights != 0)])
                    np.putmask(weights,
                               np.abs(vals - valmedian) > rms * clipLevel, 0)

            after_count += (len(weights) - np.count_nonzero(weights))

            # writing back the solutions
            sw.selection = selection
            sw.setValues(weights, weight=True)

        sw.addHistory('CLIP (over %s with %s sigma cut)' %
                      (axesToClip, clipLevel))
        logging.info('Clip, flagged data: %f %% -> %f %%' \
                % (100.*before_count/total, 100.*after_count/total))

        sw.flush()

    return 0
Example #44
0
def run( step, parset, H ):

    from losoto.h5parm import solFetcher, solWriter

    soltabs = getParSoltabs( step, parset, H )

    #check_parset('Axes','MaxCycles','MaxRms','Order','Replace','PreFlagZeros')
    axesToFlag = parset.getStringVector('.'.join(["LoSoTo.Steps", step, "Axes"]), 'time' )
    maxCycles = parset.getInt('.'.join(["LoSoTo.Steps", step, "MaxCycles"]), 5 )
    maxRms = parset.getFloat('.'.join(["LoSoTo.Steps", step, "MaxRms"]), 5. )
    fixRms = parset.getFloat('.'.join(["LoSoTo.Steps", step, "FixRms"]), 0 )
    order = parset.getIntVector('.'.join(["LoSoTo.Steps", step, "Order"]), 3 )
    replace = parset.getBool('.'.join(["LoSoTo.Steps", step, "Replace"]), False )
    preflagzeros = parset.getBool('.'.join(["LoSoTo.Steps", step, "PreFlagZeros"]), False )
    mode = parset.getString('.'.join(["LoSoTo.Steps", step, "Mode"]), 'smooth' )
    ref = parset.getString('.'.join(["LoSoTo.Steps", step, "Reference"]), '' )
    ncpu = parset.getInt('.'.join(["LoSoTo.Ncpu"]), 1 )

    if ref == '': ref = None

    if axesToFlag == []:
        logging.error("Please specify axis to flag. It must be a single one.")
        return 1

    if len(axesToFlag) != len(order):
        logging.error("AxesToFlag and order must be both 1 or 2 values.")
        return 1

    if len(order) == 1: order = order[0]
    elif len(order) == 2: order = tuple(order)

    mode = mode.lower()
    if mode != 'smooth' and mode != 'poly' and mode != 'spline':
        logging.error('Mode must be smooth, poly or spline')
        return 1

    for soltab in openSoltabs( H, soltabs ):

        # start processes for multi-thread
        mpm = multiprocManager(ncpu, flag)

        logging.info("Flagging soltab: "+soltab._v_name)

        sf = solFetcher(soltab)
        sw = solWriter(soltab, useCache=True) # remember to flush!

        # axis selection
        userSel = {}
        for axis in sf.getAxesNames():
            userSel[axis] = getParAxis( step, parset, H, axis )
        sf.setSelection(**userSel)

        for axisToFlag in axesToFlag:
            if axisToFlag not in sf.getAxesNames():
                logging.error('Axis \"'+axis+'\" not found.')
                mpm.wait()
                return 1

        # reorder axesToFlag as axes in the table
        axesToFlag_orig = axesToFlag
        axesToFlag = [coord for coord in sf.getAxesNames() if coord in axesToFlag]
        if type(order) is int: order = [order]
        if axesToFlag_orig != axesToFlag: order = order[::-1] # reverse order if we changed axesToFlag

        solType = sf.getType()

        # fill the queue (note that sf and sw cannot be put into a queue since they have file references)
        for vals, weights, coord, selection in sf.getValuesIter(returnAxes=axesToFlag, weight=True, reference=ref):
            mpm.put([vals, weights, coord, solType, order, mode, preflagzeros, maxCycles, fixRms, maxRms, replace, axesToFlag, selection])
            #v, w, sel = flag(vals, weights, coord, solType, order, mode, preflagzeros, maxCycles, fixRms, maxRms, replace, axesToFlag, selection)

        mpm.wait()
        
        for v, w, sel in mpm.get():
            sw.selection = sel
            if replace:
                # rewrite solutions (flagged values are overwritten)
                sw.setValues(v, weight=False)
            else:
                sw.setValues(w, weight=True)
        
        sw.flush()
        sw.addHistory('FLAG (over %s with %s sigma cut)' % (axesToFlag, maxRms))

        del sw
        del sf
        del soltab

    return 0
Example #45
0
def run( step, parset, H ):

    import os, random
    import numpy as np
    from losoto.h5parm import solFetcher, solHandler

    def normalize(phase):
        """
        Normalize phase to the range [-pi, pi].
        """
        # Convert to range [-2*pi, 2*pi].
        out = np.fmod(phase, 2.0 * np.pi)
        # Remove nans
        np.putmask(out, out!=out, 0)
        # Convert to range [-pi, pi]
        out[out < -np.pi] += 2.0 * np.pi
        out[out > np.pi] -= 2.0 * np.pi
        return out

    soltabs = getParSoltabs( step, parset, H )

    # 1- or 2-element array in form X, [Y]
    axesInPlot = parset.getStringVector('.'.join(["LoSoTo.Steps", step, "Axes"]), [] )
    NColFig = parset.getInt('.'.join(["LoSoTo.Steps", step, "Columns"]), 0 )
    figSize = parset.getIntVector('.'.join(["LoSoTo.Steps", step, "FigSize"]), [0,0] )
    minZ, maxZ = parset.getDoubleVector('.'.join(["LoSoTo.Steps", step, "MinMax"]), [0,0] )
    if minZ == 0: minZ = None
    if maxZ == 0: maxZ = None
    axisInTable = parset.getStringVector('.'.join(["LoSoTo.Steps", step, "TableAxis"]), [] )
    axisInCol = parset.getStringVector('.'.join(["LoSoTo.Steps", step, "ColorAxis"]), [] )
    axisInDiff = parset.getStringVector('.'.join(["LoSoTo.Steps", step, "DiffAxis"]), [] )
    # log='XYZ' to set which axes to put in Log
    log = parset.getString('.'.join(["LoSoTo.Steps", step, "Log"]), "" )
    plotflag = parset.getBool('.'.join(["LoSoTo.Steps", step, "PlotFlag"]), False )
    dounwrap = parset.getBool('.'.join(["LoSoTo.Steps", step, "Unwrap"]), False )
    ref = parset.getString('.'.join(["LoSoTo.Steps", step, "Reference"]), '' )
    tablesToAdd = parset.getStringVector('.'.join(["LoSoTo.Steps", step, "Add"]), [] )
    makeAntPlot = parset.getBool('.'.join(["LoSoTo.Steps", step, "MakeAntPlot"]), False )
    makeMovie = parset.getBool('.'.join(["LoSoTo.Steps", step, "MakeMovie"]), False )
    prefix = parset.getString('.'.join(["LoSoTo.Steps", step, "Prefix"]), '' )

    ncpu = parset.getInt('.'.join(["LoSoTo.Ncpu"]), 0 )
    if ncpu == 0:
        import multiprocessing
        ncpu = multiprocessing.cpu_count()

    if makeMovie: 
        prefix = prefix+'__tmp__'

    if os.path.dirname(prefix) != '' and not os.path.exists(os.path.dirname(prefix)):
        logging.debug('Creating '+os.path.dirname(prefix)+'.')
        os.makedirs(os.path.dirname(prefix))

    if ref == '': ref = None
    sfsAdd = [ solFetcher(soltab) for soltab in openSoltabs(H, tablesToAdd) ]

    for soltab in openSoltabs( H, soltabs ):

        logging.info("Plotting soltab: "+soltab._v_name)
        sf = solFetcher(soltab)

        # axis selection
        userSel = {}
        for axis in sf.getAxesNames():
            userSel[axis] = getParAxis( step, parset, H, axis )
        sf.setSelection(**userSel)

        # some checks
        for axis in axesInPlot:
            if axis not in sf.getAxesNames():
                logging.error('Axis \"'+axis+'\" not found.')
                return 1

        cmesh = False
        if len(axesInPlot) == 2:
            cmesh = True
            # not color possible in 3D
            axisInCol = []
        elif len(axesInPlot) != 1:
            logging.error('Axes must be a len 1 or 2 array.')
            return 1

        if len(set(axisInTable+axesInPlot+axisInCol+axisInDiff)) != len(axisInTable+axesInPlot+axisInCol+axisInDiff):
            logging.error('Axis defined multiple times.')
            return 1

        # just because we use lists, check that they are 1-d
        if len(axisInTable) > 1 or len(axisInCol) > 1 or len(axisInDiff) > 1:
            logging.error('Too many TableAxis/ColAxis/DiffAxis, they must be at most one each.')
            return 1

        # all axes that are not iterated by anything else
        axesInFile = sf.getAxesNames()
        for axis in axisInTable+axesInPlot+axisInCol+axisInDiff:
            axesInFile.remove(axis)
 
        # set subplots scheme
        if axisInTable != []:
            Nplots = sf.getAxisLen(axisInTable[0])
        else:
            Nplots = 1

        # prepare antennas coord in makeAntPlot case
        if makeAntPlot:
            if axesInPlot != ['ant']:
                logging.error('If makeAntPlot is selected the "Axes" values must be "ant"')
                return 1
            antCoords = [[],[]]
            for ant in sf.getAxisValues('ant'): # select only user-selected antenna in proper order
                antCoords[0].append(H.getAnt(sf.getAddress().split('/')[0])[ant][0])
                antCoords[1].append(H.getAnt(sf.getAddress().split('/')[0])[ant][1])
        else:
            antCoords = []
            
        datatype = sf.getType()

        # start processes for multi-thread
        mpm = multiprocManager(ncpu, plot)

        # cycle on files
        if makeMovie: pngs = [] # store png filenames
        for vals, coord, selection in sf.getValuesIter(returnAxes=axisInDiff+axisInTable+axisInCol+axesInPlot):
           
            # set filename
            filename = ''
            for axis in axesInFile:
                filename += axis+str(coord[axis])+'_'
            filename = filename[:-1] # remove last _

            # axis vals (they are always the same, regulat arrays)
            xvals = coord[axesInPlot[0]]
            # if plotting antenna - convert to number
            if axesInPlot[0] == 'ant':
                xvals = np.arange(len(xvals))
            
            # if plotting time - convert in h/min/s
            xlabelunit=''
            if axesInPlot[0] == 'time':
                if xvals[-1] - xvals[0] > 3600:
                    xvals = (xvals-xvals[0])/3600.  # hrs
                    xlabelunit = ' [hr]'
                elif xvals[-1] - xvals[0] > 60:
                    xvals = (xvals-xvals[0])/60.   # mins
                    xlabelunit = ' [min]'
                else:
                    xvals = (xvals-xvals[0])  # sec
                    xlabelunit = ' [s]'
            # if plotting freq convert in MHz
            elif axesInPlot[0] == 'freq': 
                xvals = xvals/1.e6 # MHz
                xlabelunit = ' [MHz]'

            if cmesh:
                # axis vals (they are always the same, regular arrays)
                yvals = coord[axesInPlot[1]]
                # same as above but for y-axis
                if axesInPlot[1] == 'ant':
                    yvals = np.arange(len(yvals))

                if len(xvals) <= 1 or len(yvals) <=1:
                    logging.error('3D plot must have more then one value per axes.')
                    mpm.wait()
                    return 1

                ylabelunit=''
                if axesInPlot[1] == 'time':
                    if yvals[-1] - yvals[0] > 3600:
                        yvals = (yvals-yvals[0])/3600.  # hrs
                        ylabelunit = ' [hr]'
                    elif yvals[-1] - yvals[0] > 60:
                        yvals = (yvals-yvals[0])/60.   # mins
                        ylabelunit = ' [min]'
                    else:
                        yvals = (yvals-yvals[0])  # sec
                        ylabelunit = ' [s]'
                elif axesInPlot[1] == 'freq':  # Mhz
                    yvals = yvals/1.e6
                    ylabelunit = ' [MHz]'
            else: 
                yvals = None
                ylabelunit = None

            sf2 = solFetcher(soltab)
            sf2.selection = selection
            # cycle on tables
            titles = []
            dataCube = []
            weightCube = []
            for Ntab, (vals, coord, selection) in enumerate(sf2.getValuesIter(returnAxes=axisInDiff+axisInCol+axesInPlot)):
                dataCube.append([])
                weightCube.append([])

                # set tile
                titles.append('')
                for axis in coord:
                    if axis in axesInFile+axesInPlot+axisInCol: continue
                    titles[Ntab] += axis+':'+str(coord[axis])+' '
                titles[Ntab] = titles[Ntab][:-1] # remove last ' '

                sf3 = solFetcher(soltab)
                sf3.selection = selection
                # cycle on colors

                for Ncol, (vals, weight, coord, selection) in enumerate(sf3.getValuesIter(returnAxes=axisInDiff+axesInPlot, weight=True, reference=ref)):
                    dataCube[Ntab].append([])
                    weightCube[Ntab].append([])
        
                    # differential plot
                    if axisInDiff != []:
                        # find ordered list of axis
                        names = [axis for axis in sf.getAxesNames() if axis in axisInDiff+axesInPlot]
                        if axisInDiff[0] not in names:
                            logging.error("Axis to differentiate (%s) not found." % axisInDiff[0])
                            mpm.wait()
                            return 1
                        if len(coord[axisInDiff[0]]) != 2:
                            logging.error("Axis to differentiate (%s) has too many values, only 2 is allowed." % axisInDiff[0])
                            mpm.wait()
                            return 1

                        # find position of interesting axis
                        diff_idx = names.index(axisInDiff[0])
                        # roll to first place
                        vals = np.rollaxis(vals,diff_idx,0)
                        vals = vals[0] - vals[1]
                        weight = np.rollaxis(weight,diff_idx,0)
                        weight = ((weight[0]==1) & (weight[1]==1))
                        del coord[axisInDiff[0]]
 

                    # add tables if required (e.g. phase/tec)
                    for sfAdd in sfsAdd:
                        newCoord = {}
                        for axisName in coord.keys():
                            if axisName in sfAdd.getAxesNames():
                                if type(coord[axisName]) is np.ndarray:
                                    newCoord[axisName] = coord[axisName]
                                else:
                                    newCoord[axisName] = [coord[axisName]] # avoid being interpreted as regexp, faster
                        sfAdd.setSelection(**newCoord)
                        valsAdd = np.squeeze(sfAdd.getValues(retAxesVals=False, weight=False, reference=ref))
                        if sfAdd.getType() == 'clock':
                            valsAdd = 2. * np.pi * valsAdd * newCoord['freq']
                        elif sfAdd.getType() == 'tec':
                            valsAdd = -8.44797245e9 * valsAdd / newCoord['freq']
                        else:
                            logging.warning('Only Clock or TEC can be added to solutions. Ignoring: '+sfAdd.getType()+'.')
                            continue

                        # If clock/tec are single pol then duplicate it (TODO)
                        # There still a problem with commonscalarphase and pol-dependant clock/tec
                        #but there's not easy way to combine them
                        if not 'pol' in sfAdd.getAxesNames() and 'pol' in sf.getAxesNames():
                            # find pol axis positions
                            polAxisPos = sf.getAxesNames().key_idx('pol')
                            # create a new axes for the table to add and duplicate the values
                            valsAdd = np.addaxes(valsAdd, polAxisPos)

                        if valsAdd.shape != vals.shape:
                            logging.error('Cannot combine the table '+sfAdd.getType()+' with '+sf4.getType()+'. Wrong shape.')
                            mpm.wait()
                            return 1

                        vals += valsAdd

                    # normalize
                    if (sf.getType() == 'phase' or sf.getType() == 'scalarphase'):
                        vals = normalize(vals)

                    # unwrap if required
                    if (sf.getType() == 'phase' or sf.getType() == 'scalarphase') and dounwrap:
                        vals = unwrap(vals)
                    
                    # is user requested axis in an order that is different from h5parm, we need to transpose
                    if len(axesInPlot) == 2:
                        if sf3.getAxesNames().index(axesInPlot[0]) < sf3.getAxesNames().index(axesInPlot[1]): vals = vals.T

                    dataCube[Ntab][Ncol] = np.ma.masked_array(vals, mask=(weight == 0))

            # if dataCube too large (> 500 MB) do not go parallel
            if np.array(dataCube).nbytes > 1024*1024*500: 
                logging.debug('Big plot, parallel not possible.')
                plot(Nplots, NColFig, figSize, cmesh, axesInPlot, axisInTable, xvals, yvals, xlabelunit, ylabelunit, datatype, prefix+filename, titles, log, dataCube, minZ, maxZ, plotflag, makeMovie, antCoords, None)
            else:
                mpm.put([Nplots, NColFig, figSize, cmesh, axesInPlot, axisInTable, xvals, yvals, xlabelunit, ylabelunit, datatype, prefix+filename, titles, log, dataCube, minZ, maxZ, plotflag, makeMovie, antCoords])
            if makeMovie: pngs.append(prefix+filename+'.png')

        mpm.wait()

        if makeMovie:
            def long_substr(strings):
                """
                Find longest common substring
                """
                substr = ''
                if len(strings) > 1 and len(strings[0]) > 0:
                    for i in range(len(strings[0])):
                        for j in range(len(strings[0])-i+1):
                            if j > len(substr) and all(strings[0][i:i+j] in x for x in strings):
                                substr = strings[0][i:i+j]
                return substr
            movieName = long_substr(pngs)
            assert movieName != '' # need a common prefix, use prefix keyword in case
            logging.info('Making movie: '+movieName)
            # make every movie last 20 sec, min one second per slide
            fps = np.ceil(len(pngs)/200.)
            ss="mencoder -ovc lavc -lavcopts vcodec=mpeg4:vpass=1:vbitrate=6160000:mbd=2:keyint=132:v4mv:vqmin=3:lumi_mask=0.07:dark_mask=0.2:"+\
                    "mpeg_quant:scplx_mask=0.1:tcplx_mask=0.1:naq -mf type=png:fps="+str(fps)+" -nosound -o "+movieName.replace('__tmp__','')+".mpg mf://"+movieName+"*  > mencoder.log 2>&1"
            os.system(ss)
            #print ss
            for png in pngs: os.system('rm '+png)

    return 0
Example #46
0
def run( step, parset, H ):
    """
    subtract a clock and/or tec from a phase.
    """
    import numpy as np
    from losoto.h5parm import solFetcher, solWriter

    soltabs = getParSoltabs( step, parset, H )
    soltabsToSub = parset.getStringVector('.'.join(["LoSoTo.Steps", step, "Sub"]), [] )
    ratio = parset.getBool('.'.join(["LoSoTo.Steps", step, "Ratio"]), False )

    for soltab in openSoltabs( H, soltabs ):
        logging.info("--> Working on soltab: "+soltab._v_name)

        sf = solFetcher(soltab)
        sw = solWriter(soltab, useCache = True)

        sfss = [] # sol fetcher to sub tables
        for soltabToSub in soltabsToSub:
            ss, st = soltabToSub.split('/')
            sfs = solFetcher(H.getSoltab(ss, st))
            if sf.getType() != 'phase' and (sfs.getType() == 'tec' or sfs.getType() == 'clock' or sfs.getType() == 'rotationmeasure' or sfs.getType() == 'tec3rd'):
                logging.warning(soltabToSub+' is of type clock/tec/rm and should be subtracted from a phase. Skipping it.')
                continue
            sfss.append( sfs )
            logging.info('Subtracting table: '+soltabToSub)

            # a major speed up if tables are assumed with same axes, check that (should be the case in almost any case)
            for axisName in sfs.getAxesNames():
                assert all(sfs.getAxisValues(axisName) == sf.getAxisValues(axisName))
        
        if sf.getType() == 'phase' and (sfs.getType() == 'tec' or sfs.getType() == 'clock' or sfs.getType() == 'rotationmeasure' or sfs.getType() == 'tec3rd' ):
            # the only return axes is freq, slower but better code
            for vals, weights, coord, selection in sf.getValuesIter(returnAxes='freq', weight = True):

                for sfs in sfss:

                    # restrict to have the same coordinates of phases
                    for i, axisName in enumerate(sfs.getAxesNames()):
                        sfs.selection[i] = selection[sf.getAxesNames().index(axisName)]

                    valsSub = np.squeeze(sfs.getValues(retAxesVals=False, weight=False))
                    weightsSub = np.squeeze(sfs.getValues(retAxesVals=False, weight=True))

                    if sfs.getType() == 'clock':
                        vals -= 2. * np.pi * valsSub * coord['freq']

                    elif sfs.getType() == 'tec':
                        vals -= -8.44797245e9 * valsSub / coord['freq']

                    elif sfs.getType() == 'tec3rd':
                        vals -= - 1.e21 * valsSub / np.power(coord['freq'],3)

                    elif sfs.getType() == 'rotationmeasure':
                        wav = 2.99792458e8/coord['freq']
                        ph = wav * wav * valsSub
                        if coord['pol'] == 'XX':
                            vals += ph
                        elif coord['pol'] == 'YY':
                            vals -= ph
                    else:
                        vals -= valsSub

                    # flag data that are contaminated by flagged clock/tec data
                    if weightsSub == 0: weights[:] = 0

                sw.selection = selection
                sw.setValues(vals)
                sw.setValues(weights, weight = True)
        else:
                if ratio: sw.setValues((sf.getValues(retAxesVals=False)-sfs.getValues(retAxesVals=False))/sfs.getValues(retAxesVals=False))
                else: sw.setValues(sf.getValues(retAxesVals=False)-sfs.getValues(retAxesVals=False))
                weight = sf.getValues(retAxesVals=False, weight=True)
                weight[sfs.getValues(retAxesVals=False, weight=True) == 0] = 0
                sw.setValues(weight, weight = True)
            
        sw.addHistory('RESIDUALS by subtracting tables '+' '.join(soltabsToSub))
        sw.flush()
        del sf
        del sw
        
    return 0
Example #47
0
def run( step, parset, H ):

    import os, random
    import numpy as np
    from losoto.h5parm import solFetcher, solHandler

    def normalize(phase):
        """
        Normalize phase to the range [-pi, pi].
        """
        # Convert to range [-2*pi, 2*pi].
        out = np.fmod(phase, 2.0 * np.pi)
        # Remove nans
        np.putmask(out, out!=out, 0)
        # Convert to range [-pi, pi]
        out[out < -np.pi] += 2.0 * np.pi
        out[out > np.pi] -= 2.0 * np.pi
        return out

    soltabs = getParSoltabs( step, parset, H )

    # 1- or 2-element array in form X, [Y]
    axesInPlot = parset.getStringVector('.'.join(["LoSoTo.Steps", step, "Axes"]), [] )
    NColFig = parset.getInt('.'.join(["LoSoTo.Steps", step, "Columns"]), 0 )
    figSize = parset.getIntVector('.'.join(["LoSoTo.Steps", step, "FigSize"]), [0,0] )
    minZ, maxZ = parset.getDoubleVector('.'.join(["LoSoTo.Steps", step, "MinMax"]), [0,0] )
    if minZ == 0: minZ = None
    if maxZ == 0: maxZ = None
    # the axis to plot on one page - e.g. ant to get all antenna's on one plot #
    axisInTable = parset.getStringVector('.'.join(["LoSoTo.Steps", step, "TableAxis"]), [] )
    # the axis to plot in different colours - e.g. pol to get all correlations on one plot #
    axisInCol = parset.getStringVector('.'.join(["LoSoTo.Steps", step, "ColorAxis"]), [] )
    # log='XYZ' to set which axes to put in Log
    log = parset.getString('.'.join(["LoSoTo.Steps", step, "Log"]), "" )
    plotflag = parset.getBool('.'.join(["LoSoTo.Steps", step, "PlotFlag"]), False )
    dounwrap = parset.getBool('.'.join(["LoSoTo.Steps", step, "Unwrap"]), False )
    ref = parset.getString('.'.join(["LoSoTo.Steps", step, "Reference"]), '' )
    tablesToAdd = parset.getStringVector('.'.join(["LoSoTo.Steps", step, "Add"]), [] )
    makeAntPlot = parset.getBool('.'.join(["LoSoTo.Steps", step, "MakeAntPlot"]), False )
    makeMovie = parset.getBool('.'.join(["LoSoTo.Steps", step, "MakeMovie"]), False )
    prefix = parset.getString('.'.join(["LoSoTo.Steps", step, "Prefix"]), '' )
    ncpu = parset.getInt('.'.join(["LoSoTo.Ncpu"]), 1 )

    if makeMovie: 
        prefix = prefix+'__tmp__'

    if os.path.dirname(prefix) != '' and not os.path.exists(os.path.dirname(prefix)):
        logging.debug('Creating '+os.path.dirname(prefix)+'.')
        os.makedirs(os.path.dirname(prefix))

    if ref == '': ref = None
    sfsAdd = [ solFetcher(soltab) for soltab in openSoltabs(H, tablesToAdd) ]

    # start processes for multi-thread
    mpm = multiprocManager(ncpu, plot)

    for soltab in openSoltabs( H, soltabs ):

        logging.info("Plotting soltab: "+soltab._v_name)
        sf = solFetcher(soltab)

        # axis selection
        userSel = {}
        for axis in sf.getAxesNames():
            userSel[axis] = getParAxis( step, parset, H, axis )
        sf.setSelection(**userSel)

        # some checks
        for axis in axesInPlot:
            if axis not in sf.getAxesNames():
                logging.error('Axis \"'+axis+'\" not found.')
                return 1

        cmesh = False
        if len(axesInPlot) == 2:
            cmesh = True
            # not color possible in 3D
            axisInCol = []
        elif len(axesInPlot) != 1:
            logging.error('Axes must be a len 1 or 2 array.')
            return 1

        if len(set(axisInTable+axesInPlot+axisInCol)) != len(axisInTable+axesInPlot+axisInCol):
            logging.error('Axis defined multiple times.')
            return 1

        if len(axisInTable) > 1 or len(axisInCol) > 1:
            logging.error('Too many AxisInTable/AxisInCol, they must be at most one each.')
            return 1

        # all axes that are not iterated by anything else
        axesInFile = sf.getAxesNames()
        for axis in axisInTable+axesInPlot+axisInCol:
            axesInFile.remove(axis)
 
        # set subplots scheme
        if axisInTable != []:
            Nplots = sf.getAxisLen(axisInTable[0])
        else:
            Nplots = 1

        # prepare antennas coord in makeAntPlot case
        if makeAntPlot:
            if axesInPlot != ['ant']:
                logging.error('If makeAntPlot is selected the "Axes" values must be "ant"')
                return 1
            antCoords = [[],[]]
            for ant in sf.getAxisValues('ant'): # select only user-selected antenna in proper order
                antCoords[0].append(H.getAnt(sf.getAddress().split('/')[0])[ant][0])
                antCoords[1].append(H.getAnt(sf.getAddress().split('/')[0])[ant][1])
        else:
            antCoords = []
            
        datatype = sf.getType()

        # cycle on files
        if makeMovie: pngs = [] # store png filenames
        for vals, coord, selection in sf.getValuesIter(returnAxes=axisInTable+axisInCol+axesInPlot):
            
            # set filename
            filename = ''
            for axis in axesInFile:
                filename += axis+str(coord[axis])+'_'
            filename = filename[:-1] # remove last _

            # axis vals (they are always the same, regulat arrays)
            xvals = coord[axesInPlot[0]]
            # if plotting antenna - convert to number
            if axesInPlot[0] == 'ant':
                xvals = np.arange(len(xvals))
            
            # if plotting time - convert in h/min/s
            xlabelunit=''
            if axesInPlot[0] == 'time':
                if len(xvals) > 3600:
                    xvals = (xvals-xvals[0])/3600.  # hrs
                    xlabelunit = ' [hr]'
                elif len(xvals) > 60:
                    xvals = (xvals-xvals[0])/60.   # mins
                    xlabelunit = ' [min]'
                else:
                    xvals = (xvals-xvals[0])  # sec
                    xlabelunit = ' [s]'
            # if plotting freq convert in MHz
            elif axesInPlot[0] == 'freq': 
                xvals = xvals/1.e6 # MHz
                xlabelunit = ' [MHz]'

            if cmesh:
                # axis vals (they are always the same, regular arrays)
                yvals = coord[axesInPlot[1]]
                # same as above but for y-axis
                if axesInPlot[1] == 'ant':
                    yvals = np.arange(len(yvals))

                if len(xvals) <= 1 or len(yvals) <=1:
                    logging.error('3D plot must have more then one value per axes.')
                    return 1

                ylabelunit=''
                if axesInPlot[1] == 'time':
                    if len(yvals) > 3600:
                        yvals = (yvals-yvals[0])/3600.  # hrs
                        ylabelunit = ' [hr]'
                    elif len(yvals) > 60:
                        yvals = (yvals-yvals[0])/60.   # mins
                        ylabelunit = ' [min]'
                    else:
                        yvals = (yvals-yvals[0])  # sec
                        ylabelunit = ' [s]'
                elif axesInPlot[1] == 'freq':  # Mhz
                    yvals = yvals/1.e6
                    ylabelunit = ' [MHz]'
            else: 
                yvals = None
                ylabelunit = None

            sf2 = solFetcher(soltab)
            sf2.selection = selection
            # cycle on tables
            titles = []
            dataCube = []
            weightCube = []
            for Ntab, (vals, coord, selection) in enumerate(sf2.getValuesIter(returnAxes=axisInCol+axesInPlot)):
                dataCube.append([])
                weightCube.append([])

                # set tile
                titles.append('')
                for axis in coord:
                    if axis in axesInFile+axesInPlot+axisInCol: continue
                    titles[Ntab] += axis+':'+str(coord[axis])+' '
                titles[Ntab] = titles[Ntab][:-1] # remove last ' '

                sf3 = solFetcher(soltab)
                sf3.selection = selection
                # cycle on colors

                for Ncol, (vals, weight, coord, selection) in enumerate(sf3.getValuesIter(returnAxes=axesInPlot, weight=True, reference=ref)):
                    dataCube[Ntab].append([])
                    weightCube[Ntab].append([])

                    # add tables if required (e.g. phase/tec)
                    for sfAdd in sfsAdd:
                        newCoord = {}
                        for axisName in coord.keys():
                            if axisName in sfAdd.getAxesNames():
                                if type(coord[axisName]) is np.ndarray:
                                    newCoord[axisName] = coord[axisName]
                                else:
                                    newCoord[axisName] = [coord[axisName]] # avoid being interpreted as regexp, faster
                        sfAdd.setSelection(**newCoord)
                        valsAdd = np.squeeze(sfAdd.getValues(retAxesVals=False, weight=False, reference=ref))
                        if sfAdd.getType() == 'clock':
                            valsAdd = 2. * np.pi * valsAdd * newCoord['freq']
                        elif sfAdd.getType() == 'tec':
                            valsAdd = -8.44797245e9 * valsAdd / newCoord['freq']
                        else:
                            logging.warning('Only Clock or TEC can be added to solutions. Ignoring: '+sfAdd.getType()+'.')
                            continue

                        # If clock/tec are single pol then duplicate it (TODO)
                        # There still a problem with commonscalarphase and pol-dependant clock/tec
                        #but there's not easy way to combine them
                        if not 'pol' in sfAdd.getAxesNames() and 'pol' in sf.getAxesNames():
                            # find pol axis positions
                            polAxisPos = sf.getAxesNames().key_idx('pol')
                            # create a new axes for the table to add and duplicate the values
                            valsAdd = np.addaxes(valsAdd, polAxisPos)

                        if valsAdd.shape != vals.shape:
                            logging.error('Cannot combine the table '+sfAdd.getType()+' with '+sf4.getType()+'. Wrong shape.')
                            return 1

                        vals += valsAdd

                    # normalize
                    if (sf.getType() == 'phase' or sf.getType() == 'scalarphase'):
                        vals = normalize(vals)

                    # unwrap if required
                    if (sf.getType() == 'phase' or sf.getType() == 'scalarphase') and dounwrap:
                        vals = unwrap(vals)

                    dataCube[Ntab][Ncol] = np.ma.masked_array(vals, mask=(weight == 0))

            # if dataCube too large (> 500 MB) do not go parallel
            if np.array(dataCube).nbytes > 1024*1024*500: 
                logging.debug('Big plot, parallel not possible.')
                plot(Nplots, NColFig, figSize, cmesh, axesInPlot, axisInTable, xvals, yvals, xlabelunit, ylabelunit, datatype, prefix+filename, titles, log, dataCube, minZ, maxZ, plotflag, makeMovie, antCoords, None)
            else:
                mpm.put([Nplots, NColFig, figSize, cmesh, axesInPlot, axisInTable, xvals, yvals, xlabelunit, ylabelunit, datatype, prefix+filename, titles, log, dataCube, minZ, maxZ, plotflag, makeMovie, antCoords])
            if makeMovie: pngs.append(prefix+filename+'.png')

        mpm.wait()

        if makeMovie:
            def long_substr(strings):
                """
                Find longest common substring
                """
                substr = ''
                if len(strings) > 1 and len(strings[0]) > 0:
                    for i in range(len(strings[0])):
                        for j in range(len(strings[0])-i+1):
                            if j > len(substr) and all(strings[0][i:i+j] in x for x in strings):
                                substr = strings[0][i:i+j]
                return substr
            movieName = long_substr(pngs)
            assert movieName != '' # need a common prefix, use prefix keyword in case
            logging.info('Making movie: '+movieName)
            # make every movie last 20 sec, min one second per slide
            fps = np.ceil(len(pngs)/200.)
            ss="mencoder -ovc lavc -lavcopts vcodec=mpeg4:vpass=1:vbitrate=6160000:mbd=2:keyint=132:v4mv:vqmin=3:lumi_mask=0.07:dark_mask=0.2:"+\
                    "mpeg_quant:scplx_mask=0.1:tcplx_mask=0.1:naq -mf type=png:fps="+str(fps)+" -nosound -o "+movieName.replace('__tmp__','')+".mpg mf://"+movieName+"*  > mencoder.log 2>&1"
            os.system(ss)
            #print ss
            for png in pngs: os.system('rm '+png)

    return 0