def run(step, parset, H): """ Copy values from a table to another (of the same kind) If tables have different sampling, then resample the values """ import numpy as np from losoto.h5parm import solFetcher, solWriter inTable = parset.getString('.'.join(["LoSoTo.Steps", step, "InTable"]), '') # complete solset/soltab outTable = parset.getString('.'.join(["LoSoTo.Steps", step, "OutTable"]), '') # complete solset/soltab or '' if inTable == '': logging.error('InTable is undefined.') return 1 if outTable == '': outSolsetName = inTable.split('/')[0] outTableName = None else: outSolsetName = outTable.split('/')[0] outTableName = outTable.split('/')[1] ss, st = inTable.split('/') sf = solFetcher(H.getSoltab(ss, st)) t = H.makeSoltab(solset = outSolsetName, soltype = sf.getType(), soltab = outTableName, axesNames=sf.getAxesNames(), \ axesVals=[sf.getAxisValues(axisName) for axisName in sf.getAxesNames()], \ vals=sf.getValues(retAxesVals = False), weights=sf.getValues(weight = True, retAxesVals = False), parmdbType=sf.t._v_attrs['parmdb_type']) sw = solWriter(t) sw.addHistory('DUPLICATE (from table %s)' % (inTable)) return 0
def run( step, parset, H ): import numpy as np from losoto.h5parm import solFetcher, solWriter soltabs = getParSoltabs( step, parset, H ) for soltab in openSoltabs( H, soltabs ): logging.info("Taking ABSolute value of soltab: "+soltab._v_name) sf = solFetcher(soltab) sw = solWriter(soltab) # axis selection userSel = {} for axis in sf.getAxesNames(): userSel[axis] = getParAxis( step, parset, H, axis ) sf.setSelection(**userSel) vals = sf.getValues(retAxesVals = False) count = np.count_nonzero(vals<0) logging.info('Abs: %i points initially negative (%f %%)' % (count,100*float(count)/np.count_nonzero(vals))) # writing back the solutions sw.setValues(np.abs(vals)) sw.addHistory('ABSolute value taken') return 0
def run(step, parset, H): import numpy as np from losoto.h5parm import solFetcher, solWriter soltabs = getParSoltabs(step, parset, H) for soltab in openSoltabs(H, soltabs): logging.info("Taking ABSolute value of soltab: " + soltab._v_name) sf = solFetcher(soltab) sw = solWriter(soltab) # axis selection userSel = {} for axis in sf.getAxesNames(): userSel[axis] = getParAxis(step, parset, H, axis) sf.setSelection(**userSel) vals = sf.getValues(retAxesVals=False) count = np.count_nonzero(vals < 0) logging.info('Abs: %i points initially negative (%f %%)' % (count, 100 * float(count) / np.count_nonzero(vals))) # writing back the solutions sw.setValues(np.abs(vals)) sw.addHistory('ABSolute value taken') return 0
def run(step, parset, H): from losoto.h5parm import solWriter soltabs = getParSoltabs(step, parset, H) for soltab in openSoltabs(H, soltabs): logging.info("Resetting soltab: " + soltab._v_name) t = solWriter(soltab) # axis selection userSel = {} for axis in t.getAxesNames(): userSel[axis] = getParAxis(step, parset, H, axis) t.setSelection(**userSel) solType = t.getType() if solType == 'amplitude': t.setValues(1.) else: t.setValues(0.) t.addHistory('RESET') return 0
def run( step, parset, H ): from losoto.h5parm import solWriter soltabs = getParSoltabs( step, parset, H ) for soltab in openSoltabs( H, soltabs ): logging.info("Resetting soltab: "+soltab._v_name) t = solWriter(soltab) # axis selection userSel = {} for axis in t.getAxesNames(): userSel[axis] = getParAxis( step, parset, H, axis ) t.setSelection(**userSel) solType = t.getType() if solType == 'amplitude': t.setValues(1.) else: t.setValues(0.) t.addHistory('RESET') return 0
def run( step, parset, H ): """ Copy values from a table to another (of the same kind) If tables have different sampling, then resample the values """ import numpy as np from losoto.h5parm import solFetcher, solWriter inTable = parset.getString('.'.join(["LoSoTo.Steps", step, "InTable"]), '' ) # complete solset/soltab outTable = parset.getString('.'.join(["LoSoTo.Steps", step, "OutTable"]), '' ) # complete solset/soltab or '' if inTable == '': logging.error('InTable is undefined.') return 1 if outTable == '': outSolsetName = inTable.split('/')[0] outTableName = None else: outSolsetName = outTable.split('/')[0] outTableName = outTable.split('/')[1] ss, st = inTable.split('/') sf = solFetcher(H.getSoltab(ss, st)) t = H.makeSoltab(solset = outSolsetName, soltype = sf.getType(), soltab = outTableName, axesNames=sf.getAxesNames(), \ axesVals=[sf.getAxisValues(axisName) for axisName in sf.getAxesNames()], \ vals=sf.getValues(retAxesVals = False), weights=sf.getValues(weight = True, retAxesVals = False), parmdbType=sf.t._v_attrs['parmdb_type']) sw = solWriter(t) sw.addHistory('DUPLICATE (from table %s)' % (inTable)) return 0
def plugin_main(args, **kwargs): fileid = kwargs['mapfile_in'] datamap = DataMap.load(fileid) hdf5File = os.path.join(kwargs['hdf5_dir'],kwargs['hdf5file']) if kwargs.has_key('instrument'): instrument = kwargs['instrument'] else: instrument = '/instrument' if kwargs.has_key('compression'): compression = int(kwargs['compression']) else: compression = 5 if kwargs.has_key('solset'): solsetName = kwargs['solset'] else: solsetName = None # Check is all the necessary files are available antennaFile = os.path.join(datamap[0].file,'ANTENNA') if not os.path.isdir(antennaFile): logging.critical('Missing ANTENNA table.') sys.exit(1) fieldFile = os.path.join(datamap[0].file,'FIELD') if not os.path.isdir(fieldFile): logging.critical('Missing FIELD table.') sys.exit(1) skydbFile = os.path.join(datamap[0].file,'sky') if not os.path.isdir(skydbFile): logging.critical('Missing sky table.') sys.exit(1) #generate list of parmDB-filenames parmDBnames = [ MS.file+instrument for MS in datamap ] #create and fill the hdf5-file: solset = parmDBs2h5parm(hdf5File, parmDBnames, antennaFile, fieldFile, skydbFile, compression=compression, solsetName=solsetName) # Add CREATE entry to history h5parmDB = h5parm(hdf5File, readonly = False) soltabs = h5parmDB.getSoltabs(solset=solset) for st in soltabs: sw = solWriter(soltabs[st]) sw.addHistory('CREATE (by PipelineStep_losotoImporter from %s / %s - %s)' % (os.path.abspath(''), os.path.basename(parmDBnames[0]), os.path.basename(parmDBnames[-1]) ) ) h5parmDB.close() #generate mapfile and wrap up mapfileentry = {} mapfileentry['host'] = 'localhost' mapfileentry['file'] = hdf5File mapfileentry['skip'] = False outfileid = os.path.join(kwargs['mapfile_dir'], kwargs['filename']) outmap = open(outfileid, 'w') outmap.write(repr([mapfileentry])) outmap.close() result = {} result['mapfile'] = outfileid return result
def run(step, parset, H): from losoto.h5parm import solWriter import numpy as np soltabs = getParSoltabs(step, parset, H) weightVal = parset.getFloat('.'.join(["LoSoTo.Steps", step, "WeightVal"]), 1.) mergeSoltab = parset.getString( '.'.join(["LoSoTo.Steps", step, "MergeFromSoltab"]), '') flagBad = parset.getBool('.'.join(["LoSoTo.Steps", step, "FlagBad"]), False) for soltab in openSoltabs(H, soltabs): logging.info("Reweighting soltab: " + soltab._v_name) sw = solWriter(soltab) # axis selection userSel = {} for axis in sw.getAxesNames(): userSel[axis] = getParAxis(step, parset, H, axis) sw.setSelection(**userSel) if mergeSoltab != '': mss, mst = mergeSoltab.split('/') msf = solFetcher(H.getSoltab(mss, mst)) msf.setSelection(**userSel) sf = solFetcher(soltab) sf.setSelection(**userSel) weights, axes = sf.getValues(weight=True) mergeWeights, mergeAxes = msf.getValues(weight=True) if axes.keys() != mergeAxes.keys( ) or weights.shape != mergeWeights.shape: logging.error( 'Impossible merge two tables with different axes values') return 1 weights[np.where(mergeWeights == 0)] = 0. sw.addHistory('WEIGHT merged from ' + mergeSoltab + ' for selection:' + str(userSel)) else: weights = weightVal sw.addHistory('REWEIGHTED to ' + str(weightVal) + ' for selection:' + str(userSel)) sw.setValues(weights, weight=True) if flagBad: sf = solFetcher(soltab) sf.setSelection(**userSel) weights = sf.getValues(weight=True, retAxesVals=False) vals = sf.getValues(retAxesVals=False) if sf.getType() == 'amplitude': weights[np.where(vals == 1)] = 0 else: weights[np.where(vals == 0)] = 0 sw.setValues(weights, weight=True) return 0
def run(step, parset, H): import scipy.ndimage.filters import numpy as np from losoto.h5parm import solFetcher, solWriter soltabs = getParSoltabs(step, parset, H) s = parset.getIntVector('.'.join(["LoSoTo.Steps", step, "smooth"]), 10) for soltab in openSoltabs(H, soltabs): logging.info("Smoothing soltab: " + soltab._v_name) sf = solFetcher(soltab) sw = solWriter(soltab) if sf.getType() != 'clock': logging.error( 'Only clock-type solutions can be run in this operation.') return 1 # axis selection userSel = {} for axis in sf.getAxesNames(): userSel[axis] = getParAxis(step, parset, H, axis) sf.setSelection(**userSel) for vals, weights, coord, selection in sf.getValuesIter( returnAxes='time', weight=True): x = coord['time'][weights != 0] y = vals[weights != 0] weights = weights[weights != 0] spline = scipy.interpolate.UnivariateSpline(x, y, weights, k=1, s=1e-15) plot = True if plot: import matplotlib as mpl mpl.use("Agg") import matplotlib.pyplot as plt plt.plot(x, y, 'k.') plt.plot(x, spline(x), 'r-') plt.savefig('test.png') sys.exit(1) sw.selection = selection sw.setValues(spline(coord['time'])) sw.addHistory('Smoothed with SMOOTHCLOCK.') del sf del sw return 0
def run( step, parset, H ): from losoto.h5parm import solFetcher, solWriter soltabs = getParSoltabs( step, parset, H ) axesToExt = parset.getStringVector('.'.join(["LoSoTo.Steps", step, "Axes"]), ['freq','time'] ) size = parset.getIntVector('.'.join(["LoSoTo.Steps", step, "Size"]), [11,11] ) percent = parset.getFloat('.'.join(["LoSoTo.Steps", step, "Percent"]), 50 ) cycles = parset.getInt('.'.join(["LoSoTo.Steps", step, "Cycles"]), 3 ) ncpu = parset.getInt('.'.join(["LoSoTo.Ncpu"]), 0 ) if ncpu == 0: import multiprocessing ncpu = multiprocessing.cpu_count() if axesToExt == []: logging.error("Please specify at least one axis to extend flag.") return 1 for soltab in openSoltabs( H, soltabs ): # start processes for multi-thread mpm = multiprocManager(ncpu, flag) logging.info("Extending flag on soltab: "+soltab._v_name) sf = solFetcher(soltab) sw = solWriter(soltab) # axis selection userSel = {} for axis in sf.getAxesNames(): userSel[axis] = getParAxis( step, parset, H, axis ) sf.setSelection(**userSel) for axisToExt in axesToExt: if axisToExt not in sf.getAxesNames(): logging.error('Axis \"'+axisToExt+'\" not found.') mpm.wait() return 1 # fill the queue (note that sf and sw cannot be put into a queue since they have file references) for vals, weights, coord, selection in sf.getValuesIter(returnAxes=axesToExt, weight=True): mpm.put([weights, coord, axesToExt, selection, percent, size, cycles]) mpm.wait() logging.info('Writing solutions') for w,sel in mpm.get(): sw.selection = sel sw.setValues(w, weight=True) # convert back to np.float16 sw.addHistory('FLAG EXTENDED (over %s)' % (str(axesToExt))) del sf del sw return 0
def run(step, parset, H): from losoto.h5parm import solFetcher, solWriter soltabs = getParSoltabs(step, parset, H) axesToExt = parset.getStringVector( '.'.join(["LoSoTo.Steps", step, "Axes"]), ['freq', 'time']) size = parset.getIntVector('.'.join(["LoSoTo.Steps", step, "Size"]), [11, 11]) percent = parset.getFloat('.'.join(["LoSoTo.Steps", step, "Percent"]), 50) cycles = parset.getInt('.'.join(["LoSoTo.Steps", step, "Cycles"]), 3) ncpu = parset.getInt('.'.join(["LoSoTo.Ncpu"]), 1) if axesToExt == []: logging.error("Please specify at least one axis to extend flag.") return 1 # start processes for multi-thread mpm = multiprocManager(ncpu, flag) for soltab in openSoltabs(H, soltabs): logging.info("Extending flag on soltab: " + soltab._v_name) sf = solFetcher(soltab) sw = solWriter(soltab) # axis selection userSel = {} for axis in sf.getAxesNames(): userSel[axis] = getParAxis(step, parset, H, axis) sf.setSelection(**userSel) for axisToExt in axesToExt: if axisToExt not in sf.getAxesNames(): logging.error('Axis \"' + axisToExt + '\" not found.') return 1 # fill the queue (note that sf and sw cannot be put into a queue since they have file references) for vals, weights, coord, selection in sf.getValuesIter( returnAxes=axesToExt, weight=True): mpm.put( [weights, coord, axesToExt, selection, percent, size, cycles]) mpm.wait() logging.info('Writing solutions') for w, sel in mpm.get(): sw.selection = sel sw.setValues(w, weight=True) # convert back to np.float16 sw.addHistory('FLAG EXTENDED (over %s)' % (str(axesToExt))) del sf del sw return 0
def main(msfileslist, hdf5fileName, hdf5_dir='.', instrument='/instrument', solsetName=None, compression=5): tmp_msfiles = msfileslist.lstrip('[').rstrip(']').split(',') msfiles = [MS.strip("\' \"") for MS in tmp_msfiles] hdf5File = os.path.join(hdf5_dir, hdf5fileName) compression = int(compression) #doesn't hurt if it is already an int instrument = instrument.strip() # Check is all the necessary files are available antennaFile = os.path.join(msfiles[0], 'ANTENNA') if not os.path.isdir(antennaFile): logging.critical('Missing ANTENNA table.') sys.exit(1) fieldFile = os.path.join(msfiles[0], 'FIELD') if not os.path.isdir(fieldFile): logging.critical('Missing FIELD table.') sys.exit(1) skydbFile = os.path.join(msfiles[0], 'sky') if not os.path.isdir(skydbFile): logging.warning( 'No sky table found. (Direction-dependent parameters will not work.)' ) skydbFile = None #generate list of parmDB-filenames parmDBnames = [MS + instrument for MS in msfiles] #create and fill the hdf5-file: solset = parmDBs2h5parm(hdf5File, parmDBnames, antennaFile, fieldFile, skydbFile, compression=compression, solsetName=solsetName) # Add CREATE entry to history h5parmDB = h5parm(hdf5File, readonly=False) soltabs = h5parmDB.getSoltabs(solset=solset) for st in soltabs: sw = solWriter(soltabs[st]) sw.addHistory('CREATE (by losotoImporter from %s / %s - %s)' % (os.path.abspath(''), os.path.basename( parmDBnames[0]), os.path.basename(parmDBnames[-1]))) h5parmDB.close() result = {} result['h5parm'] = hdf5File return result
def run( step, parset, H ): from losoto.h5parm import solWriter import numpy as np soltabs = getParSoltabs( step, parset, H ) weightVal = parset.getFloat('.'.join(["LoSoTo.Steps", step, "WeightVal"]), 1. ) mergeSoltab = parset.getString('.'.join(["LoSoTo.Steps", step, "MergeFromSoltab"]), '' ) flagBad = parset.getBool('.'.join(["LoSoTo.Steps", step, "FlagBad"]), False ) for soltab in openSoltabs( H, soltabs ): logging.info("Reweighting soltab: "+soltab._v_name) sw = solWriter(soltab) # axis selection userSel = {} for axis in sw.getAxesNames(): userSel[axis] = getParAxis( step, parset, H, axis ) sw.setSelection(**userSel) if mergeSoltab != '': mss, mst = mergeSoltab.split('/') msf = solFetcher(H.getSoltab(mss, mst)) msf.setSelection(**userSel) sf = solFetcher(soltab) sf.setSelection(**userSel) weights, axes = sf.getValues(weight = True) mergeWeights, mergeAxes = msf.getValues(weight = True) if axes.keys() != mergeAxes.keys() or weights.shape != mergeWeights.shape: logging.error('Impossible merge two tables with different axes values') return 1 weights[ np.where(mergeWeights == 0) ] = 0. sw.addHistory('WEIGHT merged from '+mergeSoltab+' for selection:'+str(userSel)) else: weights = weightVal sw.addHistory('REWEIGHTED to '+str(weightVal)+' for selection:'+str(userSel)) sw.setValues(weights, weight=True) if flagBad: sf = solFetcher(soltab) sf.setSelection(**userSel) weights = sf.getValues(weight = True, retAxesVals = False) vals = sf.getValues(retAxesVals = False) if sf.getType() == 'amplitude': weights[np.where(vals == 1)] = 0 else: weights[np.where(vals == 0)] = 0 sw.setValues(weights, weight=True) return 0
def run( step, parset, H ): import scipy.ndimage.filters import numpy as np from losoto.h5parm import solFetcher, solWriter soltabs = getParSoltabs( step, parset, H ) s = parset.getIntVector('.'.join(["LoSoTo.Steps", step, "smooth"]), 10 ) for soltab in openSoltabs( H, soltabs ): logging.info("Smoothing soltab: "+soltab._v_name) sf = solFetcher(soltab) sw = solWriter(soltab) if sf.getType() != 'clock': logging.error('Only clock-type solutions can be run in this operation.') return 1 # axis selection userSel = {} for axis in sf.getAxesNames(): userSel[axis] = getParAxis( step, parset, H, axis ) sf.setSelection(**userSel) for vals, weights, coord, selection in sf.getValuesIter(returnAxes='time', weight=True): x=coord['time'][weights != 0] y=vals[weights != 0] weights = weights[weights != 0] spline = scipy.interpolate.UnivariateSpline(x, y, weights, k=1, s=1e-15) plot = True if plot: import matplotlib as mpl mpl.use("Agg") import matplotlib.pyplot as plt plt.plot(x, y, 'k.') plt.plot(x, spline(x), 'r-') plt.savefig('test.png') sys.exit(1) sw.selection = selection sw.setValues(spline(coord['time'])) sw.addHistory('Smoothed with SMOOTHCLOCK.') del sf del sw return 0
def run(step, parset, H): """ Normalize the solutions to a given value """ import numpy as np from losoto.h5parm import solFetcher, solWriter soltabs = getParSoltabs(step, parset, H) solTypes = getParSolTypes(step, parset, H) normVal = parset.getFloat('.'.join(["LoSoTo.Steps", step, "NormVal"]), 1.) normAxes = parset.getStringVector( '.'.join(["LoSoTo.Steps", step, "NormAxes"]), ['time']) for soltab in openSoltabs(H, soltabs): logging.info("Normalizing soltab: " + soltab._v_name) tr = solFetcher(soltab) tw = solWriter(soltab, useCache=True) # remember to flush! axesNames = tr.getAxesNames() for normAxis in normAxes: if normAxis not in axesNames: logging.error('Normalization axis ' + normAxis + ' not found.') return 1 # axis selection userSel = {} for axis in tr.getAxesNames(): userSel[axis] = getParAxis(step, parset, H, axis) tr.setSelection(**userSel) for vals, weights, coord, selection in tr.getValuesIter( returnAxes=normAxes, weight=True): # rescale solutions if np.sum(weights) == 0: continue # skip flagged antenna valsMean = np.average(vals, weights=weights) valsNew = normVal * vals / valsMean logging.debug(str(coord)) logging.debug("Rescaling by: " + str(normVal / valsMean)) # writing back the solutions tw.selection = selection tw.setValues(valsNew) tw.flush() tw.addHistory('NORM (on axis %s)' % (normAxes)) return 0
def run( step, parset, H ): """ Normalize the solutions to a given value """ import numpy as np from losoto.h5parm import solFetcher, solWriter soltabs = getParSoltabs( step, parset, H ) solTypes = getParSolTypes( step, parset, H ) normVal = parset.getFloat('.'.join(["LoSoTo.Steps", step, "NormVal"]), 1. ) normAxes = parset.getStringVector('.'.join(["LoSoTo.Steps", step, "NormAxes"]), ['time'] ) for soltab in openSoltabs( H, soltabs ): logging.info("Normalizing soltab: "+soltab._v_name) tr = solFetcher(soltab) tw = solWriter(soltab, useCache = True) # remember to flush! axesNames = tr.getAxesNames() for normAxis in normAxes: if normAxis not in axesNames: logging.error('Normalization axis '+normAxis+' not found.') return 1 # axis selection userSel = {} for axis in tr.getAxesNames(): userSel[axis] = getParAxis( step, parset, H, axis ) tr.setSelection(**userSel) for vals, weights, coord, selection in tr.getValuesIter(returnAxes=normAxes, weight = True): # rescale solutions if np.sum(weights) == 0: continue # skip flagged selections valsMean = np.average(vals, weights=weights) vals[weights != 0] *= normVal/valsMean logging.debug(str(coord)) logging.debug("Rescaling by: "+str(normVal/valsMean)) # writing back the solutions tw.selection = selection tw.setValues(vals) tw.flush() tw.addHistory('NORM (on axis %s)' % (normAxes)) return 0
def main(msfileslist, hdf5fileName, hdf5_dir='.', instrument='/instrument', solsetName=None, compression=5): tmp_msfiles = msfileslist.lstrip('[').rstrip(']').split(',') msfiles = [ MS.strip("\' \"") for MS in tmp_msfiles] hdf5File = os.path.join(hdf5_dir,hdf5fileName) compression = int(compression) #doesn't hurt if it is already an int instrument = instrument.strip() # Check is all the necessary files are available antennaFile = os.path.join(msfiles[0],'ANTENNA') if not os.path.isdir(antennaFile): logging.critical('Missing ANTENNA table.') sys.exit(1) fieldFile = os.path.join(msfiles[0],'FIELD') if not os.path.isdir(fieldFile): logging.critical('Missing FIELD table.') sys.exit(1) skydbFile = os.path.join(msfiles[0],'sky') if not os.path.isdir(skydbFile): logging.critical('Missing sky table.') sys.exit(1) #generate list of parmDB-filenames parmDBnames = [ MS+instrument for MS in msfiles ] #create and fill the hdf5-file: solset = parmDBs2h5parm(hdf5File, parmDBnames, antennaFile, fieldFile, skydbFile, compression=compression, solsetName=solsetName) # Add CREATE entry to history h5parmDB = h5parm(hdf5File, readonly = False) soltabs = h5parmDB.getSoltabs(solset=solset) for st in soltabs: sw = solWriter(soltabs[st]) sw.addHistory('CREATE (by losotoImporter from %s / %s - %s)' % (os.path.abspath(''), os.path.basename(parmDBnames[0]), os.path.basename(parmDBnames[-1]) ) ) h5parmDB.close() result = {} result['h5parm'] = hdf5File return result
def run(step, parset, H): import scipy.ndimage.filters import numpy as np from losoto.h5parm import solFetcher, solWriter soltabs = getParSoltabs(step, parset, H) axesToSmooth = parset.getStringVector( '.'.join(["LoSoTo.Steps", step, "Axes"]), []) FWHM = parset.getIntVector('.'.join(["LoSoTo.Steps", step, "FWHM"]), []) mode = parset.getString('.'.join(["LoSoTo.Steps", step, "Mode"]), "runningmedian") if mode == "runningmedian" and len(axesToSmooth) != len(FWHM): logging.error("Axes and FWHM lenghts must be equal.") return 1 if mode == "runningmedian": logging.warning('Flagged data are still taken into account!') if FWHM != [] and mode != "runningmedian": logging.warning( "FWHM makes sense only with runningmedian mode, ignoring it.") for soltab in openSoltabs(H, soltabs): logging.info("Smoothing soltab: " + soltab._v_name) sf = solFetcher(soltab) sw = solWriter(soltab, useCache=True) # remember to flush! # axis selection userSel = {} for axis in sf.getAxesNames(): userSel[axis] = getParAxis(step, parset, H, axis) sf.setSelection(**userSel) for i, axis in enumerate(axesToSmooth[:]): if axis not in sf.getAxesNames(): del axesToSmooth[i] del FWHM[i] logging.warning('Axis \"' + axis + '\" not found. Ignoring.') for vals, weights, coord, selection in sf.getValuesIter( returnAxes=axesToSmooth, weight=True): if mode == 'runningmedian': valsnew = scipy.ndimage.filters.median_filter(vals, FWHM) elif mode == 'median': valsnew = np.median(vals[(weights != 0)]) elif mode == 'mean': valsnew = np.mean(vals[(weights != 0)]) else: logging.error('Mode must be: runningmedian, median or mean') return 1 sw.selection = selection sw.setValues(valsnew) sw.flush() sw.addHistory('SMOOTH (over %s with mode = %s)' % (axesToSmooth, mode)) del sf del sw return 0
def run( step, parset, H ): from losoto.h5parm import solFetcher, solWriter soltabs = getParSoltabs( step, parset, H ) #check_parset('Axes','MaxCycles','MaxRms','Order','Replace','PreFlagZeros') axesToFlag = parset.getStringVector('.'.join(["LoSoTo.Steps", step, "Axes"]), 'time' ) maxCycles = parset.getInt('.'.join(["LoSoTo.Steps", step, "MaxCycles"]), 5 ) maxRms = parset.getFloat('.'.join(["LoSoTo.Steps", step, "MaxRms"]), 5. ) fixRms = parset.getFloat('.'.join(["LoSoTo.Steps", step, "FixRms"]), 0 ) order = parset.getIntVector('.'.join(["LoSoTo.Steps", step, "Order"]), 3 ) replace = parset.getBool('.'.join(["LoSoTo.Steps", step, "Replace"]), False ) preflagzeros = parset.getBool('.'.join(["LoSoTo.Steps", step, "PreFlagZeros"]), False ) mode = parset.getString('.'.join(["LoSoTo.Steps", step, "Mode"]), 'smooth' ) ref = parset.getString('.'.join(["LoSoTo.Steps", step, "Reference"]), '' ) ncpu = parset.getInt('.'.join(["LoSoTo.Ncpu"]), 1 ) if ref == '': ref = None if axesToFlag == []: logging.error("Please specify axis to flag. It must be a single one.") return 1 if len(axesToFlag) != len(order): logging.error("AxesToFlag and order must be both 1 or 2 values.") return 1 if len(order) == 1: order = order[0] elif len(order) == 2: order = tuple(order) mode = mode.lower() if mode != 'smooth' and mode != 'poly' and mode != 'spline': logging.error('Mode must be smooth, poly or spline') return 1 for soltab in openSoltabs( H, soltabs ): # start processes for multi-thread mpm = multiprocManager(ncpu, flag) logging.info("Flagging soltab: "+soltab._v_name) sf = solFetcher(soltab) sw = solWriter(soltab, useCache=True) # remember to flush! # axis selection userSel = {} for axis in sf.getAxesNames(): userSel[axis] = getParAxis( step, parset, H, axis ) sf.setSelection(**userSel) for axisToFlag in axesToFlag: if axisToFlag not in sf.getAxesNames(): logging.error('Axis \"'+axis+'\" not found.') mpm.wait() return 1 # reorder axesToFlag as axes in the table axesToFlag_orig = axesToFlag axesToFlag = [coord for coord in sf.getAxesNames() if coord in axesToFlag] if type(order) is int: order = [order] if axesToFlag_orig != axesToFlag: order = order[::-1] # reverse order if we changed axesToFlag solType = sf.getType() # fill the queue (note that sf and sw cannot be put into a queue since they have file references) for vals, weights, coord, selection in sf.getValuesIter(returnAxes=axesToFlag, weight=True, reference=ref): mpm.put([vals, weights, coord, solType, order, mode, preflagzeros, maxCycles, fixRms, maxRms, replace, axesToFlag, selection]) #v, w, sel = flag(vals, weights, coord, solType, order, mode, preflagzeros, maxCycles, fixRms, maxRms, replace, axesToFlag, selection) mpm.wait() for v, w, sel in mpm.get(): sw.selection = sel if replace: # rewrite solutions (flagged values are overwritten) sw.setValues(v, weight=False) else: sw.setValues(w, weight=True) sw.flush() sw.addHistory('FLAG (over %s with %s sigma cut)' % (axesToFlag, maxRms)) del sw del sf del soltab return 0
elapsed = (time.clock() - start) logging.info("PARMDB -- " + str(elapsed) + " s.") start = time.clock() for i in xrange(n): H.setSelection(dir='pointing', ant='RS*') Hrot = H.getValues(retAxesVals=False) elapsed = (time.clock() - start) logging.info("H5parm -- " + str(elapsed) + " s.") #print "Equal?", (Prot == np.squeeze(Hrot)).all() ###################################################### # read+write logging.info("### Read all rotations for a dir/station and write them back") Hw = solWriter(H5.getSoltab(solset, 'amplitude000')) start = time.clock() for i in xrange(n): Prot = P.getValuesGrid('CommonRotationAngle:CS001LBA') Prot = {'test' + str(i): Prot['CommonRotationAngle:CS001LBA']} P2.addValues(Prot) # parmdb write? elapsed = (time.clock() - start) logging.info("PARMDB -- " + str(elapsed) + " s.") start = time.clock() for i in xrange(n): H.setSelection(dir='pointing', ant='CS001LBA') Hrot = H.getValues(retAxesVals=False) Hw.setSelection(dir='pointing', ant='CS001LBA')
def run(step, parset, H): """ Generic unspecified step for easy expansion. """ import numpy as np from losoto.h5parm import solFetcher, solWriter # all the following are LoSoTo function to extract information from the parset # get involved solsets using local step values or global values or all solsets = getParSolsets(step, parset, H) logging.info('Solset: ' + str(solsets)) # get involved soltabs using local step values or global values or all soltabs = getParSoltabs(step, parset, H) logging.info('Soltab: ' + str(soltabs)) # get list of SolTypes using local step values or global values or all solTypes = getParSolTypes(step, parset, H) logging.info('SolType: ' + str(solTypes)) # do something on every soltab (use the openSoltab LoSoTo function) for soltab in openSoltabs(H, soltabs): logging.info("--> Working on soltab: " + soltab._v_name) # use the solFetcher from the H5parm lib t = solFetcher(soltab) tw = solWriter(soltab) axisNames = t.getAxesNames() logging.info("Axis names are: " + str(axisNames)) solType = t.getType() logging.info("Soltab type is: " + solType) # this will make a selection for the getValues() and getValuesIter() # interpret every entry in the parset which has an axis name as a selector userSel = {} for axis in t.getAxesNames(): userSel[axis] = getParAxis(step, parset, H, axis) t.setSelection(**userSel) t.setSelection(ant=ants, pol=pols, dir=dirs) logging.info("Selection is: " + str(t.selection)) # find axis values logging.info("Antennas (no selection) are: " + str(t.getAxisValues('ant', ignoreSelection=True))) logging.info("Antennas (with selection) are: " + str(t.getAxisValues('ant'))) # but one can also use (selection is active here!) logging.info("Antennas (other method) are: " + str(t.ant)) logging.info("Frequencies are: " + str(t.freq)) logging.info("Directions are: " + str(t.dir)) logging.info("Polarizations are: " + str(t.pol)) # try to access a non-existent axis t.getAxisValues('nonexistantaxis') # now get all values given this selection logging.info("Get data using t.val") val = t.val logging.debug('shape of val: ' + str(t.val.shape)) logging.info("$ val is " + str(val[0, 0, 0, 0, 100])) weight = t.weight time = t.time thisTime = t.time[100] # another way to get the data is using the getValues() logging.info("Get data using getValues()") grid, axes = t.getValues() # axis names logging.info("Axes: " + str(t.getAxesNames())) # axis shape print axes print[t.getAxisLen(axis) for axis in axes] # not ordered, is a dict! # data array shape (same of axis shape) logging.info("Shape of values: " + str(grid.shape)) #logging.info("$ val is "+str(grid[0,0,0,0,100])) # reset selection t.setSelection() logging.info('Reset selection to \'\'') logging.info("Antennas are: " + str(t.ant)) logging.info("Frequencies are: " + str(t.freq)) logging.info("Directions are: " + str(t.dir)) logging.info("Polarizations are: " + str(t.pol)) # finally the getValuesIter allaws to iterate across all possible combinations of a set of axes logging.info('Iteration on time/freq') for vals, coord, selection in t.getValuesIter( returnAxes=['time', 'freq']): # writing back the solutions tw.selection = selection tw.setValues(vals) logging.info('Iteration on time') for vals, coord, selection in t.getValuesIter(returnAxes=['time']): # writing back the solutions tw.selection = selection tw.setValues(vals) logging.info('Iteration on dir after selection to 1 dir') t.setSelection(dir='pointing') for vals, coord, selection in t.getValuesIter(returnAxes=['dir']): # writing back the solutions tw.selection = selection tw.setValues(vals) return 0 # if everything went fine, otherwise 1
def run( step, parset, H ): import scipy.ndimage.filters import numpy as np from losoto.h5parm import solFetcher, solWriter from scipy.optimize import minimize import itertools from scipy.interpolate import griddata import scipy.cluster.vq as vq def robust_std(data, sigma=3): """ Calculate standard deviation excluding outliers ok with masked arrays """ return np.std(data[np.where(np.abs(data) < sigma * np.std(data))]) def mask_interp(vals, mask, method='nearest'): """ return interpolated values for masked elements """ this_vals = vals.copy() #this_vals[mask] = np.interp(np.where(mask)[0], np.where(~mask)[0], vals[~mask]) #this_vals[mask] = griddata(np.where(~mask)[0], vals[~mask], np.where(mask)[0], method) # griddata has nan bug with nearest, I need to use vq code, dist = vq.vq(np.where(mask)[0], np.where(~mask)[0]) this_vals[ np.where(mask)[0] ] = this_vals[code] return this_vals tec_jump_val = 0.019628 maxsize = 300 clip = 10 # TECs over these amount of jumps are flagged soltabs = getParSoltabs( step, parset, H ) for soltab in openSoltabs( H, soltabs ): logging.info("Removing TEC jumps from soltab: "+soltab._v_name) sf = solFetcher(soltab) sw = solWriter(soltab) # remember to flush! # TODO: check if it's a Tec table # axis selection userSel = {} for axis in sf.getAxesNames(): userSel[axis] = getParAxis( step, parset, H, axis ) sf.setSelection(**userSel) for vals, weights, coord, selection in sf.getValuesIter(returnAxes='time', weight=True): # skip all flagged if (weights == 0).all(): continue # skip reference if (np.diff(vals[(weights == 1)]) == 0).all(): continue # kill large values # weights[abs(vals/tec_jump_val)>clip] = 0 # interpolate flagged values to get resonable distances # vals = mask_interp(vals, mask=(weights == 0))/tec_jump_val # add edges to allow intervals to the borders # vals = np.insert(vals, 0, vals[0]) # vals = np.insert(vals, len(vals), vals[-1]) vals = np.fmod(vals,tec_jump_val) # def find_jumps(d_vals): # # jump poistion finder # d_smooth = scipy.ndimage.filters.median_filter( mask_interp(d_vals, mask=(abs(d_vals)>0.8)), 21 ) # d_vals -= d_smooth # jumps = list(np.where(np.abs(d_vals) > 1.)[0]) # return [0]+jumps+[len(d_vals)-1] # add edges # # class Jump(object): # def __init__(self, jumps_idx, med): # self.idx_left = jumps_idx[0] # self.idx_right = jumps_idx[1] # self.jump_left = np.rint(d_vals[self.idx_left]) # self.jump_right = np.rint(d_vals[self.idx_right]) # self.size = self.idx_right-self.idx_left # self.hight = np.median(vals[self.idx_left+1:self.idx_right+1]-med) # if abs((self.hight-self.jump_left)-med) > abs((self.hight-self.jump_right)-med): # self.closejump = self.jump_right # else: # self.closejump = self.jump_left # # i = 0 # while i<len(coord['time']): # # get tec[i] - tec[i+1], i.e. the derivative assuming constant timesteps # # this is in units of tec_jump_val! # d_vals = np.diff(vals) # # get jumps idx, idx=n means a jump beteen val n and n+1 # jumps_idx = find_jumps(d_vals) # # # get regions # med = np.median(vals) # jumps = [Jump(jump_idx, med) for jump_idx in zip( jumps_idx[:-1], jumps_idx[1:] )] # jumps = [jump for jump in jumps if jump.closejump != 0] # jumps = [jump for jump in jumps if jump.size != 0] # prevent bug on edges # jumps = [jump for jump in jumps if jump.size < maxsize] # # jumps.sort(key=lambda x: (np.abs(x.size), x.hight), reverse=False) #smallest first # #print [(j.hight, j.closejump) for j in jumps] # # plot = False # if plot: # import matplotlib.pyplot as plt # fig, ((ax1, ax2, ax3)) = plt.subplots(3, 1, sharex=True) # fig.subplots_adjust(hspace=0) # d_smooth = scipy.ndimage.filters.median_filter( mask_interp(d_vals, mask=(abs(d_vals)>0.8)), 31 ) # ax1.plot(d_vals,'k-') # ax2.plot(d_smooth,'k-') # ax3.plot(vals, 'k-') # [ax3.axvline(jump_idx+0.5, color='r', ls=':') for jump_idx in jumps_idx] # ax1.set_ylabel('d_vals') # ax2.set_ylabel('d_vals - smooth') # ax3.set_ylabel('TEC/jump') # ax3.set_xlabel('timestep') # ax1.set_xlim(xmin=-10, xmax=len(d_smooth)+10) # fig.savefig('plots/%stecjump_debug_%03i' % (coord['ant'], i)) # i+=1 # # if len(jumps) == 0: # break # # # move down the highest to the side closest to the median # j = jumps[0] # #print j.idx_left, j.idx_right, j.jump_left, j.jump_right, j.hight, j.closejump # # vals[j.idx_left+1:j.idx_right+1] -= j.closejump # logging.debug("%s: Number of jumps left: %i - Removing jump: %i - Size %i" % (coord['ant'], len(jumps_idx)-2, j.closejump, j.size)) # re-create proper vals # vals = vals[1:-1]*tec_jump_val # set back to 0 the values for flagged data vals[weights == 0] = 0 sw.selection = selection sw.setValues(vals) sw.setValues(weights, weight=True) sw.addHistory('TECJUMP') del sf del sw return 0
ra = skydb.getDefValues('Ra:' + source)['Ra:' + source][0][0] dec = skydb.getDefValues('Dec:' + source)['Dec:' + source][0][0] except KeyError: # Source not found in skymodel parmdb, try to find components logging.warning('Cannot find the source '+source+'. Trying components.') ra = np.array(skydb.getDefValues('Ra:*' + source + '*').values()) dec = np.array(skydb.getDefValues('Dec:*' + source + '*').values()) if len(ra) == 0 or len(dec) == 0: ra = np.nan dec = np.nan logging.error('Cannot find the source '+source+'. I leave NaNs.') else: ra = ra.mean() dec = dec.mean() logging.info('Found average direction for '+source+' at ra:'+str(ra)+' - dec:'+str(dec)) vals.append([ra, dec]) sourceTable.append(zip(*(dirs,vals))) logging.info("Total file size: "+str(int(h5parm.H.get_filesize()/1024./1024.))+" M.") # Add CREATE entry to history and print summary of tables if verbose soltabs = h5parm.getSoltabs(solset=solset) for st in soltabs: sw = solWriter(soltabs[st]) sw.addHistory('CREATE (by H5parm_importer.py from %s:%s/%s)' % (socket.gethostname(), os.path.abspath(''), globaldbFile)) if options.verbose: logging.info(str(h5parm)) del h5parm logging.info('Done.')
def run( step, parset, H ): """ Separate phase solutions into Clock and TEC. Phase solutions are assumed to be stored in solsets of the H5parm file, one solset per field. The Clock and TEC values are stored in the specified output soltab with type 'clock' and 'tec'. """ import numpy as np from losoto.h5parm import solFetcher, solWriter # get involved solsets using local step values or global values or all soltabs = getParSoltabs( step, parset, H ) flagBadChannels = parset.getBool('.'.join(["LoSoTo.Steps", step, "FlagBadChannels"]), True ) flagCut = parset.getFloat('.'.join(["LoSoTo.Steps", step, "FlagCut"]), 5. ) chi2cut = parset.getFloat('.'.join(["LoSoTo.Steps", step, "Chi2cut"]), 3000. ) combinePol = parset.getBool('.'.join(["LoSoTo.Steps", step, "CombinePol"]), False ) #fitOffset = parset.getBool('.'.join(["LoSoTo.Steps", step, "FitOffset"]), False ) removePhaseWraps=parset.getBool('.'.join(["LoSoTo.Steps", step, "RemovePhaseWraps"]), True ) fit3rdorder=parset.getBool('.'.join(["LoSoTo.Steps", step, "Fit3rdOrder"]), False ) circular=parset.getBool('.'.join(["LoSoTo.Steps", step, "Circular"]), False ) reverse=parset.getBool('.'.join(["LoSoTo.Steps", step, "Reverse"]), False ) # do something on every soltab (use the openSoltab LoSoTo function) #for soltab in openSoltabs( H, soltabs ): for soltabname in soltabs: solsetname=soltabname.split('/')[0] soltab=H.getSoltab(solset=solsetname, soltab=soltabname.split('/')[1]) logging.info("--> Working on soltab: "+soltab._v_name) t = solFetcher(soltab) tw = solWriter(soltab) # some checks solType = t.getType() if solType != 'phase': logging.warning("Soltab type of "+soltab._v_name+" is: "+solType+" should be phase. Ignoring.") continue # this will make a selection for the getValues() and getValuesIter() userSel = {} for axis in t.getAxesNames(): userSel[axis] = getParAxis( step, parset, H, axis ) t.setSelection(**userSel) # Collect station properties station_dict = H.getAnt(solsetname) stations = t.getAxisValues('ant') station_positions = np.zeros((len(stations), 3), dtype=np.float) for i, station_name in enumerate(stations): station_positions[i, 0] = station_dict[station_name][0] station_positions[i, 1] = station_dict[station_name][1] station_positions[i, 2] = station_dict[station_name][2] returnAxes=['ant','freq','pol','time'] for vals, flags, coord, selection in t.getValuesIter(returnAxes=returnAxes,weight=True): if len(coord['ant']) < 2: logging.error('Clock/TEC separation needs at least 2 antennas selected.') return 1 if len(coord['freq']) < 10: logging.error('Clock/TEC separation needs at least 10 frequency channels, preferably distributed over a wide range') return 1 freqs=coord['freq'] stations=coord['ant'] times=coord['time'] # get axes index axes=[i for i in t.getAxesNames() if i in returnAxes] # reverse time axes if reverse: vals = np.swapaxes(np.swapaxes(vals, 0, axes.index('time'))[::-1], 0, axes.index('time')) flags = np.swapaxes(np.swapaxes(flags, 0, axes.index('time'))[::-1], 0, axes.index('time')) result=doFit(vals,flags==0,freqs,stations,station_positions,axes,\ flagBadChannels=flagBadChannels,flagcut=flagCut,chi2cut=chi2cut,combine_pol=combinePol,removePhaseWraps=removePhaseWraps,fit3rdorder=fit3rdorder,circular=circular) if fit3rdorder: clock,tec,offset,tec3rd=result if reverse: clock = clock[::-1,:] tec = tec[::-1,:] tec3rd = tec3rd[::-1,:] else: clock,tec,offset=result if reverse: clock = clock[::-1,:] tec = tec[::-1,:] weights=tec>-5 tec[np.logical_not(weights)]=0 clock[np.logical_not(weights)]=0 weights=np.float16(weights) if combinePol: tf_st = H.makeSoltab(solsetname, 'tec', axesNames=['time', 'ant'], axesVals=[times, stations], vals=tec[:,:,0], weights=weights[:,:,0]) sw = solWriter(tf_st) sw.addHistory('CREATE (by CLOCKTECFIT operation)') tf_st = H.makeSoltab(solsetname, 'clock', axesNames=['time', 'ant'], axesVals=[times, stations], vals=clock[:,:,0]*1e-9, weights=weights[:,:,0]) sw = solWriter(tf_st) sw.addHistory('CREATE (by CLOCKTECFIT operation)') tf_st = H.makeSoltab(solsetname, 'phase_offset', axesNames=['ant'], axesVals=[stations], vals=offset[:,0], weights=np.ones_like(offset[:,0],dtype=np.float16)) sw = solWriter(tf_st) sw.addHistory('CREATE (by CLOCKTECFIT operation)') if fit3rdorder: tf_st = H.makeSoltab(solsetname, 'tec3rd', axesNames=['time', 'ant'], axesVals=[times, stations], vals=tec3rd[:,:,0], weights=weights[:,:,0]) sw = solWriter(tf_st) else: tf_st = H.makeSoltab(solsetname, 'tec', axesNames=['time', 'ant','pol'], axesVals=[times, stations, ['XX','YY']], vals=tec, weights=weights) sw = solWriter(tf_st) sw.addHistory('CREATE (by CLOCKTECFIT operation)') tf_st = H.makeSoltab(solsetname, 'clock', axesNames=['time', 'ant','pol'], axesVals=[times, stations, ['XX','YY']], vals=clock*1e-9, weights=weights) sw = solWriter(tf_st) sw.addHistory('CREATE (by CLOCKTECFIT operation)') tf_st = H.makeSoltab(solsetname, 'phase_offset', axesNames=['ant','pol'], axesVals=[stations, ['XX','YY']], vals=offset, weights=np.ones_like(offset,dtype=np.float16)) sw = solWriter(tf_st) sw.addHistory('CREATE (by CLOCKTECFIT operation)') if fit3rdorder: tf_st = H.makeSoltab(solsetname, 'tec3rd', axesNames=['time', 'ant','pol'], axesVals=[times, stations, ['XX','YY']], vals=tec3rd, weights=weights) sw = solWriter(tf_st) return 0
def create_h5parm(instrumentdbFiles, antennaFile, fieldFile, skydbFile, h5parmFile, complevel, solsetName, globaldbFile=None, verbose=False): """ Create the h5parm file. Input: instrumentdbFiles - list of the finenames of the solutions. antennaFile - file name of the antenna table. fieldFile - file name of the field table. skydbFile - file name of the sky table. h5parmFile - file name of the h5parm file that will be created. complevel - level of compression. It is usually 5. solsetName - Name of the solution set. Usually "sol###". globaldbFile (optional) - Name of the globaldbFile. Used only for logging purposes. """ # open/create the h5parm file and the solution-set h5parm = h5parm2(h5parmFile, readonly = False, complevel = complevel) solset = h5parm.makeSolset(solsetName) # Create tables using the first instrumentdb # TODO: all the instrument tables should be checked pdb = lofar.parmdb.parmdb(instrumentdbFiles[0]) solTypes = list(set(x[0] for x in (x.split(":") for x in pdb.getNames()))) logging.info('Found solution types: '+', '.join(solTypes)) # rewrite solTypes in order to put together # Gain <-> DirectionalGain # CommonRotationAngle <-> RotationAngle # CommonScalarPhase <-> ScalarPhase # CommonScalarAmplitude <-> ScalarAmplitude # it also separate Real/Imag/Ampl/Phase into different solTypes if "Gain" in solTypes: solTypes.remove('Gain') solTypes.append('*Gain:*:Real') solTypes.append('*Gain:*:Imag') solTypes.append('*Gain:*:Ampl') solTypes.append('*Gain:*:Phase') if "DirectionalGain" in solTypes: solTypes.remove('DirectionalGain') solTypes.append('*Gain:*:Real') solTypes.append('*Gain:*:Imag') solTypes.append('*Gain:*:Ampl') solTypes.append('*Gain:*:Phase') if "RotationAngle" in solTypes: solTypes.remove('RotationAngle') solTypes.append('*RotationAngle') if "CommonRotationAngle" in solTypes: solTypes.remove('CommonRotationAngle') solTypes.append('*RotationAngle') if "RotationMeasure" in solTypes: solTypes.remove('RotationMeasure') solTypes.append('*RotationMeasure') if "ScalarPhase" in solTypes: solTypes.remove('ScalarPhase') solTypes.append('*ScalarPhase') if "CommonScalarPhase" in solTypes: solTypes.remove('CommonScalarPhase') solTypes.append('*ScalarPhase') if "CommonScalarAmplitude" in solTypes: solTypes.remove('CommonScalarAmplitude') solTypes.append('*ScalarAmplitude') solTypes = list(set(solTypes)) # every soltype creates a different solution-table for solType in solTypes: # skip missing solTypes (not all parmdbs have e.g. TEC) if len(pdb.getNames(solType+':*')) == 0: continue pols = set(); dirs = set(); ants = set(); freqs = set(); times = set(); ptype = set() logging.info('Reading '+solType+'.') pbar = progressbar.ProgressBar(maxval=len(instrumentdbFiles)).start() ipbar = 0 for instrumentdbFile in sorted(instrumentdbFiles): pdb = lofar.parmdb.parmdb(instrumentdbFile) # create the axes grid, necessary if not all entries have the same axes lenght data = pdb.getValuesGrid(solType+':*') # check good instrument table if len(data) == 0: logging.error('Instrument table %s is empty, ignoring.' % instrumentdbFile) for solEntry in data: pol, dir, ant, parm = parmdbToAxes(solEntry) if pol != None: pols |= set([pol]) if dir != None: dirs |= set([dir]) if ant != None: ants |= set([ant]) freqs |= set(data[solEntry]['freqs']) times |= set(data[solEntry]['times']) pbar.update(ipbar) ipbar += 1 pbar.finish() pols = np.sort(list(pols)); dirs = np.sort(list(dirs)); ants = np.sort(list(ants)); freqs = np.sort(list(freqs)); times = np.sort(list(times)) shape = [i for i in (len(pols), len(dirs), len(ants), len(freqs), len(times)) if i != 0] vals = np.empty(shape) vals[:] = np.nan weights = np.zeros(shape, dtype=np.float16) logging.info('Filling table.') pbar = progressbar.ProgressBar(maxval=len(instrumentdbFiles)).start() ipbar = 0 for instrumentdbFile in instrumentdbFiles: pdb = lofar.parmdb.parmdb(instrumentdbFile) # fill the values data = pdb.getValuesGrid(solType+':*') if 'Real' in solType: dataIm = pdb.getValuesGrid(solType.replace('Real','Imag')+':*') if 'Imag' in solType: dataRe = pdb.getValuesGrid(solType.replace('Imag','Real')+':*') for solEntry in data: pol, dir, ant, parm = parmdbToAxes(solEntry) ptype |= set([solEntry.split(':')[0]]) # original parmdb solution type freq = data[solEntry]['freqs'] time = data[solEntry]['times'] val = data[solEntry]['values'] # convert Real and Imag in Amp and Phase respectively if parm == 'Real': solEntryIm = solEntry.replace('Real','Imag') valI = dataIm[solEntryIm]['values'] val = np.sqrt((val**2)+(valI**2)) if parm == 'Imag': solEntryRe = solEntry.replace('Imag','Real') valR = dataRe[solEntryRe]['values'] val = np.arctan2(val, valR) coords = [] if pol != None: polCoord = np.searchsorted(pols, pol) coords.append(polCoord) if dir != None: dirCoord = np.searchsorted(dirs, dir) coords.append(dirCoord) if ant != None: antCoord = np.searchsorted(ants, ant) coords.append(antCoord) freqCoord = np.searchsorted(freqs, freq) timeCoord = np.searchsorted(times, time) vals[tuple(coords)][np.ix_(freqCoord,timeCoord)] = val.T weights[tuple(coords)][np.ix_(freqCoord,timeCoord)] = 1 pbar.update(ipbar) ipbar += 1 np.putmask(vals, ~np.isfinite(vals), 0) # put inf and nans to 0 #vals = np.nan_to_num(vals) # replace nans with 0 (flagged later) pbar.finish() if solType == '*RotationAngle': np.putmask(weights, vals == 0., 0) # flag where val=0 h5parm.makeSoltab(solset, 'rotation', axesNames=['dir','ant','freq','time'], \ axesVals=[dirs,ants,freqs,times], vals=vals, weights=weights, parmdbType=', '.join(list(ptype))) if solType == '*RotationMeasure': np.putmask(weights, vals == 0., 0) # flag where val=0 h5parm.makeSoltab(solset, 'rotationmeasure', axesNames=['dir','ant','freq','time'], \ axesVals=[dirs,ants,freqs,times], vals=vals, weights=weights, parmdbType=', '.join(list(ptype))) elif solType == '*ScalarPhase': np.putmask(weights, vals == 0., 0) h5parm.makeSoltab(solset, 'scalarphase', axesNames=['dir','ant','freq','time'], \ axesVals=[dirs,ants,freqs,times], vals=vals, weights=weights, parmdbType=', '.join(list(ptype))) elif solType == '*ScalarAmplitude': np.putmask(weights, vals == 0., 0) h5parm.makeSoltab(solset, 'scalaramplitude', axesNames=['dir','ant','freq','time'], \ axesVals=[dirs,ants,freqs,times], vals=vals, weights=weights, parmdbType=', '.join(list(ptype))) elif solType == 'Clock': np.putmask(weights, vals == 0., 0) # clock may be diag or scalar if len(pols) == 0: h5parm.makeSoltab(solset, 'clock', axesNames=['ant','freq','time'], \ axesVals=[ants,freqs,times], vals=vals, weights=weights, parmdbType=', '.join(list(ptype))) else: h5parm.makeSoltab(solset, 'clock', axesNames=['pol','ant','freq','time'], \ axesVals=[pol,ants,freqs,times], vals=vals, weights=weights, parmdbType=', '.join(list(ptype))) elif solType == 'TEC': np.putmask(weights, vals == 0., 0) # tec may be diag or scalar if len(pols) == 0: h5parm.makeSoltab(solset, 'tec', axesNames=['dir','ant','freq','time'], \ axesVals=[dirs,ants,freqs,times], vals=vals, weights=weights, parmdbType=', '.join(list(ptype))) else: h5parm.makeSoltab(solset, 'tec', axesNames=['pol','dir','ant','freq','time'], \ axesVals=[pols,dirs,ants,freqs,times], vals=vals, weights=weights, parmdbType=', '.join(list(ptype))) elif solType == '*Gain:*:Real' or solType == '*Gain:*:Ampl': np.putmask(vals, vals == 0, 1) # nans were put to 0 before, set them to 1 np.putmask(weights, vals == 1., 0) # flag where val=1 h5parm.makeSoltab(solset, 'amplitude', axesNames=['pol','dir','ant','freq','time'], \ axesVals=[pols,dirs,ants,freqs,times], vals=vals, weights=weights, parmdbType=', '.join(list(ptype))) elif solType == '*Gain:*:Imag' or solType == '*Gain:*:Phase': np.putmask(weights, vals == 0., 0) # falg where val=0 h5parm.makeSoltab(solset, 'phase', axesNames=['pol','dir','ant','freq','time'], \ axesVals=[pols,dirs,ants,freqs,times], vals=vals, weights=weights, parmdbType=', '.join(list(ptype))) logging.info('Flagged data: %.3f%%' % (100.*(len(weights.flat)-np.count_nonzero(weights))/len(weights.flat))) logging.info('Collecting information from the ANTENNA table.') antennaTable = pt.table(antennaFile, ack=False) antennaNames = antennaTable.getcol('NAME') antennaPositions = antennaTable.getcol('POSITION') antennaTable.close() antennaTable = solset._f_get_child('antenna') antennaTable.append(zip(*(antennaNames,antennaPositions))) logging.info('Collecting information from the FIELD table.') fieldTable = pt.table(fieldFile, ack=False) phaseDir = fieldTable.getcol('PHASE_DIR') pointing = phaseDir[0, 0, :] fieldTable.close() sourceTable = solset._f_get_child('source') # add the field centre, that is also the direction for Gain and CommonRotationAngle sourceTable.append([('pointing',pointing)]) dirs = [] for tab in solset._v_children: c = solset._f_getChild(tab) if c._v_name != 'antenna' and c._v_name != 'source': if c.__contains__('dir'): dirs.extend(list(set(c.dir))) # remove duplicates dirs = list(set(dirs)) # remove any pointing (already in the table) if 'pointing' in dirs: dirs.remove('pointing') if not os.path.isdir(skydbFile) and dirs!=[]: logging.critical('Missing skydb table.') sys.exit(1) if dirs != []: logging.info('Collecting information from the sky table.') sourceFile = skydbFile + '/SOURCES' src_table = pt.table(sourceFile, ack=False) sub_tables = src_table.getsubtables() vals = [] ra = dec = np.nan has_patches_subtable = False for sub_table in sub_tables: if 'PATCHES' in sub_table: has_patches_subtable = True if has_patches_subtable: # Read values from PATCHES subtable src_table.close() sourceFile = skydbFile + '/SOURCES/PATCHES' src_table = pt.table(sourceFile, ack=False) patch_names = src_table.getcol('PATCHNAME') patch_ras = src_table.getcol('RA') patch_decs = src_table.getcol('DEC') for source in dirs: try: patch_indx = patch_names.index(source) ra = patch_ras[patch_indx] dec = patch_decs[patch_indx] except ValueError: ra = np.nan dec = np.nan logging.error('Cannot find the source '+source+'. I leave NaNs.') vals.append([ra, dec]) src_table.close() else: # Try to read default values from parmdb instead skydb = lofar.parmdb.parmdb(skydbFile) vals = [] ra = dec = np.nan for source in dirs: try: ra = skydb.getDefValues('Ra:' + source)['Ra:' + source][0][0] dec = skydb.getDefValues('Dec:' + source)['Dec:' + source][0][0] except KeyError: # Source not found in skymodel parmdb, try to find components logging.warning('Cannot find the source '+source+'. Trying components.') ra = np.array(skydb.getDefValues('Ra:*' + source + '*').values()) dec = np.array(skydb.getDefValues('Dec:*' + source + '*').values()) if len(ra) == 0 or len(dec) == 0: ra = np.nan dec = np.nan logging.error('Cannot find the source '+source+'. I leave NaNs.') else: ra = ra.mean() dec = dec.mean() logging.info('Found average direction for '+source+' at ra:'+str(ra)+' - dec:'+str(dec)) vals.append([ra, dec]) sourceTable.append(zip(*(dirs,vals))) logging.info("Total file size: "+str(int(h5parm.H.get_filesize()/1024./1024.))+" M.") # Add CREATE entry to history and print summary of tables if verbose soltabs = h5parm.getSoltabs(solset=solset) for st in soltabs: sw = solWriter(soltabs[st]) if globaldbFile is None: sw.addHistory('CREATE (by H5parm_importer.py from %s:%s/%s)' % (socket.gethostname(), os.path.abspath(''), "manual list")) else: sw.addHistory('CREATE (by H5parm_importer.py from %s:%s/%s)' % (socket.gethostname(), os.path.abspath(''), globaldbFile)) if verbose: logging.info(str(h5parm)) del h5parm logging.info('Done.')
def create_h5parm(instrumentdbFiles, antennaFile, fieldFile, skydbFile, h5parmFile, complevel, solsetName, globaldbFile=None, verbose=False): """ Create the h5parm file. Input: instrumentdbFiles - list of the finenames of the solutions. antennaFile - file name of the antenna table. fieldFile - file name of the field table. skydbFile - file name of the sky table. h5parmFile - file name of the h5parm file that will be created. complevel - level of compression. It is usually 5. solsetName - Name of the solution set. Usually "sol###". globaldbFile (optional) - Name of the globaldbFile. Used only for logging purposes. """ # open/create the h5parm file and the solution-set h5parm = h5parm_mod(h5parmFile, readonly = False, complevel = complevel) solset = h5parm.makeSolset(solsetName) # Create tables using the first instrumentdb # TODO: all the instrument tables should be checked pdb = lofar.parmdb.parmdb(instrumentdbFiles[0]) solTypes = list(set(x[0] for x in (x.split(":") for x in pdb.getNames()))) logging.info('Found solution types: '+', '.join(solTypes)) # rewrite solTypes in order to put together # Gain <-> DirectionalGain # CommonRotationAngle <-> RotationAngle # CommonScalarPhase <-> ScalarPhase # CommonScalarAmplitude <-> ScalarAmplitude # it also separate Real/Imag/Ampl/Phase into different solTypes if "Gain" in solTypes: solTypes.remove('Gain') solTypes.append('*Gain:*:Real') solTypes.append('*Gain:*:Imag') solTypes.append('*Gain:*:Ampl') solTypes.append('*Gain:*:Phase') if "DirectionalGain" in solTypes: solTypes.remove('DirectionalGain') solTypes.append('*Gain:*:Real') solTypes.append('*Gain:*:Imag') solTypes.append('*Gain:*:Ampl') solTypes.append('*Gain:*:Phase') if "RotationAngle" in solTypes: solTypes.remove('RotationAngle') solTypes.append('*RotationAngle') if "CommonRotationAngle" in solTypes: solTypes.remove('CommonRotationAngle') solTypes.append('*RotationAngle') if "RotationMeasure" in solTypes: solTypes.remove('RotationMeasure') solTypes.append('*RotationMeasure') if "ScalarPhase" in solTypes: solTypes.remove('ScalarPhase') solTypes.append('*ScalarPhase') if "CommonScalarPhase" in solTypes: solTypes.remove('CommonScalarPhase') solTypes.append('*ScalarPhase') if "CommonScalarAmplitude" in solTypes: solTypes.remove('CommonScalarAmplitude') solTypes.append('*ScalarAmplitude') solTypes = list(set(solTypes)) # every soltype creates a different solution-table for solType in solTypes: # skip missing solTypes (not all parmdbs have e.g. TEC) if len(pdb.getNames(solType+':*')) == 0: continue pols = set() dirs = set() ants = set() freqs = set() times = set() ptype = set() logging.info('Reading '+solType+'.') pbar = progressbar.ProgressBar(maxval=len(instrumentdbFiles)).start() ipbar = 0 for instrumentdbFile in sorted(instrumentdbFiles): pdb = lofar.parmdb.parmdb(instrumentdbFile) # create the axes grid, necessary if not all entries have the same axes lenght data = pdb.getValuesGrid(solType+':*') # check good instrument table if len(data) == 0: logging.error('Instrument table %s is empty, ignoring.' % instrumentdbFile) for solEntry in data: pol, dir, ant, parm = parmdbToAxes(solEntry) if pol is not None: pols |= set([pol]) if dir is not None: dirs |= set([dir]) if ant is not None: ants |= set([ant]) freqs |= set(data[solEntry]['freqs']) times |= set(data[solEntry]['times']) pbar.update(ipbar) ipbar += 1 pbar.finish() pols = np.sort(list(pols)) dirs = np.sort(list(dirs)) ants = np.sort(list(ants)) freqs = np.sort(list(freqs)) times = np.sort(list(times)) shape = [i for i in (len(pols), len(dirs), len(ants), len(freqs), len(times)) if i != 0] vals = np.empty(shape) vals[:] = np.nan weights = np.zeros(shape, dtype=np.float16) logging.info('Filling table.') pbar = progressbar.ProgressBar(maxval=len(instrumentdbFiles)).start() ipbar = 0 for instrumentdbFile in instrumentdbFiles: pdb = lofar.parmdb.parmdb(instrumentdbFile) # fill the values data = pdb.getValuesGrid(solType+':*') if 'Real' in solType: dataIm = pdb.getValuesGrid(solType.replace('Real','Imag')+':*') if 'Imag' in solType: dataRe = pdb.getValuesGrid(solType.replace('Imag','Real')+':*') for solEntry in data: pol, dir, ant, parm = parmdbToAxes(solEntry) ptype |= set([solEntry.split(':')[0]]) # original parmdb solution type freq = data[solEntry]['freqs'] time = data[solEntry]['times'] val = data[solEntry]['values'] # convert Real and Imag in Amp and Phase respectively if parm == 'Real': solEntryIm = solEntry.replace('Real','Imag') valI = dataIm[solEntryIm]['values'] val = np.sqrt((val**2)+(valI**2)) if parm == 'Imag': solEntryRe = solEntry.replace('Imag','Real') valR = dataRe[solEntryRe]['values'] val = np.arctan2(val, valR) coords = [] if pol is not None: polCoord = np.searchsorted(pols, pol) coords.append(polCoord) if dir is not None: dirCoord = np.searchsorted(dirs, dir) coords.append(dirCoord) if ant is not None: antCoord = np.searchsorted(ants, ant) coords.append(antCoord) freqCoord = np.searchsorted(freqs, freq) timeCoord = np.searchsorted(times, time) vals[tuple(coords)][np.ix_(freqCoord,timeCoord)] = val.T weights[tuple(coords)][np.ix_(freqCoord,timeCoord)] = 1 pbar.update(ipbar) ipbar += 1 np.putmask(vals, ~np.isfinite(vals), 0) # put inf and nans to 0 #vals = np.nan_to_num(vals) # replace nans with 0 (flagged later) pbar.finish() if solType == '*RotationAngle': np.putmask(weights, vals == 0., 0) # flag where val=0 h5parm.makeSoltab(solset, 'rotation', axesNames=['dir','ant','freq','time'], \ axesVals=[dirs,ants,freqs,times], vals=vals, weights=weights, parmdbType=', '.join(list(ptype))) if solType == '*RotationMeasure': np.putmask(weights, vals == 0., 0) # flag where val=0 h5parm.makeSoltab(solset, 'rotationmeasure', axesNames=['dir','ant','freq','time'], \ axesVals=[dirs,ants,freqs,times], vals=vals, weights=weights, parmdbType=', '.join(list(ptype))) elif solType == '*ScalarPhase': np.putmask(weights, vals == 0., 0) h5parm.makeSoltab(solset, 'scalarphase', axesNames=['dir','ant','freq','time'], \ axesVals=[dirs,ants,freqs,times], vals=vals, weights=weights, parmdbType=', '.join(list(ptype))) elif solType == '*ScalarAmplitude': np.putmask(weights, vals == 0., 0) h5parm.makeSoltab(solset, 'scalaramplitude', axesNames=['dir','ant','freq','time'], \ axesVals=[dirs,ants,freqs,times], vals=vals, weights=weights, parmdbType=', '.join(list(ptype))) elif solType == 'Clock': np.putmask(weights, vals == 0., 0) # clock may be diag or scalar if len(pols) == 0: h5parm.makeSoltab(solset, 'clock', axesNames=['ant','freq','time'], \ axesVals=[ants,freqs,times], vals=vals, weights=weights, parmdbType=', '.join(list(ptype))) else: h5parm.makeSoltab(solset, 'clock', axesNames=['pol','ant','freq','time'], \ axesVals=[pol,ants,freqs,times], vals=vals, weights=weights, parmdbType=', '.join(list(ptype))) elif solType == 'TEC': np.putmask(weights, vals == 0., 0) # tec may be diag or scalar if len(pols) == 0: h5parm.makeSoltab(solset, 'tec', axesNames=['dir','ant','freq','time'], \ axesVals=[dirs,ants,freqs,times], vals=vals, weights=weights, parmdbType=', '.join(list(ptype))) else: h5parm.makeSoltab(solset, 'tec', axesNames=['pol','dir','ant','freq','time'], \ axesVals=[pols,dirs,ants,freqs,times], vals=vals, weights=weights, parmdbType=', '.join(list(ptype))) elif solType == '*Gain:*:Real' or solType == '*Gain:*:Ampl': np.putmask(vals, vals == 0, 1) # nans were put to 0 before, set them to 1 np.putmask(weights, vals == 1., 0) # flag where val=1 h5parm.makeSoltab(solset, 'amplitude', axesNames=['pol','dir','ant','freq','time'], \ axesVals=[pols,dirs,ants,freqs,times], vals=vals, weights=weights, parmdbType=', '.join(list(ptype))) elif solType == '*Gain:*:Imag' or solType == '*Gain:*:Phase': np.putmask(weights, vals == 0., 0) # falg where val=0 h5parm.makeSoltab(solset, 'phase', axesNames=['pol','dir','ant','freq','time'], \ axesVals=[pols,dirs,ants,freqs,times], vals=vals, weights=weights, parmdbType=', '.join(list(ptype))) logging.info('Flagged data: %.3f%%' % (100.*(len(weights.flat)-np.count_nonzero(weights))/len(weights.flat))) logging.info('Collecting information from the ANTENNA table.') antennaTable = pt.table(antennaFile, ack=False) antennaNames = antennaTable.getcol('NAME') antennaPositions = antennaTable.getcol('POSITION') antennaTable.close() antennaTable = solset._f_get_child('antenna') antennaTable.append(zip(*(antennaNames,antennaPositions))) logging.info('Collecting information from the FIELD table.') fieldTable = pt.table(fieldFile, ack=False) phaseDir = fieldTable.getcol('PHASE_DIR') pointing = phaseDir[0, 0, :] fieldTable.close() sourceTable = solset._f_get_child('source') # add the field centre, that is also the direction for Gain and CommonRotationAngle sourceTable.append([('pointing',pointing)]) dirs = [] for tab in solset._v_children: c = solset._f_get_child(tab) if c._v_name != 'antenna' and c._v_name != 'source': if c.__contains__('dir'): dirs.extend(list(set(c.dir))) # remove duplicates dirs = list(set(dirs)) # remove any pointing (already in the table) if 'pointing' in dirs: dirs.remove('pointing') if not os.path.isdir(skydbFile) and dirs!=[]: logging.critical('Missing skydb table.') sys.exit(1) if dirs != []: logging.info('Collecting information from the sky table.') sourceFile = skydbFile + '/SOURCES' src_table = pt.table(sourceFile, ack=False) sub_tables = src_table.getsubtables() vals = [] ra = dec = np.nan has_patches_subtable = False for sub_table in sub_tables: if 'PATCHES' in sub_table: has_patches_subtable = True if has_patches_subtable: # Read values from PATCHES subtable src_table.close() sourceFile = skydbFile + '/SOURCES/PATCHES' src_table = pt.table(sourceFile, ack=False) patch_names = src_table.getcol('PATCHNAME') patch_ras = src_table.getcol('RA') patch_decs = src_table.getcol('DEC') for source in dirs: try: patch_indx = patch_names.index(source) ra = patch_ras[patch_indx] dec = patch_decs[patch_indx] except ValueError: ra = np.nan dec = np.nan logging.error('Cannot find the source '+source+'. I leave NaNs.') vals.append([ra, dec]) src_table.close() else: # Try to read default values from parmdb instead skydb = lofar.parmdb.parmdb(skydbFile) vals = [] ra = dec = np.nan for source in dirs: try: ra = skydb.getDefValues('Ra:' + source)['Ra:' + source][0][0] dec = skydb.getDefValues('Dec:' + source)['Dec:' + source][0][0] except KeyError: # Source not found in skymodel parmdb, try to find components logging.warning('Cannot find the source '+source+'. Trying components.') ra = np.array(skydb.getDefValues('Ra:*' + source + '*').values()) dec = np.array(skydb.getDefValues('Dec:*' + source + '*').values()) if len(ra) == 0 or len(dec) == 0: ra = np.nan dec = np.nan logging.error('Cannot find the source '+source+'. I leave NaNs.') else: ra = ra.mean() dec = dec.mean() logging.info('Found average direction for '+source+' at ra:'+str(ra)+' - dec:'+str(dec)) vals.append([ra, dec]) sourceTable.append(zip(*(dirs,vals))) logging.info("Total file size: "+str(int(h5parm.H.get_filesize()/1024./1024.))+" M.") # Add CREATE entry to history and print summary of tables if verbose soltabs = h5parm.getSoltabs(solset=solset) for st in soltabs: sw = solWriter(soltabs[st]) if globaldbFile is None: sw.addHistory('CREATE (by H5parm_importer.py from %s:%s/%s)' % (socket.gethostname(), os.path.abspath(''), "manual list")) else: sw.addHistory('CREATE (by H5parm_importer.py from %s:%s/%s)' % (socket.gethostname(), os.path.abspath(''), globaldbFile)) if verbose: logging.info(str(h5parm)) del h5parm logging.info('Done.')
def run( step, parset, H ): """ Fits a screen to TEC values derived by the TECFIT operation. The TEC values are read from the specified tec soltab. The results of the fit are stored in the specified tecscreen solution table. These values are the screen TEC values per station per pierce point per solution interval. The pierce point locations are stored in an auxiliary array in the output solution table. TEC screens can be plotted with the PLOT operation by setting PlotType = TECScreen. The H5parm_exporter.py tool can be used to export the screen to a parmdb that BBS and the AWimager can use. Note, however, that the output screens are not normalized properly (any normalization was lost due to the use of source-to-source phase gradients in the TECFIT operation). Therefore, a direction-independent calibration must be done after exporting the screens to a parmdb file, with the following settings in the BBS solve step: Model.Ionosphere.Enable = T Model.Ionosphere.Type = EXPION """ import numpy as np import re from losoto.h5parm import solFetcher, solWriter # Switch to the Agg backend to prevent problems with pylab imports when # DISPLAY env. variable is not set import os if 'DISPLAY' not in os.environ: import matplotlib matplotlib.use("Agg") soltabs = getParSoltabs( step, parset, H ) outSoltabs = parset.getStringVector('.'.join(["LoSoTo.Steps", step, "OutSoltab"]), [] ) height = np.array(parset.getDoubleVector('.'.join(["LoSoTo.Steps", step, "Height"]), [200e3] )) order = int(parset.getString('.'.join(["LoSoTo.Steps", step, "Order"]), '15' )) # Load TEC values from TECFIT operation indx = 0 for soltab in openSoltabs(H, soltabs): if 'tec' not in soltab._v_title: logging.warning('No TECFIT solution tables found for solution table ' '{0}'.format(soltabs[indx])) continue indx += 1 solset = soltabs[indx].split('/')[0] logging.info('Using input solution table: {0}'.format(soltabs[indx])) logging.info('Using output solution table: {0}'.format(outSoltabs[indx])) # Collect station and source names and positions and times, making sure # that they are ordered correctly. t = solFetcher(soltab) r, axis_vals = t.getValues() source_names = axis_vals['dir'] source_dict = H.getSou(solset) source_positions = [] for source in source_names: source_positions.append(source_dict[source]) station_names = axis_vals['ant'] station_dict = H.getAnt(solset) station_positions = [] for station in station_names: station_positions.append(station_dict[station]) times = axis_vals['time'] # Get sizes N_sources = len(source_names) N_times = len(times) N_stations = len(station_names) N_piercepoints = N_sources * N_stations rr = np.reshape(r.transpose([0, 2, 1]), [N_piercepoints, N_times]) heights = list(set(np.linspace(height[0], height[-1], 5))) heights.sort() if len(heights) > 1: logging.info('Trying range of heights: {0} m'.format(heights)) for i, height in enumerate(heights): # Find pierce points and airmass values for given screen height logging.info('Using height = {0} m and order = {1}'.format(height, order)) if height < 100e3: logging.warning("Height is less than 100e3 m.") pp, airmass = calculate_piercepoints(np.array(station_positions), np.array(source_positions), np.array(times), height) # Fit a TEC screen r_0 = 10e3 beta = 5.0 / 3.0 tec_screen, residual = fit_screen_to_tec(station_names, source_names, pp, airmass, rr, times, height, order, r_0, beta) total_resid = np.sum(np.abs(residual)) if i > 0: if total_resid < best_resid: tec_screen_best = tec_screen pp_best = pp height_best = height best_resid = total_resid else: tec_screen_best = tec_screen pp_best = pp height_best = height best_resid = total_resid if len(heights) > 1: logging.info('Total residual for fit: {0}\n'.format(total_resid)) # Use screen with lowest total residual if len(heights) > 1: tec_screen = tec_screen_best pp = pp_best height = height_best logging.info('Using height (with lowest total residual) of {0} m'.format(height)) # Write the results to the output solset dirs_out = source_names times_out = times ants_out = station_names # Make output tecscreen table outSolset = outSoltabs[indx].split('/')[0] outSoltab = outSoltabs[indx].split('/')[1] if not outSolset in H.getSolsets().keys(): solsetTEC = H.makeSolset(outSolset) dirs_pos = source_positions sourceTable = solsetTEC._f_get_child('source') sourceTable.append(zip(*(dirs_out, dirs_pos))) ants_pos = station_positions antennaTable = solsetTEC._f_get_child('antenna') antennaTable.append(zip(*(ants_out, ants_pos))) # Store tecscreen values. The residual values are stored in the weights # table. Flagged values of the screen have weights set to 0.0. vals = tec_screen.transpose([1, 0, 2]) weights = residual.transpose([1, 0, 2]) tec_screen_st = H.makeSoltab(outSolset, 'tecscreen', outSoltab, axesNames=['dir', 'time', 'ant'], axesVals=[dirs_out, times_out, ants_out], vals=vals, weights=weights) # Store beta, r_0, height, and order as attributes of the tecscreen # soltab tec_screen_st._v_attrs['beta'] = beta tec_screen_st._v_attrs['r_0'] = r_0 tec_screen_st._v_attrs['height'] = height tec_screen_st._v_attrs['order'] = order # Make output piercepoint table tec_screen_solset = tec_screen_st._v_parent._v_name H.H.create_carray('/'+tec_screen_solset+'/'+tec_screen_st._v_name, 'piercepoint', obj=pp) # Add histories sw = solWriter(tec_screen_st) sw.addHistory('CREATE (by TECSCREEN operation)') indx += 1 return 0
def run(step, parset, H): import numpy as np from losoto.h5parm import solFetcher, solWriter soltabs = getParSoltabs(step, parset, H) axesToClip = parset.getStringVector( '.'.join(["LoSoTo.Steps", step, "Axes"]), []) clipLevel = parset.getFloat('.'.join(["LoSoTo.Steps", step, "ClipLevel"]), 0.) log = parset.getBool('.'.join(["LoSoTo.Steps", step, "Log"]), True) if len(axesToClip) < 1: logging.error("Please specify axes to clip.") return 1 if clipLevel == 0.: logging.error( "Please specify factor above/below median at which to clip.") return 1 for soltab in openSoltabs(H, soltabs): logging.info("Clipping soltab: " + soltab._v_name) sf = solFetcher(soltab) # axis selection userSel = {} for axis in sf.getAxesNames(): userSel[axis] = getParAxis(step, parset, H, axis) sf.setSelection(**userSel) # some checks for i, axis in enumerate(axesToClip[:]): if axis not in sf.getAxesNames(): del axesToClip[i] logging.warning('Axis \"' + axis + '\" not found. Ignoring.') if sf.getType() != 'amplitude': logging.error('CLIP is for "amplitude" tables, not %s.' % sf.getType()) continue sw = solWriter(soltab, useCache=True) # remember to flush() before_count = 0 after_count = 0 total = 0 for vals, weights, coord, selection in sf.getValuesIter( returnAxes=axesToClip, weight=True): total += len(vals) before_count += (len(weights) - np.count_nonzero(weights)) # first find the median and standard deviation if (weights == 0).all(): valmedian = 0 else: if log: valmedian = np.median(np.log10(vals[(weights != 0)])) rms = np.std(np.log10(vals[(weights != 0)])) np.putmask( weights, np.abs(np.log10(vals) - valmedian) > rms * clipLevel, 0) else: valmedian = np.median(vals[(weights != 0)]) rms = np.std(vals[(weights != 0)]) np.putmask(weights, np.abs(vals - valmedian) > rms * clipLevel, 0) after_count += (len(weights) - np.count_nonzero(weights)) # writing back the solutions sw.selection = selection sw.setValues(weights, weight=True) sw.addHistory('CLIP (over %s with %s sigma cut)' % (axesToClip, clipLevel)) logging.info('Clip, flagged data: %f %% -> %f %%' \ % (100.*before_count/total, 100.*after_count/total)) sw.flush() return 0
sys.exit(1) skydbFile = os.path.join(inMSs[0], 'sky') if not os.path.isdir(skydbFile): logging.warning( 'No sky table found. (Direction-dependent parameters will not work.)' ) skydbFile = None #generate list of parmDB-filenames parmDBnames = [MS.rstrip('/') + instrument for MS in inMSs] #create and fill the hdf5-file: solset = parmDBs2h5parm(hdf5File, parmDBnames, antennaFile, fieldFile, skydbFile, compression=compression, solsetName=solsetName) # Add CREATE entry to history h5parmDB = h5parm(hdf5File, readonly=False) soltabs = h5parmDB.getSoltabs(solset=solset) for st in soltabs: sw = solWriter(soltabs[st]) sw.addHistory( 'CREATE (by PipelineStep_losotoImporter from %s / %s - %s)' % (os.path.abspath(''), os.path.basename( parmDBnames[0]), os.path.basename(parmDBnames[-1]))) h5parmDB.close()
vals = np.arange(4*10*100).reshape(4,10,100) logging.info("Create soltab") H5.makeSoltab(ss, 'amplitude', 'stTest', axesNames=['axis1','axis2','axis3'], axesVals=axesVals, vals=vals, weights=vals) logging.info("Create soltab (using same name)") H5.makeSoltab(ss, 'amplitude', 'stTest', axesNames=['axis1','axis2','axis3'], axesVals=axesVals, vals=vals, weights=vals) logging.info("Create soltab (using default name)") H5.makeSoltab(ss, 'amplitude', axesNames=['axis1','axis2','axis3'], axesVals=axesVals, vals=vals, weights=vals) logging.info('Get a soltab object') st=H5.getSoltab(ss,'stTest') logging.info('Get all soltabs:') print H5.getSoltabs(ss) print "###########################################" logging.info('### solFetcher/solWriter - General') Hsf = solFetcher(st) Hsw = solWriter(st) logging.info('Get solution Type (exp: amplitude)') print Hsf.getType() logging.info('Get Axes Names') print Hsf.getAxesNames() logging.info('Get Axis1 Len (exp: 4)') print Hsf.getAxisLen('axis1') logging.info('Get Axis1 Type (exp: str)') print Hsf.getAxisType('axis1') logging.info('Get Axis2 Type (exp: float)') print Hsf.getAxisType('axis2') logging.info('Get Axis1 Values (exp: a,b,c,d)') print Hsf.getAxisValues('axis1') logging.info('Set new axes values') Hsw.setAxisValues('axis1',['e','f','g','h']) logging.info('Get new Axis1 Values (exp: e,f,g,h)')
def run( step, parset, H ): import scipy.ndimage.filters import numpy as np from losoto.h5parm import solFetcher, solWriter soltabs = getParSoltabs( step, parset, H ) axesToSmooth = parset.getStringVector('.'.join(["LoSoTo.Steps", step, "Axes"]), [] ) FWHM = parset.getIntVector('.'.join(["LoSoTo.Steps", step, "FWHM"]), [] ) mode = parset.getString('.'.join(["LoSoTo.Steps", step, "Mode"]), "runningmedian" ) if mode == "runningmedian" and len(axesToSmooth) != len(FWHM): logging.error("Axes and FWHM lenghts must be equal.") return 1 if mode == "runningmedian": logging.warning('Flagged data are still taken into account!') if FWHM != [] and mode != "runningmedian": logging.warning("FWHM makes sense only with runningmedian mode, ignoring it.") for soltab in openSoltabs( H, soltabs ): logging.info("Smoothing soltab: "+soltab._v_name) sf = solFetcher(soltab) sw = solWriter(soltab, useCache = True) # remember to flush! # axis selection userSel = {} for axis in sf.getAxesNames(): userSel[axis] = getParAxis( step, parset, H, axis ) sf.setSelection(**userSel) for i, axis in enumerate(axesToSmooth[:]): if axis not in sf.getAxesNames(): del axesToSmooth[i] del FWHM[i] logging.warning('Axis \"'+axis+'\" not found. Ignoring.') for vals, weights, coord, selection in sf.getValuesIter(returnAxes=axesToSmooth, weight=True): if mode == 'runningmedian': valsnew = scipy.ndimage.filters.median_filter(vals, FWHM) elif mode == 'median': valsnew = np.median( vals[(weights!=0)] ) elif mode == 'mean': valsnew = np.mean( vals[(weights!=0)] ) else: logging.error('Mode must be: runningmedian, median or mean') return 1 sw.selection = selection sw.setValues(valsnew) sw.flush() sw.addHistory('SMOOTH (over %s with mode = %s)' % (axesToSmooth, mode)) del sf del sw return 0
def run( step, parset, H ): """ Interpolate the solutions from one table into a destination table """ import itertools import scipy.interpolate import numpy as np from losoto.h5parm import solFetcher, solWriter soltabs = getParSoltabs( step, parset, H ) solTypes = getParSolTypes( step, parset, H ) calSoltab = parset.getString('.'.join(["LoSoTo.Steps", step, "CalSoltab"]), '' ) calDir = parset.getString('.'.join(["LoSoTo.Steps", step, "CalDir"]), '' ) interpAxes = parset.getStringVector('.'.join(["LoSoTo.Steps", step, "InterpAxes"]), ['time','freq'] ) interpMethod = parset.getString('.'.join(["LoSoTo.Steps", step, "InterpMethod"]), 'linear' ) medAxis = parset.getString('.'.join(["LoSoTo.Steps", step, "MedAxis"]), '' ) rescale = parset.getBool('.'.join(["LoSoTo.Steps", step, "Rescale"]), False ) if interpMethod not in ["nearest", "linear", "cubic"]: logging.error('Interpolation method must be nearest, linear or cubic.') return 1 if rescale and medAxis == '': logging.error('A medAxis is needed for rescaling.') return 1 # open calibration table css, cst = calSoltab.split('/') cr = solFetcher(H.getSoltab(css, cst)) cAxesNames = cr.getAxesNames() for soltab in openSoltabs( H, soltabs ): logging.info("Interpolating soltab: "+soltab._v_name) tr = solFetcher(soltab) tw = solWriter(soltab) axesNames = tr.getAxesNames() for i, interpAxis in enumerate(interpAxes[:]): if interpAxis not in axesNames or interpAxis not in cAxesNames: logging.error('Axis '+interpAxis+' not found. Ignoring.') del interpAxes[i] if rescale and (medAxis not in axesNames or medAxis not in cAxesNames): logging.error('Axis '+medAxis+' not found. Cannot proceed.') return 1 # axis selection userSel = {} for axis in tr.getAxesNames(): userSel[axis] = getParAxis( step, parset, H, axis ) tr.setSelection(**userSel) for vals, coord, selection in tr.getValuesIter(returnAxes=interpAxes): # construct grid coordSel = removeKeys(coord, interpAxes) logging.debug("Working on coords:"+str(coordSel)) # change dir if sepcified if calDir != '': coordSel['dir'] = calDir cr.setSelection(**coordSel) calValues, calCoord = cr.getValues() # fill medAxis with the median value if rescale: axis = cAxesNames.index(medAxis) calValues = np.repeat( np.expand_dims( np.median( calValues, axis ), axis ), calValues.shape[axis], axis ) # create a list of values whose coords are calPoints calValues = np.ndarray.flatten(calValues) # create calibrator/target coordinates arrays calPoints = [] targetPoints = [] for interpAxis in interpAxes: calPoints.append(calCoord[interpAxis]) targetPoints.append(coord[interpAxis]) calPoints = np.array([x for x in itertools.product(*calPoints)]) targetPoints = np.array([x for x in itertools.product(*targetPoints)]) # interpolation valsNew = scipy.interpolate.griddata(calPoints, calValues, targetPoints, interpMethod) # fill values outside boudaries with "nearest" solutions # NOTE: in 1D is useless due to scipy bug but this is compensated in the next if if interpMethod != 'nearest': valsNewNearest = scipy.interpolate.griddata(calPoints, calValues, targetPoints, 'nearest') # NaN is != from itself valsNew[ np.isnan(valsNew) ] = valsNewNearest [ np.isnan(valsNew) ] # fix bug in Scipy which put NaNs outside the convex hull in 1D for 'nearest' if len(np.squeeze(vals).shape) == 1: import scipy.cluster.vq as vq valsNew = np.squeeze(valsNew) code, dist = vq.vq(targetPoints, calPoints) # NaN is != from itself valsNew[ np.isnan(valsNew) ] = calValues[code][ np.isnan(valsNew) ] valsNew = valsNew.reshape(vals.shape) if rescale: # rescale solutions axis = interpAxes.index(medAxis) valsMed = np.repeat( np.expand_dims( np.median( vals, axis ), axis ), vals.shape[axis], axis ) valsNewMed = np.repeat( np.expand_dims( np.median( valsNew, axis ), axis ), valsNew.shape[axis], axis ) valsNew = vals*valsNewMed/valsMed #print "Rescaling by: ", valsNewMed[:,0]/valsMed[:,0] # writing back the solutions tw.selection = selection tw.setValues(valsNew) tw.addHistory('INTERP (from table %s)' % (calSoltab)) return 0
def run( step, parset, H ): """ Generic unspecified step for easy expansion. """ import numpy as np from losoto.h5parm import solFetcher, solWriter # all the following are LoSoTo function to extract information from the parset # get involved solsets using local step values or global values or all solsets = getParSolsets( step, parset, H ) logging.info('Solset: '+str(solsets)) # get involved soltabs using local step values or global values or all soltabs = getParSoltabs( step, parset, H ) logging.info('Soltab: '+str(soltabs)) # get list of SolTypes using local step values or global values or all solTypes = getParSolTypes( step, parset, H ) logging.info('SolType: '+str(solTypes)) # do something on every soltab (use the openSoltab LoSoTo function) for soltab in openSoltabs( H, soltabs ): logging.info("--> Working on soltab: "+soltab._v_name) # use the solFetcher from the H5parm lib t = solFetcher(soltab) tw = solWriter(soltab) axisNames = t.getAxesNames() logging.info("Axis names are: "+str(axisNames)) solType = t.getType() logging.info("Soltab type is: "+solType) # this will make a selection for the getValues() and getValuesIter() # interpret every entry in the parset which has an axis name as a selector userSel = {} for axis in t.getAxesNames(): userSel[axis] = getParAxis( step, parset, H, axis ) t.setSelection(**userSel) t.setSelection(ant=ants, pol=pols, dir=dirs) logging.info("Selection is: "+str(t.selection)) # find axis values logging.info("Antennas (no selection) are: "+str(t.getAxisValues('ant', ignoreSelection=True))) logging.info("Antennas (with selection) are: "+str(t.getAxisValues('ant'))) # but one can also use (selection is active here!) logging.info("Antennas (other method) are: "+str(t.ant)) logging.info("Frequencies are: "+str(t.freq)) logging.info("Directions are: "+str(t.dir)) logging.info("Polarizations are: "+str(t.pol)) # try to access a non-existent axis t.getAxisValues('nonexistantaxis') # now get all values given this selection logging.info("Get data using t.val") val = t.val logging.debug('shape of val: '+str(t.val.shape)) logging.info("$ val is "+str(val[0,0,0,0,100])) weight = t.weight time = t.time thisTime = t.time[100] # another way to get the data is using the getValues() logging.info("Get data using getValues()") grid, axes = t.getValues() # axis names logging.info("Axes: "+str(t.getAxesNames())) # axis shape print axes print [t.getAxisLen(axis) for axis in axes] # not ordered, is a dict!
def run(step, parset, H): """ Interpolate the solutions from one table into a destination table """ import itertools import scipy.interpolate import numpy as np from losoto.h5parm import solFetcher, solWriter soltabs = getParSoltabs(step, parset, H) solTypes = getParSolTypes(step, parset, H) calSoltab = parset.getString('.'.join(["LoSoTo.Steps", step, "CalSoltab"]), '') calDir = parset.getString('.'.join(["LoSoTo.Steps", step, "CalDir"]), '') interpAxes = parset.getStringVector( '.'.join(["LoSoTo.Steps", step, "InterpAxes"]), ['time', 'freq']) interpMethod = parset.getString( '.'.join(["LoSoTo.Steps", step, "InterpMethod"]), 'linear') medAxis = parset.getString('.'.join(["LoSoTo.Steps", step, "MedAxis"]), '') rescale = parset.getBool('.'.join(["LoSoTo.Steps", step, "Rescale"]), False) if interpMethod not in ["nearest", "linear", "cubic"]: logging.error('Interpolation method must be nearest, linear or cubic.') return 1 if rescale and medAxis == '': logging.error('A medAxis is needed for rescaling.') return 1 # open calibration table css, cst = calSoltab.split('/') cr = solFetcher(H.getSoltab(css, cst)) cAxesNames = cr.getAxesNames() for soltab in openSoltabs(H, soltabs): logging.info("Interpolating soltab: " + soltab._v_name) tr = solFetcher(soltab) tw = solWriter(soltab) axesNames = tr.getAxesNames() for i, interpAxis in enumerate(interpAxes[:]): if interpAxis not in axesNames or interpAxis not in cAxesNames: logging.error('Axis ' + interpAxis + ' not found. Ignoring.') del interpAxes[i] if rescale and (medAxis not in axesNames or medAxis not in cAxesNames): logging.error('Axis ' + medAxis + ' not found. Cannot proceed.') return 1 # axis selection userSel = {} for axis in tr.getAxesNames(): userSel[axis] = getParAxis(step, parset, H, axis) tr.setSelection(**userSel) for vals, coord, selection in tr.getValuesIter(returnAxes=interpAxes): # construct grid coordSel = removeKeys(coord, interpAxes) logging.debug("Working on coords:" + str(coordSel)) # change dir if sepcified if calDir != '': coordSel['dir'] = calDir cr.setSelection(**coordSel) calValues, calCoord = cr.getValues() # fill medAxis with the median value if rescale: axis = cAxesNames.index(medAxis) calValues = np.repeat( np.expand_dims(np.median(calValues, axis), axis), calValues.shape[axis], axis) # create a list of values whose coords are calPoints calValues = np.ndarray.flatten(calValues) # create calibrator/target coordinates arrays calPoints = [] targetPoints = [] for interpAxis in interpAxes: calPoints.append(calCoord[interpAxis]) targetPoints.append(coord[interpAxis]) calPoints = np.array([x for x in itertools.product(*calPoints)]) targetPoints = np.array( [x for x in itertools.product(*targetPoints)]) # interpolation valsNew = scipy.interpolate.griddata(calPoints, calValues, targetPoints, interpMethod) # fill values outside boudaries with "nearest" solutions # NOTE: in 1D is useless due to scipy bug but this is compensated in the next if if interpMethod != 'nearest': valsNewNearest = scipy.interpolate.griddata( calPoints, calValues, targetPoints, 'nearest') # NaN is != from itself valsNew[np.isnan(valsNew)] = valsNewNearest[np.isnan(valsNew)] # fix bug in Scipy which put NaNs outside the convex hull in 1D for 'nearest' if len(np.squeeze(vals).shape) == 1: import scipy.cluster.vq as vq valsNew = np.squeeze(valsNew) code, dist = vq.vq(targetPoints, calPoints) # NaN is != from itself valsNew[np.isnan(valsNew)] = calValues[code][np.isnan(valsNew)] valsNew = valsNew.reshape(vals.shape) if rescale: # rescale solutions axis = interpAxes.index(medAxis) valsMed = np.repeat( np.expand_dims(np.median(vals, axis), axis), vals.shape[axis], axis) valsNewMed = np.repeat( np.expand_dims(np.median(valsNew, axis), axis), valsNew.shape[axis], axis) valsNew = vals * valsNewMed / valsMed #print "Rescaling by: ", valsNewMed[:,0]/valsMed[:,0] # writing back the solutions tw.selection = selection tw.setValues(valsNew) tw.addHistory('INTERP (from table %s)' % (calSoltab)) return 0
def run(step, parset, H): """ Fit phase solutions to obtain TEC values per direction. Phase solutions are assumed to be stored in solsets of the H5parm file, one solset per band per field. Only phase- or scalarphase-type solution tables are used. If direction-independent solutions are found (in addition to the direction-dependent ones), they are added, after averaging, to the corresponding direction-dependent ones. Phase solutions are automatically grouped by field and by band. The derived TEC values are stored in the specified output soltab of type 'tec', with one TEC value per station per direction per solution interval. The TEC values are derived using the ``lofar.expion.baselinefitting.fit()`` function to fit a TEC value to the phases. The term that is minimized includes all baselines, so there is no preferred reference station, and the residual is computed as the complex number 1.0 - exp(1i phasedifference), which is zero when the phase difference is a multiple of 2pi. The TEC solution table may be used to derive TEC screens using the TECSCREEN operation. """ from losoto.h5parm import solFetcher, solWriter import numpy as np # Switch to the Agg backend to prevent problems with pylab imports when # DISPLAY env. variable is not set import os if 'DISPLAY' not in os.environ: import matplotlib matplotlib.use("Agg") from pylab import find import re from .tecscreen import calculate_piercepoints from numpy.linalg import norm solsets = getParSolsets(step, parset, H) ants = getParAxis(step, parset, H, 'ant') pols = getParAxis(step, parset, H, 'pol') dirs = getParAxis(step, parset, H, 'dir') nband_min = int( parset.getString('.'.join(["LoSoTo.Steps", step, "MinBands"]), '8')) niter = int( parset.getString('.'.join(["LoSoTo.Steps", step, "NumIter"]), '1')) dist_cut_m = np.float( parset.getString('.'.join(["LoSoTo.Steps", step, "DistCut"]), '2e3')) nstations_max = int( parset.getString('.'.join(["LoSoTo.Steps", step, "MaxStations"]), '100')) outSolset = parset.getString('.'.join(["LoSoTo.Steps", step, "OutSoltab"]), '').split('/')[0] outSoltab = parset.getString('.'.join(["LoSoTo.Steps", step, "OutSoltab"]), '').split('/')[1] # Collect solutions, etc. into arrays for fitting. (phases0, phases1, flags, mask, station_names, station_positions, source_names, source_positions, freqs, times, pointing, soln_type) = collect_solutions(H, dirs=dirs, solsets=solsets) if phases0 is None: return 1 # Build list of stations to include included_stations = [] if ants is not None: for ant in ants: included_stations += [ s for s in station_names if re.search(ant, s) ] else: included_stations = station_names excluded_stations = [ s for s in station_names if s not in included_stations ] # Select stations to use for first pass if len(excluded_stations) > 0: logging.info("Excluding stations: {0}".format( np.sort(excluded_stations))) mean_position = np.array([ np.median(station_positions[:, 0]), np.median(station_positions[:, 1]), np.median(station_positions[:, 2]) ]) dist = np.sqrt(np.sum((station_positions - mean_position)**2, axis=1)) dist_sort_ind = np.argsort(dist) station_selection1 = find(dist < dist_cut_m) station_selection = np.array([ i for i in dist_sort_ind if i in station_selection1 and station_names[i] not in excluded_stations ]) if len(station_selection) > nstations_max: station_selection = station_selection[:nstations_max] logging.info("Using normal fitting (no iterative search) for {0} stations " "within {1} km of the core:\n{2}".format( len(station_selection), dist_cut_m / 1000.0, station_names[station_selection])) station_selection_orig = station_selection for iter in xrange(niter): # Loop over groups of nearby pierce points to identify bad stations and # remove them from the station_selection. nsig = 2.5 # number of sigma for cut radius = 2.0 # projected radius in km within which to compare if iter > 0: logging.info("Identifying bad stations from outlier TEC fits...") logging.info( "Finding nearby piercepoints (assuming typical screen " "height of 200 km...") # For each source, find all the pierce points within give (projected) # distance from the median pierce point x,y location. Assume a typical # screen height of 200 km. pp, airmass = calculate_piercepoints( station_positions[station_selection], source_positions[source_selection], times, height=200e3) pp = pp[0, :, :] # use first time [times, stations, dimension] x, y, z = station_positions[station_selection][0, :] east = np.array([-y, x, 0]) east = east / norm(east) north = np.array([-x, -y, (x * x + y * y) / z]) north = north / norm(north) up = np.array([x, y, z]) up = up / norm(up) T = np.concatenate([east[:, np.newaxis], north[:, np.newaxis]], axis=1) pp1 = np.dot(pp, T).reshape( (len(source_names), len(station_selection), 2)) for i in xrange(len(source_names)): x_median = np.median(pp1[i, :, 0]) / 1000.0 y_median = np.median(pp1[i, :, 1]) / 1000.0 dist = np.sqrt((pp1[i, :, 0] / 1000.0 - x_median)**2 + (pp1[i, :, 1] / 1000.0 - y_median)**2) within_radius = np.where(dist <= radius)[0] outside_radius = np.where(dist > radius) if len(within_radius) < 10: logging.info( "Insufficient number of closely-spaced pierce " "points for bad-station detection. Skipping...") abort_iter = True break else: abort_iter = False r_median = np.median(r[i, :, within_radius], axis=1) r_tot_meddiff = np.zeros(len(station_selection[within_radius]), dtype=float) for j in xrange(len(station_selection[within_radius])): r_tot_meddiff[j] = np.sum( np.abs(r[i, :, within_radius[j]] - r_median[j])) if abort_iter: break good_stations = np.where( r_tot_meddiff < nsig * np.median(r_tot_meddiff)) station_selection = np.append( station_selection[within_radius[good_stations]], station_selection[outside_radius]) new_excluded_stations = [ station_names[s] for s in station_selection_orig if (s not in station_selection and s not in excluded_stations) ] if len(new_excluded_stations) > 0: logging.info( 'Excluding stations due to TEC solutions that differ ' 'significantly from mean: {0}'.format( np.sort(new_excluded_stations))) logging.info('Updating fit...') excluded_stations += new_excluded_stations nstations_max -= len(new_excluded_stations) else: logging.info('No bad stations found.') break # Fit a TEC value to the phase solutions per source pair. # No iterative search for the global minimum is done if soln_type == 'scalarphase': r, source_selection = fit_tec_per_source_pair( phases0[:, station_selection, :, :], flags[:, station_selection, :, :], mask, freqs, propagate=True, nband_min=nband_min) if r is None: return 1 else: r0, source_selection = fit_tec_per_source_pair( phases0[:, station_selection, :, :], flags[:, station_selection, :, :], mask, freqs, propagate=True, nband_min=nband_min) r1, source_selection = fit_tec_per_source_pair( phases1[:, station_selection, :, :], flags[:, station_selection, :, :], mask, freqs, propagate=True, nband_min=nband_min) if r0 is None or r1 is None: return 1 # take the mean of the two polarizations r = (r0 + r1) / 2 # Add stations by searching iteratively for global minimum in solution space station_selection, r = add_stations(station_selection, phases0, phases1, flags, mask, station_names, station_positions, source_names, source_selection, times, freqs, r, nband_min=nband_min, soln_type=soln_type, nstations_max=nstations_max, excluded_stations=excluded_stations, search_full_tec_range=False) # Save TEC values to the output solset solset = H.makeSolset(outSolset) dirs_out = [] dirs_pos = [] for s in source_selection: dirs_out.append(source_names[s]) dirs_pos.append(source_positions[s]) sourceTable = solset._f_get_child('source') sourceTable.append(zip(*(dirs_out, dirs_pos))) times_out = times ants_out = [] ants_pos = [] for s in station_selection: ants_out.append(station_names[s]) ants_pos.append(station_positions[s]) antennaTable = solset._f_get_child('antenna') antennaTable.append(zip(*(ants_out, ants_pos))) tf_st = H.makeSoltab(solset._v_name, 'tec', outSoltab, axesNames=['dir', 'time', 'ant'], axesVals=[dirs_out, times, ants_out], vals=r[source_selection, :, :], weights=np.ones_like(r[source_selection, :, :])) # Add history sw = solWriter(tf_st) sw.addHistory('CREATE (by TECFIT operation)') return 0
def run(step, parset, H): """ Fits a screen to TEC values derived by the TECFIT operation. The TEC values are read from the specified tec soltab. The results of the fit are stored in the specified tecscreen solution table. These values are the screen TEC values per station per pierce point per solution interval. The pierce point locations are stored in an auxiliary array in the output solution table. TEC screens can be plotted with the PLOT operation by setting PlotType = TECScreen. The H5parm_exporter.py tool can be used to export the screen to a parmdb that BBS and the AWimager can use. Note, however, that the output screens are not normalized properly (any normalization was lost due to the use of source-to-source phase gradients in the TECFIT operation). Therefore, a direction-independent calibration must be done after exporting the screens to a parmdb file, with the following settings in the BBS solve step: Model.Ionosphere.Enable = T Model.Ionosphere.Type = EXPION """ import numpy as np import re from losoto.h5parm import solFetcher, solWriter # Switch to the Agg backend to prevent problems with pylab imports when # DISPLAY env. variable is not set import os if 'DISPLAY' not in os.environ: import matplotlib matplotlib.use("Agg") soltabs = getParSoltabs(step, parset, H) outSoltabs = parset.getStringVector( '.'.join(["LoSoTo.Steps", step, "OutSoltab"]), []) height = np.array( parset.getDoubleVector('.'.join(["LoSoTo.Steps", step, "Height"]), [200e3])) order = int( parset.getString('.'.join(["LoSoTo.Steps", step, "Order"]), '15')) # Load TEC values from TECFIT operation indx = 0 for soltab in openSoltabs(H, soltabs): if 'tec' not in soltab._v_title: logging.warning( 'No TECFIT solution tables found for solution table ' '{0}'.format(soltabs[indx])) continue indx += 1 solset = soltabs[indx].split('/')[0] logging.info('Using input solution table: {0}'.format(soltabs[indx])) logging.info('Using output solution table: {0}'.format( outSoltabs[indx])) # Collect station and source names and positions and times, making sure # that they are ordered correctly. t = solFetcher(soltab) r, axis_vals = t.getValues() source_names = axis_vals['dir'] source_dict = H.getSou(solset) source_positions = [] for source in source_names: source_positions.append(source_dict[source]) station_names = axis_vals['ant'] station_dict = H.getAnt(solset) station_positions = [] for station in station_names: station_positions.append(station_dict[station]) times = axis_vals['time'] # Get sizes N_sources = len(source_names) N_times = len(times) N_stations = len(station_names) N_piercepoints = N_sources * N_stations rr = np.reshape(r.transpose([0, 2, 1]), [N_piercepoints, N_times]) heights = list(set(np.linspace(height[0], height[-1], 5))) heights.sort() if len(heights) > 1: logging.info('Trying range of heights: {0} m'.format(heights)) for i, height in enumerate(heights): # Find pierce points and airmass values for given screen height logging.info('Using height = {0} m and order = {1}'.format( height, order)) if height < 100e3: logging.warning("Height is less than 100e3 m.") pp, airmass = calculate_piercepoints(np.array(station_positions), np.array(source_positions), np.array(times), height) # Fit a TEC screen r_0 = 10e3 beta = 5.0 / 3.0 tec_screen, residual = fit_screen_to_tec(station_names, source_names, pp, airmass, rr, times, height, order, r_0, beta) total_resid = np.sum(np.abs(residual)) if i > 0: if total_resid < best_resid: tec_screen_best = tec_screen pp_best = pp height_best = height best_resid = total_resid else: tec_screen_best = tec_screen pp_best = pp height_best = height best_resid = total_resid if len(heights) > 1: logging.info( 'Total residual for fit: {0}\n'.format(total_resid)) # Use screen with lowest total residual if len(heights) > 1: tec_screen = tec_screen_best pp = pp_best height = height_best logging.info( 'Using height (with lowest total residual) of {0} m'.format( height)) # Write the results to the output solset dirs_out = source_names times_out = times ants_out = station_names # Make output tecscreen table outSolset = outSoltabs[indx].split('/')[0] outSoltab = outSoltabs[indx].split('/')[1] if not outSolset in H.getSolsets().keys(): solsetTEC = H.makeSolset(outSolset) dirs_pos = source_positions sourceTable = solsetTEC._f_get_child('source') sourceTable.append(zip(*(dirs_out, dirs_pos))) ants_pos = station_positions antennaTable = solsetTEC._f_get_child('antenna') antennaTable.append(zip(*(ants_out, ants_pos))) # Store tecscreen values. The residual values are stored in the weights # table. Flagged values of the screen have weights set to 0.0. vals = tec_screen.transpose([1, 0, 2]) weights = residual.transpose([1, 0, 2]) tec_screen_st = H.makeSoltab(outSolset, 'tecscreen', outSoltab, axesNames=['dir', 'time', 'ant'], axesVals=[dirs_out, times_out, ants_out], vals=vals, weights=weights) # Store beta, r_0, height, and order as attributes of the tecscreen # soltab tec_screen_st._v_attrs['beta'] = beta tec_screen_st._v_attrs['r_0'] = r_0 tec_screen_st._v_attrs['height'] = height tec_screen_st._v_attrs['order'] = order # Make output piercepoint table tec_screen_solset = tec_screen_st._v_parent._v_name H.H.create_carray('/' + tec_screen_solset + '/' + tec_screen_st._v_name, 'piercepoint', obj=pp) # Add histories sw = solWriter(tec_screen_st) sw.addHistory('CREATE (by TECSCREEN operation)') indx += 1 return 0
elapsed = (time.clock() - start) logging.info("PARMDB -- " + str(elapsed) + " s.") start = time.clock() for i in range(n): H.setSelection(dir='pointing', ant='RS*') Hrot = H.getValues(retAxesVals=False) elapsed = (time.clock() - start) logging.info("H5parm -- " + str(elapsed) + " s.") #print "Equal?", (Prot == np.squeeze(Hrot)).all() ###################################################### # read+write logging.info("### Read all rotations for a dir/station and write them back") Hw = solWriter(H5.getSoltab(solset, 'rotation000')) start = time.clock() for i in range(n): Prot = P.getValuesGrid('CommonRotationAngle:CS001LBA') Prot = {'test' + str(i): Prot['CommonRotationAngle:CS001LBA']} P2.addValues(Prot) # parmdb write? elapsed = (time.clock() - start) logging.info("PARMDB -- " + str(elapsed) + " s.") start = time.clock() for i in range(n): H.setSelection(dir='pointing', ant='CS001LBA') Hrot = H.getValues(retAxesVals=False) Hw.setSelection(dir='pointing', ant='CS001LBA')
def run( step, parset, H ): """ subtract a clock and/or tec from a phase. """ import numpy as np from losoto.h5parm import solFetcher, solWriter soltabs = getParSoltabs( step, parset, H ) soltabsToSub = parset.getStringVector('.'.join(["LoSoTo.Steps", step, "Sub"]), [] ) ratio = parset.getBool('.'.join(["LoSoTo.Steps", step, "Ratio"]), False ) for soltab in openSoltabs( H, soltabs ): logging.info("--> Working on soltab: "+soltab._v_name) sf = solFetcher(soltab) sw = solWriter(soltab, useCache = True) sfss = [] # sol fetcher to sub tables for soltabToSub in soltabsToSub: ss, st = soltabToSub.split('/') sfs = solFetcher(H.getSoltab(ss, st)) if sf.getType() != 'phase' and (sfs.getType() == 'tec' or sfs.getType() == 'clock' or sfs.getType() == 'rotationmeasure' or sfs.getType() == 'tec3rd'): logging.warning(soltabToSub+' is of type clock/tec/rm and should be subtracted from a phase. Skipping it.') continue sfss.append( sfs ) logging.info('Subtracting table: '+soltabToSub) # a major speed up if tables are assumed with same axes, check that (should be the case in almost any case) for axisName in sfs.getAxesNames(): assert all(sfs.getAxisValues(axisName) == sf.getAxisValues(axisName)) if sf.getType() == 'phase' and (sfs.getType() == 'tec' or sfs.getType() == 'clock' or sfs.getType() == 'rotationmeasure' or sfs.getType() == 'tec3rd' ): # the only return axes is freq, slower but better code for vals, weights, coord, selection in sf.getValuesIter(returnAxes='freq', weight = True): for sfs in sfss: # restrict to have the same coordinates of phases for i, axisName in enumerate(sfs.getAxesNames()): sfs.selection[i] = selection[sf.getAxesNames().index(axisName)] valsSub = np.squeeze(sfs.getValues(retAxesVals=False, weight=False)) weightsSub = np.squeeze(sfs.getValues(retAxesVals=False, weight=True)) if sfs.getType() == 'clock': vals -= 2. * np.pi * valsSub * coord['freq'] elif sfs.getType() == 'tec': vals -= -8.44797245e9 * valsSub / coord['freq'] elif sfs.getType() == 'tec3rd': vals -= - 1.e21 * valsSub / np.power(coord['freq'],3) elif sfs.getType() == 'rotationmeasure': wav = 2.99792458e8/coord['freq'] ph = wav * wav * valsSub if coord['pol'] == 'XX' or coord['pol'] == 'RR': vals -= ph elif coord['pol'] == 'YY' or coord['pol'] == 'LL': vals += ph else: vals -= valsSub # flag data that are contaminated by flagged clock/tec data if weightsSub == 0: weights[:] = 0 sw.selection = selection sw.setValues(vals) sw.setValues(weights, weight = True) else: if ratio: sw.setValues((sf.getValues(retAxesVals=False)-sfs.getValues(retAxesVals=False))/sfs.getValues(retAxesVals=False)) else: sw.setValues(sf.getValues(retAxesVals=False)-sfs.getValues(retAxesVals=False)) weight = sf.getValues(retAxesVals=False, weight=True) weight[sfs.getValues(retAxesVals=False, weight=True) == 0] = 0 sw.setValues(weight, weight = True) sw.addHistory('RESIDUALS by subtracting tables '+' '.join(soltabsToSub)) sw.flush() del sf del sw return 0
def run( step, parset, H ): """ Fit phase solutions to obtain TEC values per direction. Phase solutions are assumed to be stored in solsets of the H5parm file, one solset per band per field. Only phase- or scalarphase-type solution tables are used. If direction-independent solutions are found (in addition to the direction-dependent ones), they are added, after averaging, to the corresponding direction-dependent ones. Phase solutions are automatically grouped by field and by band. The derived TEC values are stored in the specified output soltab of type 'tec', with one TEC value per station per direction per solution interval. The TEC values are derived using the ``lofar.expion.baselinefitting.fit()`` function to fit a TEC value to the phases. The term that is minimized includes all baselines, so there is no preferred reference station, and the residual is computed as the complex number 1.0 - exp(1i phasedifference), which is zero when the phase difference is a multiple of 2pi. The TEC solution table may be used to derive TEC screens using the TECSCREEN operation. """ from losoto.h5parm import solFetcher, solWriter import numpy as np # Switch to the Agg backend to prevent problems with pylab imports when # DISPLAY env. variable is not set import os if 'DISPLAY' not in os.environ: import matplotlib matplotlib.use("Agg") from pylab import find import re from .tecscreen import calculate_piercepoints from numpy.linalg import norm solsets = getParSolsets( step, parset, H ) ants = getParAxis( step, parset, H, 'ant' ) pols = getParAxis( step, parset, H, 'pol' ) dirs = getParAxis( step, parset, H, 'dir' ) nband_min = int(parset.getString('.'.join(["LoSoTo.Steps", step, "MinBands"]), '8' )) niter = int(parset.getString('.'.join(["LoSoTo.Steps", step, "NumIter"]), '1' )) dist_cut_m = np.float(parset.getString('.'.join(["LoSoTo.Steps", step, "DistCut"]), '2e3' )) nstations_max = int(parset.getString('.'.join(["LoSoTo.Steps", step, "MaxStations"]), '100' )) outSolset = parset.getString('.'.join(["LoSoTo.Steps", step, "OutSoltab"]), '' ).split('/')[0] outSoltab = parset.getString('.'.join(["LoSoTo.Steps", step, "OutSoltab"]), '' ).split('/')[1] # Collect solutions, etc. into arrays for fitting. (phases0, phases1, flags, mask, station_names, station_positions, source_names, source_positions, freqs, times, pointing, soln_type) = collect_solutions(H, dirs=dirs, solsets=solsets) if phases0 is None: return 1 # Build list of stations to include included_stations = [] if ants is not None: for ant in ants: included_stations += [s for s in station_names if re.search(ant, s)] else: included_stations = station_names excluded_stations = [s for s in station_names if s not in included_stations] # Select stations to use for first pass if len(excluded_stations) > 0: logging.info("Excluding stations: {0}".format(np.sort(excluded_stations))) mean_position = np.array([np.median(station_positions[:, 0]), np.median(station_positions[:, 1]), np.median(station_positions[:, 2])]) dist = np.sqrt(np.sum((station_positions - mean_position)**2, axis=1)) dist_sort_ind = np.argsort(dist) station_selection1 = find(dist < dist_cut_m) station_selection = np.array([i for i in dist_sort_ind if i in station_selection1 and station_names[i] not in excluded_stations]) if len(station_selection) > nstations_max: station_selection = station_selection[:nstations_max] logging.info("Using normal fitting (no iterative search) for {0} stations " "within {1} km of the core:\n{2}".format(len(station_selection), dist_cut_m/1000.0, station_names[station_selection])) station_selection_orig = station_selection for iter in xrange(niter): # Loop over groups of nearby pierce points to identify bad stations and # remove them from the station_selection. nsig = 2.5 # number of sigma for cut radius = 2.0 # projected radius in km within which to compare if iter > 0: logging.info("Identifying bad stations from outlier TEC fits...") logging.info("Finding nearby piercepoints (assuming typical screen " "height of 200 km...") # For each source, find all the pierce points within give (projected) # distance from the median pierce point x,y location. Assume a typical # screen height of 200 km. pp, airmass = calculate_piercepoints(station_positions[station_selection], source_positions[source_selection], times, height = 200e3) pp = pp[0, :, :] # use first time [times, stations, dimension] x, y, z = station_positions[station_selection][0,:] east = np.array([-y, x, 0]) east = east / norm(east) north = np.array([ -x, -y, (x*x + y*y)/z]) north = north / norm(north) up = np.array([x ,y, z]) up = up / norm(up) T = np.concatenate([east[:, np.newaxis], north[:, np.newaxis]], axis=1) pp1 = np.dot(pp, T).reshape((len(source_names), len(station_selection), 2)) for i in xrange(len(source_names)): x_median = np.median(pp1[i, :, 0]) / 1000.0 y_median = np.median(pp1[i, :, 1]) / 1000.0 dist = np.sqrt( (pp1[i, :, 0] / 1000.0 - x_median)**2 + (pp1[i, :, 1] / 1000.0 - y_median)**2 ) within_radius = np.where(dist <= radius)[0] outside_radius = np.where(dist > radius) if len(within_radius) < 10: logging.info("Insufficient number of closely-spaced pierce " "points for bad-station detection. Skipping...") abort_iter = True break else: abort_iter = False r_median = np.median(r[i, :, within_radius], axis=1) r_tot_meddiff = np.zeros(len(station_selection[within_radius]), dtype=float) for j in xrange(len(station_selection[within_radius])): r_tot_meddiff[j] = np.sum(np.abs(r[i, :, within_radius[j]] - r_median[j])) if abort_iter: break good_stations = np.where(r_tot_meddiff < nsig * np.median(r_tot_meddiff)) station_selection = np.append(station_selection[within_radius[good_stations]], station_selection[outside_radius]) new_excluded_stations = [station_names[s] for s in station_selection_orig if (s not in station_selection and s not in excluded_stations)] if len(new_excluded_stations) > 0: logging.info('Excluding stations due to TEC solutions that differ ' 'significantly from mean: {0}'.format(np.sort(new_excluded_stations))) logging.info('Updating fit...') excluded_stations += new_excluded_stations nstations_max -= len(new_excluded_stations) else: logging.info('No bad stations found.') break # Fit a TEC value to the phase solutions per source pair. # No iterative search for the global minimum is done if soln_type == 'scalarphase': r, source_selection = fit_tec_per_source_pair( phases0[:, station_selection, :, :], flags[:, station_selection, :, :], mask, freqs, propagate=True, nband_min=nband_min) if r is None: return 1 else: r0, source_selection = fit_tec_per_source_pair( phases0[:, station_selection, :, :], flags[:, station_selection, :, :], mask, freqs, propagate=True, nband_min=nband_min) r1, source_selection = fit_tec_per_source_pair( phases1[:, station_selection, :, :], flags[:, station_selection, :, :], mask, freqs, propagate=True, nband_min=nband_min) if r0 is None or r1 is None: return 1 # take the mean of the two polarizations r = (r0 + r1) / 2 # Add stations by searching iteratively for global minimum in solution space station_selection, r = add_stations(station_selection, phases0, phases1, flags, mask, station_names, station_positions, source_names, source_selection, times, freqs, r, nband_min=nband_min, soln_type=soln_type, nstations_max=nstations_max, excluded_stations= excluded_stations, search_full_tec_range=False) # Save TEC values to the output solset solset = H.makeSolset(outSolset) dirs_out = [] dirs_pos = [] for s in source_selection: dirs_out.append(source_names[s]) dirs_pos.append(source_positions[s]) sourceTable = solset._f_get_child('source') sourceTable.append(zip(*(dirs_out, dirs_pos))) times_out = times ants_out = [] ants_pos = [] for s in station_selection: ants_out.append(station_names[s]) ants_pos.append(station_positions[s]) antennaTable = solset._f_get_child('antenna') antennaTable.append(zip(*(ants_out, ants_pos))) tf_st = H.makeSoltab(solset._v_name, 'tec', outSoltab, axesNames=['dir', 'time', 'ant'], axesVals=[dirs_out, times, ants_out], vals=r[source_selection, :, :], weights=np.ones_like(r[source_selection, :, :])) # Add history sw = solWriter(tf_st) sw.addHistory('CREATE (by TECFIT operation)') return 0
def run( step, parset, H ): import numpy as np from losoto.h5parm import solFetcher, solWriter soltabs = getParSoltabs( step, parset, H ) axesToClip = parset.getStringVector('.'.join(["LoSoTo.Steps", step, "Axes"]), [] ) clipLevel = parset.getFloat('.'.join(["LoSoTo.Steps", step, "ClipLevel"]), 0. ) log = parset.getBool('.'.join(["LoSoTo.Steps", step, "Log"]), True ) if len(axesToClip) < 1: logging.error("Please specify axes to clip.") return 1 if clipLevel == 0.: logging.error("Please specify factor above/below median at which to clip.") return 1 for soltab in openSoltabs( H, soltabs ): logging.info("Clipping soltab: "+soltab._v_name) sf = solFetcher(soltab) # axis selection userSel = {} for axis in sf.getAxesNames(): userSel[axis] = getParAxis( step, parset, H, axis ) sf.setSelection(**userSel) # some checks for i, axis in enumerate(axesToClip[:]): if axis not in sf.getAxesNames(): del axesToClip[i] logging.warning('Axis \"'+axis+'\" not found. Ignoring.') if sf.getType() != 'amplitude': logging.error('CLIP is for "amplitude" tables, not %s.' % sf.getType()) continue sw = solWriter(soltab, useCache=True) # remember to flush() before_count=0 after_count=0 total=0 for vals, weights, coord, selection in sf.getValuesIter(returnAxes=axesToClip, weight = True): total+=len(vals) before_count+=(len(weights)-np.count_nonzero(weights)) # first find the median and standard deviation if (weights == 0).all(): valmedian = 0 else: if log: valmedian = np.median(np.log10(vals[(weights != 0)])) rms = np.std(np.log10(vals[(weights != 0)])) np.putmask(weights, np.abs(np.log10(vals)-valmedian) > rms * clipLevel, 0) else: valmedian = np.median(vals[(weights != 0)]) rms = np.std(vals[(weights != 0)]) np.putmask(weights, np.abs(vals-valmedian) > rms * clipLevel, 0) after_count+=(len(weights)-np.count_nonzero(weights)) # writing back the solutions sw.selection = selection sw.setValues(weights, weight=True) sw.addHistory('CLIP (over %s with %s sigma cut)' % (axesToClip, clipLevel)) logging.info('Clip, flagged data: %f %% -> %f %%' \ % (100.*before_count/total, 100.*after_count/total)) sw.flush() return 0
def run( step, parset, H ): from losoto.h5parm import solFetcher, solWriter import numpy as np import scipy.optimize delaycomplex = lambda d, freq, y: abs(np.cos(d[0]*freq) - np.cos(y)) + abs(np.sin(d[0]*freq) - np.sin(y)) # get involved solsets using local step values or global values or all soltabs = getParSoltabs( step, parset, H ) refAnt = parset.getString('.'.join(["LoSoTo.Steps", step, "RefAnt"]), '' ) outTab = parset.getString('.'.join(["LoSoTo.Steps", step, "OutTable"]), 'phasediff' ) maxres = parset.getFloat('.'.join(["LoSoTo.Steps", step, "MaxResidual"]), 1.) for t, soltab in enumerate(openSoltabs( H, soltabs )): logging.info("--> Working on soltab: "+soltab._v_name) sf = solFetcher(soltab) # times and ants needs to be complete or selection is much slower times = sf.getAxisValues('time') ants = sf.getAxisValues('ant') # this will make a selection for the getValues() and getValuesIter() userSel = {} for axis in sf.getAxesNames(): userSel[axis] = getParAxis( step, parset, H, axis ) sf.setSelection(**userSel) # some checks solType = sf.getType() if solType != 'phase': logging.warning("Soltab type of "+soltab._v_name+" is of type "+solType+", should be phase. Ignoring.") continue if refAnt != '' and not refAnt in ants: logging.error('Reference antenna '+refAnt+' not found.') return 1 if refAnt == '': refAnt = ants[0] # create new table solsetname = soltabs[t].split('/')[0] st = H.makeSoltab(solsetname, soltype = sf.getType(), soltab = outTab, axesNames=sf.getAxesNames(), \ axesVals=[sf.getAxisValues(axisName) for axisName in sf.getAxesNames()], \ vals=sf.getValues(retAxesVals = False), weights=sf.getValues(weight = True, retAxesVals = False), parmdbType=sf.t._v_attrs['parmdb_type']) sw = solWriter(st) sw.addHistory('Created by CROSSDELAY operation.') for vals, weights, coord, selection in sf.getValuesIter(returnAxes=['freq','pol','time'], weight=True, reference = refAnt): fitdelayguess = 1.e-10 # good guess, do not use 0 as it seems the minimizer is unstable with that if 'RR' in coord['pol'] and 'LL' in coord['pol']: coord1 = np.where(coord['pol'] == 'RR')[0][0] coord2 = np.where(coord['pol'] == 'LL')[0][0] elif 'XX' in coord['pol'] and 'YY' in coord['pol']: coord1 = np.where(coord['pol'] == 'XX')[0][0] coord2 = np.where(coord['pol'] == 'YY')[0][0] if not coord['ant'] == refAnt: logging.debug('Working on ant: '+coord['ant']+'...') if (weights == 0.).all() == True: logging.warning('Skipping flagged antenna: '+coord['ant']) weights[:] = 0 else: for t, time in enumerate(times): # apply flags idx = ((weights[coord1,:,t] != 0.) & (weights[coord2,:,t] != 0.)) freq = np.copy(coord['freq'])[idx] phase1 = vals[coord1,:,t][idx] phase2 = vals[coord2,:,t][idx] if len(freq) < 10: vals[:,:,t] = 0. weights[:,:,t] = 0. logging.debug('Not enough unflagged point for the timeslot '+str(t)) continue if (len(idx) - len(freq))/len(freq) > 1/4.: logging.debug('High number of filtered out data points for the timeslot '+str(t)+': '+str(len(idx) - len(freq))) phase_diff = (phase1 - phase2) fitresultdelay, success = scipy.optimize.leastsq(delaycomplex, [fitdelayguess], args=(freq, phase_diff)) #if t%100==0: print fitresultdelay # fractional residual residual = np.mean(np.abs(np.mod(fitresultdelay*freq-phase_diff + np.pi, 2.*np.pi) - np.pi)) #print "t:", t, "result:", fitresultdelay, "residual:", residual if residual < maxres: fitdelayguess = fitresultdelay[0] weight = 1 else: # high residual, flag logging.warning('Bad solution for ant: '+coord['ant']+' (time: '+str(t)+', resdiaul: '+str(residual)+').') weight = 0 vals[:,:,t] = 0. vals[coord1,:,t][idx] = fitresultdelay[0]*freq/2. vals[coord2,:,t][idx] = -1.*(fitresultdelay[0]*freq)/2. weights[:,:,t] = 0. weights[coord1,:,t][idx] = weight weights[coord2,:,t][idx] = weight # Debug plot doplot = False if doplot and t%500==0 and coord['ant'] == 'CS004LBA': if not 'matplotlib' in sys.modules: import matplotlib as mpl mpl.rc('font',size =8 ) mpl.rc('figure.subplot',left=0.05, bottom=0.05, right=0.95, top=0.95,wspace=0.22, hspace=0.22 ) mpl.use("Agg") import matplotlib.pyplot as plt fig = plt.figure() fig.subplots_adjust(wspace=0) ax = fig.add_subplot(111) # plot rm fit plotdelay = lambda delay, freq: np.mod( delay*freq + np.pi, 2.*np.pi) - np.pi ax.plot(freq, plotdelay(fitresultdelay[0], freq), "-", color='purple') ax.plot(freq, np.mod(phase1 + np.pi, 2.*np.pi) - np.pi, 'ob' ) ax.plot(freq, np.mod(phase2 + np.pi, 2.*np.pi) - np.pi, 'og' ) ax.plot(freq, np.mod(phase_diff + np.pi, 2.*np.pi) - np.pi , '.', color='purple' ) residual = np.mod(plotdelay(fitresultdelay[0], freq)-phase_diff + np.pi,2.*np.pi)-np.pi ax.plot(freq, residual, '.', color='yellow') ax.set_xlabel('freq') ax.set_ylabel('phase') ax.set_ylim(ymin=-np.pi, ymax=np.pi) logging.warning('Save pic: '+str(t)+'_'+coord['ant']+'.png') plt.savefig(coord['ant']+'_'+str(t)+'.png', bbox_inches='tight') del fig sw.setSelection(**coord) sw.setValues( vals ) sw.setValues( weights, weight=True ) del st del sw del sf return 0
def run(step, parset, H): from losoto.h5parm import solFetcher, solWriter soltabs = getParSoltabs(step, parset, H) #check_parset('Axes','MaxCycles','MaxRms','Order','Replace','PreFlagZeros') axesToFlag = parset.getStringVector( '.'.join(["LoSoTo.Steps", step, "Axes"]), 'time') maxCycles = parset.getInt('.'.join(["LoSoTo.Steps", step, "MaxCycles"]), 5) maxRms = parset.getFloat('.'.join(["LoSoTo.Steps", step, "MaxRms"]), 5.) fixRms = parset.getFloat('.'.join(["LoSoTo.Steps", step, "FixRms"]), 0) order = parset.getIntVector('.'.join(["LoSoTo.Steps", step, "Order"]), 3) replace = parset.getBool('.'.join(["LoSoTo.Steps", step, "Replace"]), False) preflagzeros = parset.getBool( '.'.join(["LoSoTo.Steps", step, "PreFlagZeros"]), False) mode = parset.getString('.'.join(["LoSoTo.Steps", step, "Mode"]), 'smooth') ref = parset.getString('.'.join(["LoSoTo.Steps", step, "Reference"]), '') ncpu = parset.getInt('.'.join(["LoSoTo.Ncpu"]), 1) if ref == '': ref = None if axesToFlag == []: logging.error("Please specify axis to flag. It must be a single one.") return 1 if len(axesToFlag) != len(order): logging.error("AxesToFlag and order must be both 1 or 2 values.") return 1 if len(order) == 1: order = order[0] elif len(order) == 2: order = tuple(order) mode = mode.lower() if mode != 'smooth' and mode != 'poly' and mode != 'spline': logging.error('Mode must be smooth, poly or spline') return 1 # start processes for multi-thread mpm = multiprocManager(ncpu, flag) for soltab in openSoltabs(H, soltabs): logging.info("Flagging soltab: " + soltab._v_name) sf = solFetcher(soltab) sw = solWriter(soltab, useCache=True) # remember to flush! # axis selection userSel = {} for axis in sf.getAxesNames(): userSel[axis] = getParAxis(step, parset, H, axis) sf.setSelection(**userSel) for axisToFlag in axesToFlag: if axisToFlag not in sf.getAxesNames(): logging.error('Axis \"' + axis + '\" not found.') return 1 # reorder axesToFlag as axes in the table axesToFlag_orig = axesToFlag axesToFlag = [ coord for coord in sf.getAxesNames() if coord in axesToFlag ] if type(order) is int: order = [order] if axesToFlag_orig != axesToFlag: order = order[::-1] # reverse order if we changed axesToFlag solType = sf.getType() # fill the queue (note that sf and sw cannot be put into a queue since they have file references) for vals, weights, coord, selection in sf.getValuesIter( returnAxes=axesToFlag, weight=True, reference=ref): mpm.put([ vals, weights, coord, solType, order, mode, preflagzeros, maxCycles, fixRms, maxRms, replace, axesToFlag, selection ]) #v, w, sel = flag(vals, weights, coord, solType, order, mode, preflagzeros, maxCycles, fixRms, maxRms, replace, axesToFlag, selection) mpm.wait() for v, w, sel in mpm.get(): sw.selection = sel if replace: # rewrite solutions (flagged values are overwritten) sw.setValues(v, weight=False) else: sw.setValues(w, weight=True) sw.flush() sw.addHistory('FLAG (over %s with %s sigma cut)' % (axesToFlag, maxRms)) del sw del sf del soltab return 0
def run( step, parset, H ): """ Separate phase solutions into FR, Clock and TEC. The Clock and TEC values are stored in the specified output soltab with type 'clock', 'tec', 'FR'. """ from losoto.h5parm import solFetcher, solWriter import numpy as np import scipy.optimize rmwavcomplex = lambda RM, wav, y: abs(np.cos(2.*RM[0]*wav*wav) - np.cos(y)) + abs(np.sin(2.*RM[0]*wav*wav) - np.sin(y)) c = 2.99792458e8 # get involved solsets using local step values or global values or all soltabs = getParSoltabs( step, parset, H ) refAnt = parset.getString('.'.join(["LoSoTo.Steps", step, "RefAnt"]), '' ) ncpu = parset.getInt('.'.join(["LoSoTo.Ncpu"]), 1 ) for t, soltab in enumerate(openSoltabs( H, soltabs )): logging.info("--> Working on soltab: "+soltab._v_name) sf = solFetcher(soltab) # times and ants needs to be complete or selection is much slower times = sf.getAxisValues('time') ants = sf.getAxisValues('ant') # this will make a selection for the getValues() and getValuesIter() userSel = {} for axis in sf.getAxesNames(): userSel[axis] = getParAxis( step, parset, H, axis ) sf.setSelection(**userSel) # some checks solType = sf.getType() if solType != 'phase': logging.warning("Soltab type of "+soltab._v_name+" is of type "+solType+", should be phase. Ignoring.") continue if refAnt != '' and not refAnt in ants: logging.error('Reference antenna '+refAnt+' not found.') return 1 if refAnt == '': refAnt = ants[0] solsetname = soltabs[t].split('/')[0] st = H.makeSoltab(solsetname, 'rotationmeasure', axesNames=['ant','time'], axesVals=[ants, times], vals=np.zeros((len(ants),len(times))), weights=np.ones((len(ants),len(times)))) sw = solWriter(st) sw.addHistory('Created by FARADAY operation.') for vals, weights, coord, selection in sf.getValuesIter(returnAxes=['freq','pol','time'], weight=True, reference = refAnt): if len(coord['freq']) < 10: logging.error('Faraday rotation estimation needs at least 10 frequency channels, preferably distributed over a wide range.') return 1 fitrm = np.zeros(len(times)) fitweights = np.ones(len(times)) fitrmguess = 0 # good guess if 'RR' in coord['pol'] and 'LL' in coord['pol']: coord_rr = np.where(coord['pol'] == 'RR')[0][0] coord_ll = np.where(coord['pol'] == 'LL')[0][0] elif 'XX' in coord['pol'] and 'YY' in coord['pol']: logging.warning('Linear polarization detected, LoSoTo assumes XX->RR and YY->LL.') coord_rr = np.where(coord['pol'] == 'XX')[0][0] coord_ll = np.where(coord['pol'] == 'YY')[0][0] else: logging.error("Cannot proceed with Faraday estimation with polarizations: "+str(coord['pol'])) return 1 if not coord['ant'] == refAnt: logging.debug('Working on ant: '+coord['ant']+'...') for t, time in enumerate(times): # apply flags idx = ((weights[0,:,t] != 0.) & (weights[1,:,t] != 0.)) freq = np.copy(coord['freq'])[idx] phase_rr = vals[coord_rr,:,t][idx] phase_ll = vals[coord_ll,:,t][idx] if (len(weights[0,:,t]) - len(idx))/len(weights[0,:,t]) > 1/4.: logging.debug('High number of filtered out data points for the timeslot '+str(t)+': '+str(len(weights[0,:,t]) - len(idx))) if len(freq) < 10: fitweights[t] = 0 logging.warning('No valid data found for Faraday fitting for antenna: '+coord['ant']) continue # RR-LL to be consistent with BBS/NDPPP phase_diff = (phase_rr - phase_ll) # not divide by 2 otherwise jump problem, then later fix this wav = c/freq fitresultrm_wav, success = scipy.optimize.leastsq(rmwavcomplex, [fitrmguess], args=(wav, phase_diff)) # fractional residual residual = np.mean(np.abs(np.mod((2.*fitresultrm_wav*wav*wav)-phase_diff,2.*np.pi) - np.pi)) # print "t:", t, "result:", fitresultrm_wav, "residual:", residual if residual > 0.5: fitrmguess = fitresultrm_wav[0] weight = 1 else: # high residual, flag logging.warning('Bad solution for ant: '+coord['ant']+' (time: '+str(t)+', resdiaul: '+str(residual)+').') weight = 0 # Debug plot doplot = False if doplot and coord['ant'] == 'RS310LBA' and t%10==0: print "Plotting" if not 'matplotlib' in sys.modules: import matplotlib as mpl mpl.rc('font',size =8 ) mpl.rc('figure.subplot',left=0.05, bottom=0.05, right=0.95, top=0.95,wspace=0.22, hspace=0.22 ) mpl.use("Agg") import matplotlib.pyplot as plt fig = plt.figure() fig.subplots_adjust(wspace=0) ax = fig.add_subplot(110) # plot rm fit plotrm = lambda RM, wav: np.mod( (2.*RM*wav*wav) + np.pi, 2.*np.pi) - np.pi # notice the factor of 2 ax.plot(freq, plotrm(fitresultrm_wav, c/freq[:]), "-", color='purple') ax.plot(freq, np.mod(phase_rr + np.pi, 2.*np.pi) - np.pi, 'ob' ) ax.plot(freq, np.mod(phase_ll + np.pi, 2.*np.pi) - np.pi, 'og' ) ax.plot(freq, np.mod(phase_diff + np.pi, 2.*np.pi) - np.pi , '.', color='purple' ) residual = np.mod(plotrm(fitresultrm_wav, c/freq[:])-phase_diff+np.pi,2.*np.pi)-np.pi ax.plot(freq, residual, '.', color='yellow') ax.set_xlabel('freq') ax.set_ylabel('phase') ax.set_ylim(ymin=-np.pi, ymax=np.pi) logging.warning('Save pic: '+str(t)+'_'+coord['ant']+'.png') plt.savefig(str(t)+'_'+coord['ant']+'.png', bbox_inches='tight') del fig fitrm[t] = fitresultrm_wav[0] fitweights[t] = weight sw.setSelection(ant=coord['ant'], time=coord['time']) sw.setValues( np.expand_dims(fitrm, axis=1) ) sw.setValues( np.expand_dims(fitweights, axis=1), weight=True ) del st del sw del sf return 0
def run( step, parset, H ): """ subtract a clock and/or tec from a phase. """ import numpy as np from losoto.h5parm import solFetcher, solWriter soltabs = getParSoltabs( step, parset, H ) soltabsToSub = parset.getStringVector('.'.join(["LoSoTo.Steps", step, "Sub"]), [] ) ratio = parset.getBool('.'.join(["LoSoTo.Steps", step, "Ratio"]), False ) for soltab in openSoltabs( H, soltabs ): logging.info("--> Working on soltab: "+soltab._v_name) sf = solFetcher(soltab) sw = solWriter(soltab, useCache = True) sfss = [] # sol fetcher to sub tables for soltabToSub in soltabsToSub: ss, st = soltabToSub.split('/') sfs = solFetcher(H.getSoltab(ss, st)) if sf.getType() != 'phase' and (sfs.getType() == 'tec' or sfs.getType() == 'clock' or sfs.getType() == 'rotationmeasure' or sfs.getType() == 'tec3rd'): logging.warning(soltabToSub+' is of type clock/tec/rm and should be subtracted from a phase. Skipping it.') continue sfss.append( sfs ) logging.info('Subtracting table: '+soltabToSub) # a major speed up if tables are assumed with same axes, check that (should be the case in almost any case) for axisName in sfs.getAxesNames(): assert all(sfs.getAxisValues(axisName) == sf.getAxisValues(axisName)) if sf.getType() == 'phase' and (sfs.getType() == 'tec' or sfs.getType() == 'clock' or sfs.getType() == 'rotationmeasure' or sfs.getType() == 'tec3rd' ): # the only return axes is freq, slower but better code for vals, weights, coord, selection in sf.getValuesIter(returnAxes='freq', weight = True): for sfs in sfss: # restrict to have the same coordinates of phases for i, axisName in enumerate(sfs.getAxesNames()): sfs.selection[i] = selection[sf.getAxesNames().index(axisName)] valsSub = np.squeeze(sfs.getValues(retAxesVals=False, weight=False)) weightsSub = np.squeeze(sfs.getValues(retAxesVals=False, weight=True)) if sfs.getType() == 'clock': vals -= 2. * np.pi * valsSub * coord['freq'] elif sfs.getType() == 'tec': vals -= -8.44797245e9 * valsSub / coord['freq'] elif sfs.getType() == 'tec3rd': vals -= - 1.e21 * valsSub / np.power(coord['freq'],3) elif sfs.getType() == 'rotationmeasure': wav = 2.99792458e8/coord['freq'] ph = wav * wav * valsSub if coord['pol'] == 'XX': vals += ph elif coord['pol'] == 'YY': vals -= ph else: vals -= valsSub # flag data that are contaminated by flagged clock/tec data if weightsSub == 0: weights[:] = 0 sw.selection = selection sw.setValues(vals) sw.setValues(weights, weight = True) else: if ratio: sw.setValues((sf.getValues(retAxesVals=False)-sfs.getValues(retAxesVals=False))/sfs.getValues(retAxesVals=False)) else: sw.setValues(sf.getValues(retAxesVals=False)-sfs.getValues(retAxesVals=False)) weight = sf.getValues(retAxesVals=False, weight=True) weight[sfs.getValues(retAxesVals=False, weight=True) == 0] = 0 sw.setValues(weight, weight = True) sw.addHistory('RESIDUALS by subtracting tables '+' '.join(soltabsToSub)) sw.flush() del sf del sw return 0
def run(step, parset, H): """ Separate phase solutions into Clock and TEC. Phase solutions are assumed to be stored in solsets of the H5parm file, one solset per field. The Clock and TEC values are stored in the specified output soltab with type 'clock' and 'tec'. """ import numpy as np from losoto.h5parm import solFetcher, solWriter # get involved solsets using local step values or global values or all soltabs = getParSoltabs(step, parset, H) flagBadChannels = parset.getBool( '.'.join(["LoSoTo.Steps", step, "FlagBadChannels"]), True) flagCut = parset.getFloat('.'.join(["LoSoTo.Steps", step, "FlagCut"]), 5.) chi2cut = parset.getFloat('.'.join(["LoSoTo.Steps", step, "Chi2cut"]), 3000.) combinePol = parset.getBool('.'.join(["LoSoTo.Steps", step, "CombinePol"]), False) #fitOffset = parset.getBool('.'.join(["LoSoTo.Steps", step, "FitOffset"]), False ) removePhaseWraps = parset.getBool( '.'.join(["LoSoTo.Steps", step, "RemovePhaseWraps"]), True) fit3rdorder = parset.getBool( '.'.join(["LoSoTo.Steps", step, "Fit3rdOrder"]), False) circular = parset.getBool('.'.join(["LoSoTo.Steps", step, "Circular"]), False) reverse = parset.getBool('.'.join(["LoSoTo.Steps", step, "Reverse"]), False) # do something on every soltab (use the openSoltab LoSoTo function) #for soltab in openSoltabs( H, soltabs ): for soltabname in soltabs: solsetname = soltabname.split('/')[0] soltab = H.getSoltab(solset=solsetname, soltab=soltabname.split('/')[1]) logging.info("--> Working on soltab: " + soltab._v_name) t = solFetcher(soltab) tw = solWriter(soltab) # some checks solType = t.getType() if solType != 'phase': logging.warning("Soltab type of " + soltab._v_name + " is: " + solType + " should be phase. Ignoring.") continue # this will make a selection for the getValues() and getValuesIter() userSel = {} for axis in t.getAxesNames(): userSel[axis] = getParAxis(step, parset, H, axis) t.setSelection(**userSel) # Collect station properties station_dict = H.getAnt(solsetname) stations = t.getAxisValues('ant') station_positions = np.zeros((len(stations), 3), dtype=np.float) for i, station_name in enumerate(stations): station_positions[i, 0] = station_dict[station_name][0] station_positions[i, 1] = station_dict[station_name][1] station_positions[i, 2] = station_dict[station_name][2] returnAxes = ['ant', 'freq', 'pol', 'time'] for vals, flags, coord, selection in t.getValuesIter( returnAxes=returnAxes, weight=True): if len(coord['ant']) < 2: logging.error( 'Clock/TEC separation needs at least 2 antennas selected.') return 1 if len(coord['freq']) < 10: logging.error( 'Clock/TEC separation needs at least 10 frequency channels, preferably distributed over a wide range' ) return 1 freqs = coord['freq'] stations = coord['ant'] times = coord['time'] # get axes index axes = [i for i in t.getAxesNames() if i in returnAxes] # reverse time axes if reverse: vals = np.swapaxes( np.swapaxes(vals, 0, axes.index('time'))[::-1], 0, axes.index('time')) flags = np.swapaxes( np.swapaxes(flags, 0, axes.index('time'))[::-1], 0, axes.index('time')) result=doFit(vals,flags==0,freqs,stations,station_positions,axes,\ flagBadChannels=flagBadChannels,flagcut=flagCut,chi2cut=chi2cut,combine_pol=combinePol,removePhaseWraps=removePhaseWraps,fit3rdorder=fit3rdorder,circular=circular) if fit3rdorder: clock, tec, offset, tec3rd = result if reverse: clock = clock[::-1, :] tec = tec[::-1, :] tec3rd = tec3rd[::-1, :] else: clock, tec, offset = result if reverse: clock = clock[::-1, :] tec = tec[::-1, :] weights = tec > -5 tec[np.logical_not(weights)] = 0 clock[np.logical_not(weights)] = 0 weights = np.float16(weights) if combinePol: tf_st = H.makeSoltab(solsetname, 'tec', axesNames=['time', 'ant'], axesVals=[times, stations], vals=tec[:, :, 0], weights=weights[:, :, 0]) sw = solWriter(tf_st) sw.addHistory('CREATE (by CLOCKTECFIT operation)') tf_st = H.makeSoltab(solsetname, 'clock', axesNames=['time', 'ant'], axesVals=[times, stations], vals=clock[:, :, 0] * 1e-9, weights=weights[:, :, 0]) sw = solWriter(tf_st) sw.addHistory('CREATE (by CLOCKTECFIT operation)') tf_st = H.makeSoltab(solsetname, 'phase_offset', axesNames=['ant'], axesVals=[stations], vals=offset[:, 0], weights=np.ones_like(offset[:, 0], dtype=np.float16)) sw = solWriter(tf_st) sw.addHistory('CREATE (by CLOCKTECFIT operation)') if fit3rdorder: tf_st = H.makeSoltab(solsetname, 'tec3rd', axesNames=['time', 'ant'], axesVals=[times, stations], vals=tec3rd[:, :, 0], weights=weights[:, :, 0]) sw = solWriter(tf_st) else: tf_st = H.makeSoltab(solsetname, 'tec', axesNames=['time', 'ant', 'pol'], axesVals=[times, stations, ['XX', 'YY']], vals=tec, weights=weights) sw = solWriter(tf_st) sw.addHistory('CREATE (by CLOCKTECFIT operation)') tf_st = H.makeSoltab(solsetname, 'clock', axesNames=['time', 'ant', 'pol'], axesVals=[times, stations, ['XX', 'YY']], vals=clock * 1e-9, weights=weights) sw = solWriter(tf_st) sw.addHistory('CREATE (by CLOCKTECFIT operation)') tf_st = H.makeSoltab(solsetname, 'phase_offset', axesNames=['ant', 'pol'], axesVals=[stations, ['XX', 'YY']], vals=offset, weights=np.ones_like(offset, dtype=np.float16)) sw = solWriter(tf_st) sw.addHistory('CREATE (by CLOCKTECFIT operation)') if fit3rdorder: tf_st = H.makeSoltab( solsetname, 'tec3rd', axesNames=['time', 'ant', 'pol'], axesVals=[times, stations, ['XX', 'YY']], vals=tec3rd, weights=weights) sw = solWriter(tf_st) return 0
elapsed = (time.clock() - start) logging.info("PARMDB -- "+str(elapsed)+" s.") start = time.clock() for i in xrange(n): H.setSelection(dir='pointing', ant='RS*') Hrot = H.getValues(retAxesVals = False) elapsed = (time.clock() - start) logging.info("H5parm -- "+str(elapsed)+" s.") #print "Equal?", (Prot == np.squeeze(Hrot)).all() ###################################################### # read+write logging.info("### Read all rotations for a dir/station and write them back") Hw = solWriter(H5.getSoltab(solset,'rotation000')) start = time.clock() for i in xrange(n): Prot = P.getValuesGrid('CommonRotationAngle:CS001LBA') Prot = {'test'+str(i):Prot['CommonRotationAngle:CS001LBA']} P2.addValues(Prot) # parmdb write? elapsed = (time.clock() - start) logging.info("PARMDB -- "+str(elapsed)+" s.") start = time.clock() for i in xrange(n): H.setSelection(dir='pointing', ant='CS001LBA') Hrot = H.getValues(retAxesVals = False) Hw.setSelection(dir='pointing', ant='CS001LBA')
logging.info("Create soltab (using default name)") H5.makeSoltab(ss, 'amplitude', axesNames=['axis1', 'axis2', 'axis3'], axesVals=axesVals, vals=vals, weights=vals) logging.info('Get a soltab object') st = H5.getSoltab(ss, 'stTest') logging.info('Get all soltabs:') print H5.getSoltabs(ss) print "###########################################" logging.info('### solFetcher/solWriter - General') Hsf = solFetcher(st) Hsw = solWriter(st) logging.info('Get solution Type (exp: amplitude)') print Hsf.getType() logging.info('Get Axes Names') print Hsf.getAxesNames() logging.info('Get Axis1 Len (exp: 4)') print Hsf.getAxisLen('axis1') logging.info('Get Axis1 Type (exp: str)') print Hsf.getAxisType('axis1') logging.info('Get Axis2 Type (exp: float)') print Hsf.getAxisType('axis2') logging.info('Get Axis1 Values (exp: a,b,c,d)') print Hsf.getAxisValues('axis1') logging.info('Set new axes values') Hsw.setAxisValues('axis1', ['e', 'f', 'g', 'h']) logging.info('Get new Axis1 Values (exp: e,f,g,h)')