def _flag(weights, coord, axesToExt, selection, percent=50, size=[0], maxCycles=3, outQueue=None): """ Flag data if surreounded by other flagged data weights = the weights to convert into flags percent = percent of surrounding flagged point to extend the flag return: flags array and final rms """ def extendFlag(flags, percent): #flags = flags.astype(np.int) if float(np.sum(flags)) / len(flags) > percent / 100.: return 1 else: return 0 def percentFlagged(w): return 100. * (weights.size - np.count_nonzero(weights)) / float( weights.size) import scipy.ndimage import numpy as np initPercent = percentFlagged(weights) # if size=0 then extend to all 2*axis, this otherwise create issues with mirroring for i, s in enumerate(size): if s == 0: size[i] = 2 * weights.shape[i] for cycle in range(maxCycles): flag = scipy.ndimage.filters.generic_filter( (weights == 0), extendFlag, size=size, mode='mirror', cval=0.0, origin=0, extra_keywords={'percent': percent}) weights[(flag == 1)] = 0 # no new flags if cycle != 0 and np.count_nonzero(flag) == oldFlagCount: break oldFlagCount = np.count_nonzero(flag) if percentFlagged(weights) == initPercent: logging.debug('Percentage of data flagged (%s): %.3f -> None' \ % (removeKeys(coord, axesToExt), initPercent)) else: logging.debug('Percentage of data flagged (%s): %.3f -> %.3f %%' \ % (removeKeys(coord, axesToExt), initPercent, percentFlagged(weights))) outQueue.put([weights, selection])
def correctWrapsFromResiduals(residualarray,flags,freq): '''corrects solution jumps due to 2 pi phasewraps based on the average residuals''' resflags = np.logical_or(flags[:, np.newaxis], residualarray == 0) maskedresiduals = np.ma.array(residualarray, mask=resflags) # avgResiduals=np.average(residualarray,axis=0) avgResiduals = np.ma.average(maskedresiduals, axis=0) (wraps, steps) = getResidualPhaseWraps(avgResiduals, freq) # fitting of the wraps from the time-avg residuals # step[0] is the step in TEC corresponding to 1 phase wrap and step[1] is in ns (clock) wraps = np.round(wraps - wraps[0]) # reference to station 0 logging.debug('Wraps from residuals: ' + str(wraps)) return (wraps, steps)
def run(soltab, axesToNorm, normVal=1., log=False): """ Normalize the solutions to a given value WEIGHT: Weights compliant Parameters ---------- axesToNorm : array of str Axes along which compute the normalization. normVal : float, optional Number to normalize to vals = vals * (normVal/valsMean), by default 1. log : bool, optional clip is done in log10 space, by default False. """ import numpy as np logging.info("Normalizing soltab: " + soltab.name) # input check axesNames = soltab.getAxesNames() for normAxis in axesToNorm: if normAxis not in axesNames: logging.error('Normalization axis ' + normAxis + ' not found.') return 1 if soltab.getType() == 'amplitude' and not log: logging.warning( 'Amplitude solution tab detected and log=False. Amplitude solution tables should be treated in log space.' ) for vals, weights, coord, selection in soltab.getValuesIter( returnAxes=axesToNorm, weight=True): if log: vals = np.log10(vals) # rescale solutions if np.all(weights == 0): continue # skip flagged selections valsMean = np.nanmean(vals[weights != 0]) if log: vals[weights != 0] += np.log10(normVal) - valsMean else: vals[weights != 0] *= normVal / valsMean logging.debug("Rescaling by: " + str(normVal / valsMean)) # writing back the solutions if log: vals = 10**vals soltab.setValues(vals, selection) soltab.flush() soltab.addHistory('NORM (on axis %s)' % (axesToNorm)) return 0
def run( soltab, ms, inverse=False, useElementResponse=True, useArrayFactor=True, useChanFreq=True ): """ Generic unspecified step for easy expansion. Parameters ---------- opt1 : float Is a mandatory parameter. """ # load specific libs import numpy as np import casacore.tables as pt from lofar.stationresponse import stationresponse sr = stationresponse(ms, inverse, useElementResponse, useArrayFactor, useChanFreq) numants = pt.taql('select gcount(*) as numants from '+ms+'::ANTENNA').getcol('numants')[0] times = soltab.getAxisValues('time') for vals, coord, selection in soltab.getValuesIter(returnAxes=['ant','time','pol','freq'], weight=False): vals = reorderAxes( vals, soltab.getAxesNames(), ['ant','time','freq','pol']) for stationnum in range(numants): logging.debug('Working on station number %i' % stationnum) for itime, time in enumerate(times): beam = sr.evaluateStation(time=time, station=stationnum) # Reshape from [nfreq, 2, 2] to [nfreq, 4] beam = beam.reshape(beam.shape[0], 4) if soltab.getAxisLen('pol') == 2: beam = beam[:,[0,3]] # get only XX and YY if soltab.getType() == 'amplitude': vals[stationnum, itime, :, :] = np.abs(beam) elif soltab.getType() == 'phase': vals[stationnum, itime, :, :] = np.angle(beam) else: logging.error('Beam prediction work only for amplitude/phase solution tables.') return 1 vals = reorderAxes( vals, ['ant','time','freq','pol'], [ax for ax in soltab.getAxesNames() if ax in ['ant','time','freq','pol']] ) soltab.setValues(vals, selection) return 0
def getResidualPhaseWraps(avgResiduals, freqs): flags = np.average(avgResiduals.mask,axis=1)>0.5 nSt = avgResiduals.shape[1] nF = freqs.shape[0] wraps = np.zeros((nSt, ), dtype=np.float) tmpflags = flags tmpfreqs = freqs[np.logical_not(tmpflags)] steps=[0,0] if np.ma.count(tmpfreqs) < 3: logging.debug('Cannot unwrap, too many channels flagged') return (wraps,steps) (tmpbasef, steps) = getPhaseWrapBase(tmpfreqs) basef = np.zeros(freqs.shape) basef[np.logical_not(tmpflags)] = tmpbasef basef = basef.reshape((-1, 1)) data = avgResiduals[:, :] if has_fitting: wraps = fitting.fit(data, basef, wraps, flags).flatten() return (wraps, steps)
def __init__(self, procs=0, funct=None): """ Manager for multiprocessing procs: number of processors, if 0 use all available funct: function to parallelize / note that the last parameter of this function must be the outQueue and it will be linked to the output queue """ if procs == 0: procs = multiprocessing.cpu_count() self.procs = procs self._threads = [] self.inQueue = multiprocessing.JoinableQueue() self.outQueue = multiprocessing.Queue() self.runs = 0 logging.debug('Spawning %i threads...' % self.procs) for proc in range(self.procs): t = self.multiThread(self.inQueue, self.outQueue, funct) self._threads.append(t) t.start()
def run(soltab, axesToNorm, normVal=1.): """ Normalize the solutions to a given value WEIGHT: Weights compliant Parameters ---------- axesToNorm : array of str Axes along which compute the normalization. normVal : float, optional Number to normalize to vals = vals * (normVal/valsMean), by default 1. """ import numpy as np logging.info("Normalizing soltab: " + soltab.name) # input check axesNames = soltab.getAxesNames() for normAxis in axesToNorm: if normAxis not in axesNames: logging.error('Normalization axis ' + normAxis + ' not found.') return 1 for vals, weights, coord, selection in soltab.getValuesIter( returnAxes=axesToNorm, weight=True): # rescale solutions if np.all(weights == 0): continue # skip flagged selections valsMean = np.nanmean(vals[weights != 0]) vals[weights != 0] *= normVal / valsMean logging.debug(str(coord)) logging.debug("Rescaling by: " + str(normVal / valsMean)) # writing back the solutions soltab.setValues(vals, selection) soltab.flush() soltab.addHistory('NORM (on axis %s)' % (axesToNorm)) return 0
def _run_timestep(t,coord_rr,coord_ll,weights,vals,solType,coord,maxResidual): c = 2.99792458e8 if solType == 'phase': idx = ((weights[coord_rr,:] != 0.) & (weights[coord_ll,:] != 0.)) freq = np.copy(coord['freq'])[idx] phase_rr = vals[coord_rr,:][idx] phase_ll = vals[coord_ll,:][idx] # RR-LL to be consistent with BBS/NDPPP phase_diff = (phase_rr - phase_ll) # not divide by 2 otherwise jump problem, then later fix this else: # rotation table idx = ((weights[:] != 0.) & (weights[:] != 0.)) freq = np.copy(coord['freq'])[idx] phase_diff = 2.*vals[:][idx] # a rotation is between -pi and +pi if len(freq) < 20: fitresultrm_wav = [0] weight = 0 logging.warning('No valid data found for Faraday fitting for antenna: '+coord['ant']+' at timestamp '+str(t)) else: # if more than 1/4 of chans are flagged if (len(idx) - len(freq))/float(len(idx)) > 1/4.: logging.debug('High number of filtered out data points for the timeslot %i: %i/%i' % (t, len(idx) - len(freq), len(idx)) ) wav = c/freq #fitresultrm_wav, success = scipy.optimize.leastsq(rmwavcomplex, [fitrmguess], args=(wav, phase_diff)) ranges = slice(-0.1, 0.1, 2e-4) fitresultrm_wav = scipy.optimize.brute(costfunctionRM, (ranges,), finish=scipy.optimize.leastsq, args=(wav, phase_diff)) # fractional residual residual = np.nanmean(np.abs(np.mod((2.*fitresultrm_wav*wav*wav)-phase_diff + np.pi, 2.*np.pi) - np.pi)) if maxResidual == 0 or residual < maxResidual: fitrmguess = fitresultrm_wav[0] # Don't know what this is for... weight = 1 else: # high residual, flag logging.warning('Bad solution for ant: '+coord['ant']+' (time: '+str(t)+', resdiaul: '+str(residual)+').') weight = 0 return fitresultrm_wav[0],weight
def correctWraps( tecarray, residualarray, freq, pos, ): '''corrects solution jumps due to 2 pi phasewraps based on spatial correlations of averaged TEC solutions. Also returns average constant phase offset per station. International stations should be excluded from this''' nT = tecarray.shape[0] nSt = tecarray.shape[1] flags = tecarray < -5 (wraps, steps) = correctWrapsFromResiduals(residualarray, flags, freq) lats = np.degrees( np.arctan2(pos[:, 2], np.sqrt(pos[:, 0] * pos[:, 0] + pos[:, 1] * pos[:, 1]))) lats -= lats[0] lons = np.degrees(np.arctan2(pos[:, 1], pos[:, 0])) lons -= lons[0] lonlat = np.concatenate((lons, lats)).reshape((2, ) + lons.shape) # refine (is it needed/helpful for LBA? the offsets seem to make sense for LBA) myselect = np.sqrt(np.sum(lonlat**2, axis=0)) > 0.5 for nr_iter in range(2): # recreate best TEC at the moment TEC = tecarray - tecarray[:, [0]] + steps[0] * (np.round(wraps) - np.round(wraps[0])) #TEC = np.ma.array(TEC, mask=flags) TEC = np.ma.array(TEC, mask=np.logical_or(flags, myselect[np.newaxis])) # fit 2d linear TEC screen over stations slope = np.ma.dot(np.linalg.inv(np.dot(lonlat, lonlat.T)), np.ma.dot(lonlat, TEC.T)) # flag bad time steps maybe because TEC screen is not linear chi2 = np.ma.sum(np.square(TEC - np.ma.dot(lonlat.T, slope).T), axis=1) / nSt chi2select = chi2 < np.ma.average(chi2) chi2select = chi2 < np.ma.average(chi2[chi2select]) # calculate offset per station wrt time-averaged TEC screen offsets = -1 * np.ma.average( TEC[chi2select] - np.ma.dot(slope.T, lonlat)[chi2select], axis=0) * 2. * np.pi / steps[0] remainingwraps = np.round( offsets / (2 * np.pi)) # -np.round(wraps[stationIndices]) offsets[myselect] = 0 remainingwraps[myselect] = 0 logging.debug('Offsets: ' + str(offsets)) logging.debug('AvgTEC: ' + str(np.ma.average(TEC[chi2select], axis=0))) logging.debug('Remaining: ' + str(remainingwraps)) wraps += remainingwraps # TODO: remove also the offset before the second cycle if np.sum(np.absolute(remainingwraps)) == 0: break return (offsets, wraps, steps)
#!/usr/bin/env python # -*- coding: utf-8 -*- from losoto.lib_operations import * from losoto._logging import logger as logging logging.debug('Loading DELETEAXIS module.') def _run_parser(soltab, parser, step): axisDelete = parser.getstr( step, 'axisDelete' ) # no default fromCell = parser.getstr( step, 'fromCell', '' ) parser.checkSpelling( step, soltab, ['axisDelete', 'fromCell']) return run(soltab, axisDelete, fromCell) def run( soltab, axisDelete, fromCell): """ Delete an axis only keeping the values of a certain slice. Parameters ---------- axisDelete : str Axis to delete. fromCell : str A cell value in axisDelete from which to keep the data values. If it is the string "first"/"last" then uses the first/last element of the axis. """ import numpy as np if not axisDelete in soltab.getAxesNames():
def run(soltab, doUnwrap=False, refAnt='', plotName='', ndiv=1): """ Find the structure function from phase solutions of core stations. Parameters ---------- doUnwrap : bool, optional refAnt : str, optional Reference antenna, by default the first. plotName : str, optional Plot file name, by default no plot. ndiv : int, optional """ import numpy as np from losoto.lib_unwrap import unwrap, unwrap_2d logging.info("Find structure function for soltab: " + soltab.name) # input check solType = soltab.getType() if solType != 'phase': logging.warning("Soltab type of " + soltab._v_name + " is of type " + solType + ", should be phase.") return 1 ants = soltab.getAxisValues('ant') if refAnt != '' and refAnt != 'closest' and not refAnt in soltab.getAxisValues( 'ant', ignoreSelection=True): logging.error('Reference antenna ' + refAnt + ' not found. Using: ' + ants[1]) refAnt = ants[1] if refAnt == '' and doUnwrap: logging.error('Unwrap requires reference antenna. Using: ' + ants[1]) refAnt = ants[1] if refAnt == '': refAnt = None soltab.setSelection(ant='CS*', update=True) posAll = soltab.getSolset().getAnt() for vals, weights, coord, selection in soltab.getValuesIter( returnAxes=['freq', 'pol', 'ant', 'time'], weight=True, reference=refAnt): # reorder axes vals = reorderAxes(vals, soltab.getAxesNames(), ['pol', 'ant', 'freq', 'time']) weights = reorderAxes(weights, soltab.getAxesNames(), ['pol', 'ant', 'freq', 'time']) # order positions pos = np.array([list(posAll[ant]) for ant in coord['ant']]) # avg pols vals = np.cos(vals) + 1.j * np.sin(vals) vals = np.nansum(vals, axis=0) vals = np.angle(vals) flags = np.array((weights[0] == 0) | (weights[1] == 0), dtype=bool) # unwrap if doUnwrap: # remove mean to facilitate unwrapping for a, ant in enumerate(coord['ant']): if not (flags[a, :, :] == True).all() and ant != refAnt: logging.debug('Unwrapping: ' + ant) mean = np.angle(np.nanmean(np.exp(1j * vals[a].flatten()))) vals[a] -= mean vals[a] = np.mod(vals[a] + np.pi, 2 * np.pi) - np.pi vals[a, :, :] = unwrap_2d(vals[a, :, :], flags[a, :, :], coord['freq'], coord['time']) logging.debug('Computing differential values...') t1 = np.ma.array(vals, mask=flags) # mask flagged data dph = t1[np.newaxis] - t1[:, np.newaxis] # ant x ant x freq x time D = pos[np.newaxis] - pos[:, np.newaxis] # ant x ant x 3 D2 = np.triu( np.sqrt(np.sum(D**2, axis=-1)) ) # calc distance and keep only uppoer triangle larger than 0 myselect = (D2 > 0) if not doUnwrap: logging.debug('Re-normalising...') dph = np.mod(dph + np.pi, 2 * np.pi) - np.pi avgdph = np.ma.average( dph, axis=2) # avg in freq (can do because is between -pi and pi) #one extra step to remove most(all) phase wraps, phase wraps disturbe the averaging... dph = np.remainder( dph - np.ma.average(avgdph, axis=-1)[:, :, np.newaxis, np.newaxis] + np.pi, 2 * np.pi) + np.ma.average( avgdph, axis=-1)[:, :, np.newaxis, np.newaxis] - np.pi #center around the avg value logging.debug('Computing sructure function...') avgdph = np.ma.average(dph, axis=2) # avg in freq to reduce noise variances = [] pars = [] avgdph = avgdph[ ..., avgdph.shape[-1] % ndiv:] # remove a few timeslots to make the array divisible by np.split for i, avgdphSplit in enumerate(np.split(avgdph, ndiv, axis=-1)): variance = np.ma.var(avgdphSplit, axis=-1) * ( np.average(coord['freq']) / 150.e6)**2 # get time variance and rescale to 150 MHz # linear regression #A = np.ones((2,D2[myselect].shape[0]),dtype=float) #A[1,:] = np.log10(D2[myselect][~variance.mask]) #par = np.dot(np.linalg.inv(np.dot(A,A.T)),np.dot(A,np.log10(variance[myselect]))) mask = variance[myselect].mask A = np.vstack([ np.log10(D2[myselect][~mask]), np.ones(len(D2[myselect][~mask])) ]) par = np.linalg.lstsq(A.T, np.log10(variance[myselect][~mask]))[0] S0 = 10**(-1 * par[1] / par[0]) logging.info(r't%i: beta=%.2f - R_diff=%.2f km' % (i, par[0], S0 / 1.e3)) variances.append(variance) pars.append(par) if plotName != '': if plotName.split('.')[-1] != 'png': plotName += '.png' # add png if not 'matplotlib' in sys.modules: import matplotlib as mpl mpl.use("Agg") import matplotlib.pyplot as plt fig = plt.figure() fig.subplots_adjust(wspace=0) ax = fig.add_subplot(111) ax1 = ax.twinx() for i, variance in enumerate(variances): if len(variances) > 1: color = plt.cm.jet( i / float(len(variances) - 1)) # from 0 to 1 else: color = 'black' ax.plot(D2[myselect] / 1.e3, variance[myselect], marker='o', linestyle='', color=color, markeredgecolor='none', label='T') # regression par = pars[i] x = D2[myselect] S0 = 10**(-1 * par[1] / par[0]) if color == 'black': color = 'red' # in case of single color, use red line that is more visible ax1.plot(x.flatten() / 1.e3, par[0] * np.log10(x.flatten()) + par[1], linestyle='-', color=color, label=r'$\beta=%.2f$ - $R_{\rm diff}=%.2f$ km' % (par[0], S0 / 1.e3)) ax.set_xlabel('Distance (km)') ax.set_ylabel(r'Phase variance @150 MHz (rad$^2$)') ax.set_xscale('log') ax.set_yscale('log') ymin = np.min(variance[myselect]) ymax = np.max(variance[myselect]) ax.set_xlim(xmin=0.1, xmax=3) ax.set_ylim(ymin, ymax) ax1.set_ylim(np.log10(ymin), np.log10(ymax)) ax1.legend(loc='lower right', frameon=False) ax1.set_yticks([]) logging.warning('Save pic: %s' % plotName) plt.savefig(plotName, bbox_inches='tight') return 0
#!/usr/bin/env python # -*- coding: utf-8 -*- from losoto.lib_operations import * from losoto._logging import logger as logging logging.debug('Loading STRUCTURE module.') def _run_parser(soltab, parser, step): doUnwrap = parser.getbool(step, 'doUnwrap', False) refAnt = parser.getstr(step, 'refAnt', '') plotName = parser.getstr(step, 'plotName', '') ndiv = parser.getint(step, 'ndiv', 1) parser.checkSpelling(step, soltab, ['doUnwrap', 'refAnt', 'plotName', 'ndiv']) return run(soltab, doUnwrap, refAnt, plotName, ndiv) def run(soltab, doUnwrap=False, refAnt='', plotName='', ndiv=1): """ Find the structure function from phase solutions of core stations. Parameters ---------- doUnwrap : bool, optional refAnt : str, optional Reference antenna, by default the first.
#!/usr/bin/env python # -*- coding: utf-8 -*- from losoto.lib_operations import * from losoto._logging import logger as logging logging.debug('Loading REFERENCE module.') def _run_parser(soltab, parser, step): refAnt = parser.getstr( step, 'refAnt', '' ) refDir = parser.getstr( step, 'refDir', '' ) parser.checkSpelling( step, soltab, ['refAnt','refDir']) return run(soltab, refAnt, refDir) def run( soltab, refAnt='', refDir=''): """ Reference to an antenna Parameters ---------- refAnt : str, optional Reference antenna for phases or "best" to use the least flagged antenna. Empty string does not change phases. Default: ''. refDir : str, optional Reference direction for phases. Empty string does not change phases. Default: ''. """ if not soltab.getType() in ['phase', 'scalarphase', 'rotation', 'tec', 'clock', 'tec3rd', 'rotationmeasure']: logging.error('Reference possible only for phase, scalarphase, clock, tec, tec3rd, rotation and rotationmeasure solution tables. Ignore referencing.') return 1
#!/usr/bin/env python # -*- coding: utf-8 -*- from losoto.lib_operations import * from losoto._logging import logger as logging logging.debug('Loading SPLITLEAK module.') def _run_parser(soltab, parser, step): soltabOutG = parser.getstr(step, 'soltabOutG') # no default soltabOutD = parser.getstr(step, 'soltabOutD') # no default parser.checkSpelling(step, soltab, ['soltabOutG', 'soltabOutD']) return run(soltab, soltabOutG, soltabOutD) def run(soltab, soltabOutG=None, soltabOutD=None): """ Duplicate a table Parameters ---------- soltabOutG : str, optional Output table name (diagonal component). By default choose next available from table type. soltabOutD : str, optional Output table name (leakage component). By default choose next available from table type. """ logging.info('Split leakage tables %s -> %s + %s' %
#!/usr/bin/env python # -*- coding: utf-8 -*- from losoto.lib_operations import * from losoto._logging import logger as logging logging.debug('Loading NORM module.') def _run_parser(soltab, parser, step): axesToNorm = parser.getarraystr(step, 'axesToNorm') # no default normVal = parser.getfloat(step, 'normVal', 1.) log = parser.getbool(step, 'log', False) parser.checkSpelling(step, soltab, ['axesToNorm', 'normVal', 'log']) return run(soltab, axesToNorm, normVal, log) def run(soltab, axesToNorm, normVal=1., log=False): """ Normalize the solutions to a given value WEIGHT: Weights compliant Parameters ---------- axesToNorm : array of str Axes along which compute the normalization. normVal : float, optional Number to normalize to vals = vals * (normVal/valsMean), by default 1.
#import warnings #warnings.simplefilter('error', UserWarning) from losoto.lib_operations import * from losoto._logging import logger as logging # avoids error if re-setting "agg" a second run of plot if not 'matplotlib' in sys.modules: import matplotlib as mpl mpl.rcParams['xtick.labelsize'] = 20 mpl.rcParams['font.size'] = 20 mpl.use("Agg") import matplotlib.pyplot as plt # after setting "Agg" to speed up logging.debug('Loading PLOT module.') def _run_parser(soltab, parser, step): axesInPlot = parser.getarraystr(step, 'axesInPlot') # no default axisInTable = parser.getstr(step, 'axisInTable', '') axisInCol = parser.getstr(step, 'axisInCol', '') axisDiff = parser.getstr(step, 'axisDiff', '') NColFig = parser.getint(step, 'NColFig', 0) figSize = parser.getarrayint(step, 'figSize', [0, 0]) markerSize = parser.getint(step, 'markerSize', 2) minmax = parser.getarrayfloat(step, 'minmax', [0., 0.]) log = parser.getstr(step, 'log', '') plotFlag = parser.getbool(step, 'plotFlag', False) doUnwrap = parser.getbool(step, 'doUnwrap', False) refAnt = parser.getstr(step, 'refAnt', '')
#!/usr/bin/env python # -*- coding: utf-8 -*- from losoto.lib_operations import * from losoto._logging import logger as logging logging.debug('Loading TEC module.') def _run_parser(soltab, parser, step): soltabOut = parser.getstr(step, 'soltabOut', 'tec000') refAnt = parser.getstr(step, 'refAnt', '') maxResidualFlag = parser.getfloat(step, 'maxResidualFlag', 2.5) maxResidualProp = parser.getfloat(step, 'maxResidualProp', 1.) parser.checkSpelling( step, soltab, ['soltabOut', 'refAnt', 'maxResidualFlag', 'maxResidualProp']) return run(soltab, soltabOut, refAnt, maxResidualFlag, maxResidualProp) def run(soltab, soltabOut='tec000', refAnt='', maxResidualFlag=2.5, maxResidualProp=1.): """ Bruteforce TEC extraction from phase solutions. Parameters ----------
#!/usr/bin/env python # -*- coding: utf-8 -*- # This is the station-screen operation for LoSoTo from losoto.lib_operations import * from losoto._logging import logger as logging logging.debug('Loading STATIONSCREEN module.') def _run_parser(soltab, parser, step): outSoltab = parser.getstr(step, "outSoltab") order = parser.getint(step, "Order", 5) beta = parser.getfloat(step, "Beta", 5.0 / 3.0) niter = parser.getint(step, "niter", 2) nsigma = parser.getfloat(step, "nsigma", 5.0) refAnt = parser.getint(step, "RefAnt", -1) scale_order = parser.getbool(step, "ScaleOrder", True) scale_dist = parser.getfloat(step, "scaleDist", 25000.0) min_order = parser.getint(step, "MinOrder", 5) adjust_order = parser.getbool(step, "AdjustOrder", True) parser.checkSpelling( step, soltab, ['outSoltab', 'order', 'beta', 'niter', 'nsigma',\ 'refAnt', 'scale_order', 'scale_dist', 'min_order', 'adjust_order']) return run(soltab, outSoltab, order, beta, niter, nsigma, refAnt, scale_order, scale_dist, min_order, adjust_order) def _calculate_piercepoints(station_positions, source_positions): """
#!/usr/bin/env python # -*- coding: utf-8 -*- from losoto.lib_operations import * from losoto._logging import logger as logging logging.debug('Loading GLOBALDELAY module.') def _run_parser(soltab, parser, step): soltabOut = parser.getstr(step, 'soltabOut', 'delay000') refAnt = parser.getstr(step, 'refAnt', '') parser.checkSpelling(step, soltab, ['soltabOut', 'refAnt']) return run(soltab, soltabOut, refAnt) def run(soltab, soltabOut='tec000', refAnt=''): """ Bruteforce TEC extraction from phase solutions. Parameters ---------- soltabOut : str, optional output table name (same solset), by deault "tec". refAnt : str, optional Reference antenna, by default the first. """ import numpy as np
def getClockTECFitStation( ph, freq, stationname, initSol=[], returnResiduals=True, chi2cut=1e8, fit3rdorder=False, double_search_space=False ): '''get the c/t separation per station''' nT = ph.shape[0] nF = freq.shape[0] data = ph tecarrayst = np.zeros((nT,), dtype=np.float32) clockarrayst = np.zeros((nT,), dtype=np.float32) if returnResiduals: residualarrayst = np.zeros((nT, nF), dtype=np.float32) if fit3rdorder: tec3rdarrayst= np.zeros((nT,), dtype=np.float32) A = np.ones((nF, 2+fit3rdorder), dtype=np.float) A[:, 1] = freq * 2 * np.pi * 1e-9 A[:, 0] = -8.44797245e9 / freq if fit3rdorder: A[:, 2] = -1e21 / freq**3 steps = np.ma.dot(np.ma.dot(np.linalg.inv(np.ma.dot(A.T, A)), A.T), 2 * np.pi * np.ones((freq.shape[0], ), dtype=np.float)) succes=False initprevsol=False nrFail=0 sol = np.zeros((2+fit3rdorder), dtype=np.float) prevsol = np.zeros_like(sol) n3rd=0 for itm in range(nT): datatmp=np.ma.copy(data[itm, :]) if itm == 0 or not succes: if itm == 0 or not initprevsol: if hasattr(initSol, '__len__') and len(initSol) > 0 : sol[:initSol.shape[0]]=initSol ndt=1 ndtec=1 if fit3rdorder: n3rd=1 else: sol[:]=0 if fit3rdorder: n3rd=200 if 'CS' in stationname: ndt=4 ndtec=10 if 'LBA' in stationname: ndt=2 ndtec=40 else: if 'RS' in stationname: ndt=200 ndtec=80 else: # large TEC variation for EU stations ndt=200 ndtec=160 if 'LBA' in stationname: ndt=60 # no init clock possible due to large TEC effect #stepsize of dtec is small ndtec=320 else: # further steps with non success sol[:] = prevsol[:] ndtec=min(nrFail+1,100) if not 'CS' in stationname: ndt=min(nrFail+1,200) else: ndt=min(nrFail+1,4) if fit3rdorder: n3rd=min(nrFail+1,200) datatmpist = datatmp[:] if datatmpist.count() / float(nF) > 0.5: # do brutforce and update data, unwrp pdata,update flags #logging.debug("Getting init par for time %d:station %s ntec %d ndt %d n3rd %d"%(itm,stationname,ndtec,ndt,n3rd)+str(sol)) par,datatmp[:] = getInitPar(datatmpist, freq,nrTEC=ndtec*(1+double_search_space),nrClock=ndt*(1+double_search_space),nrthird=n3rd*(1+double_search_space),initsol=sol[:]) sol[:] = par[:] #logging.debug("Getting init par for station %d:%s "%(itm,stationname)+str(sol)) #now do the real fitting datatmpist=datatmp[:] if datatmpist.count() / float(nF) < 0.5: logging.debug("Too many data points flagged 2nd t=%d st=%s flags=%d" % (itm,stationname,datatmpist.count()) + str(sol[:])) sol[:] = [-10.,]*sol.shape[0] else: fitdata=np.dot(sol,A.T) datatmpist=unwrapPhases(datatmpist,freq,fitdata) mymask=datatmpist.mask maskedfreq=np.ma.array(freq,mask=mymask) A2=np.ma.array(A,mask=np.tile(mymask,(A.shape[1],1)).T) if datatmpist.count() / float(nF) < 0.5: logging.debug("Too many data points flagged 3rd t=%d st=%s flags=%d" % (itm,stationname,datatmpist.count()) + str(sol[:])) sol[:] = [-10.,]*sol.shape[0] else: sol[:] = np.ma.dot(np.linalg.inv(np.ma.dot(A2.T, A2)), np.ma.dot(A2.T, datatmpist)).T if initprevsol and np.abs((sol[1]-prevsol[1])/steps[1])>0.5 and (np.abs((sol[1]-prevsol[1])/steps[1])>0.75 or np.abs(np.sum((sol-prevsol)/steps,axis=-1))>0.5*A2.shape[0]): #logging.debug("removing jumps, par for station %d , itm %d"%(ist,itm)+str(sol[ist])+str(prevsol[ist])+str(steps)) sol[:]-=np.round((sol[1]-prevsol[1])/steps[1])*steps # calculate chi2 per station residual = data[itm] - np.dot(A, sol) residual = np.ma.remainder(residual + np.pi, 2 * np.pi) - np.pi chi2 = np.ma.sum(np.square(np.degrees(residual)), axis=0) / nF if returnResiduals: residualarrayst[itm] = residual chi2select = (chi2 > chi2cut) or (sol[0] < -5) # select bad points chi2select = chi2select or initprevsol*np.sum(np.abs((sol-prevsol)/steps),axis=-1)>(0.3*sol.shape[0]*(1+nrFail)) #also discard data where there is a "half" jump for any parameter wrst the previous solution (if previous solution exists). Multiply with number of fails, since after a large number of fails there is no clear match with the previous solution expected anyway... if not initprevsol: prevsol= sol[:] if chi2select: logging.debug('High chi2 of fit, itm: %d ' % (itm) + 'station:' + stationname) succes = False nrFail += 1 else: prevsol = 0.5 * prevsol + 0.5 * np.copy(sol) succes = True initprevsol = True nrFail = 0 tecarrayst[itm] = sol[0] clockarrayst[itm] = sol[1] if fit3rdorder: tec3rdarrayst[itm]=sol[2] if returnResiduals: if fit3rdorder: return (tecarrayst, clockarrayst, residualarrayst,tec3rdarrayst) else: return (tecarrayst, clockarrayst, residualarrayst) if fit3rdorder: return (tecarrayst, clockarrayst,tec3rdarrayst) return (tecarrayst, clockarrayst)
def getClockTECFit( ph, freq, stations, initSol=[], returnResiduals=True, chi2cut=1e8, fit3rdorder=False, double_search_space=False ): nT = ph.shape[0] nF = freq.shape[0] nSt = ph.shape[2] data = ph tecarray = np.zeros((nT, nSt), dtype=np.float32) clockarray = np.zeros((nT, nSt), dtype=np.float32) if returnResiduals: residualarray = np.zeros((nT, nF, nSt), dtype=np.float32) if fit3rdorder: tec3rdarray= np.zeros((nT, nSt), dtype=np.float32) A = np.ones((nF, 2+fit3rdorder), dtype=np.float) A[:, 1] = freq * 2 * np.pi * 1e-9 A[:, 0] = -8.44797245e9 / freq if fit3rdorder: A[:, 2] = -1e21 / freq**3 steps = np.ma.dot(np.ma.dot(np.linalg.inv(np.ma.dot(A.T, A)), A.T), 2 * np.pi * np.ones((freq.shape[0], ), dtype=np.float)) succes=False initprevsol=np.zeros(nSt,dtype=bool) nrFail=np.zeros(nSt,dtype=int) sol = np.zeros((nSt, 2+fit3rdorder), dtype=np.float) prevsol = np.zeros_like(sol) n3rd=0 for itm in range(nT): datatmp=np.ma.copy(data[itm, :]) if itm == 0 or not succes: for ist in range(nSt): if itm == 0 or not initprevsol[ist]: if hasattr(initSol, '__len__') and len(initSol) > ist: sol[ist,:initSol[ist].shape[0]]=initSol[ist] ndt=1 ndtec=1 if fit3rdorder: n3rd=1 else: sol[ist]=0 if fit3rdorder: n3rd=200 if 'CS' in stations[ist]: ndt=4 ndtec=10 if 'LBA' in stations[ist]: ndt=2 ndtec=40 else: if 'RS' in stations[ist]: ndt=200 ndtec=80 else: # large TEC variation for EU stations ndt=200 ndtec=160 if 'LBA' in stations[ist]: ndt=60 # no init clock possible due to large TEC effect #stepsize of dtec is small ndtec=520 else: # further steps with non success sol[ist, :] = prevsol[ist, :] ndtec=min(nrFail[ist]+1,100) if not 'CS' in stations[ist]: ndt=min(nrFail[ist]+1,200) else: ndt=min(nrFail[ist]+1,4) if fit3rdorder: n3rd=min(nrFail[ist]+1,200) datatmpist = datatmp[:, ist] if datatmpist.count() / float(nF) > 0.5: # do brutforce and update data, unwrp pdata,update flags #if ist==23 or ist==25: #logging.debug("Getting init par for time %d:station %d ntec %d ndt %d n3rd %d"%(itm,ist,ndtec,ndt,n3rd)+str(sol[ist])) par,datatmp[:, ist] = getInitPar(datatmpist, freq,nrTEC=ndtec*(1+double_search_space),nrClock=ndt*(1+double_search_space),nrthird=n3rd*(1+double_search_space),initsol=sol[ist,:]) sol[ist, :] = par[:] #if itm%100==0: #logging.debug("Getting init par for station %d:%d "%(itm,ist)+str(sol[ist])) for ist in range(nSt): #now do the real fitting datatmpist=datatmp[:,ist] if datatmpist.count() / float(nF) < 0.5: logging.debug("Too many data points flagged first t=%d st=%d flags=%d" % (itm,ist,datatmpist.count()) + str(sol[ist])) sol[ist] = [-10.,]*sol.shape[1] continue fitdata=np.dot(sol[ist],A.T) datatmpist=unwrapPhases(datatmpist,freq,fitdata) #if itm%100==0: #logging.debug(" init par for station itm %d:%d "%(itm,ist)+str(sol[ist])) mymask=datatmpist.mask maskedfreq=np.ma.array(freq,mask=mymask) A2=np.ma.array(A,mask=np.tile(mymask,(A.shape[1],1)).T) sol[ist] = np.ma.dot(np.linalg.inv(np.ma.dot(A2.T, A2)), np.ma.dot(A2.T, datatmpist)).T if initprevsol[ist] and np.abs((sol[ist,1]-prevsol[ist,1])/steps[1])>0.5 and (np.abs((sol[ist,1]-prevsol[ist,1])/steps[1])>0.75 or np.abs(np.sum((sol[ist]-prevsol[ist])/steps,axis=-1))>0.5*A2.shape[0]): #logging.debug("removing jumps, par for station %d , itm %d"%(ist,itm)+str(sol[ist])+str(prevsol[ist])+str(steps)) sol[ist,:]-=np.round((sol[ist,1]-prevsol[ist,1])/steps[1])*steps #logging.debug("removed jumps, par for station %d "%ist+str(sol[ist])+str(prevsol[ist])) #if itm%100==0: #logging.debug("par for station itm %d:%d "%(itm,ist)+str(sol[ist])) # calculate chi2 per station residual = data[itm] - np.dot(A, sol.T) tmpresid = residual - residual[:, 0][:, np.newaxis] # residuals relative to station 0 residual = np.ma.remainder(tmpresid + np.pi, 2 * np.pi) - np.pi chi2 = np.ma.sum(np.square(np.degrees(residual)), axis=0) / nF if returnResiduals: residualarray[itm] = residual chi2select = np.logical_or(np.array(chi2 > chi2cut), sol[:, 0] < -5) # select bad points chi2select = np.logical_or(chi2select,initprevsol*np.sum(np.abs((sol-prevsol)/steps),axis=-1)>(0.3*sol.shape[1]*(1+nrFail))) #also discard data where there is a "half" jump for any parameter wrst the previous solution (if previous solution exists). Multiply with number of fails, since after a large number of fails there is no clear match with the previous solution expected anyway... if np.any(chi2select): #logging.debug('high chi2 of fit, itm: %d %d ' % (itm, np.sum(chi2select)) + str(sol[chi2select]) + 'prevsol'+str(prevsol[chi2select])+'stations:' + str(np.arange(nSt)[chi2select]) + ' chi2 ' + str(chi2[chi2select])) logging.debug('High chi2 of fit, itm: %d %d ' % (itm, np.sum(chi2select)) + 'stations:' + str(np.arange(nSt)[chi2select])) succes = False nrFail[chi2select] += 1 nrFail[~chi2select] = 0 prevsol[np.logical_and(initprevsol == False,chi2select==False)] = sol[np.logical_and(initprevsol == False,chi2select==False)] # compensate missing prevsol at first rounds prevsol[~chi2select] = 0.5 * prevsol[~chi2select] + 0.5 * sol[~chi2select] # init solution to 0.5 * this solution + 0.5 previous solution initprevsol[~chi2select] = True # once is True it never becomes False else: # prevsol=np.copy(sol) prevsol[initprevsol==False] = sol[initprevsol == False] # compensate missing prevsol at first rounds prevsol = 0.5 * prevsol + 0.5 * np.copy(sol) succes = True initprevsol = np.ones(nSt, dtype=bool) nrFail = np.zeros(sol.shape[0], dtype=int) tecarray[itm] = sol[:, 0] clockarray[itm] = sol[:, 1] if fit3rdorder: tec3rdarray[itm]=sol[:, 2] if returnResiduals: if fit3rdorder: return (tecarray, clockarray, residualarray,tec3rdarray) else: return (tecarray, clockarray, residualarray) if fit3rdorder: return (tecarray, clockarray,tec3rdarray) return (tecarray, clockarray)
#!/usr/bin/python # -*- coding: utf-8 -*- import numpy as np import numpy.ma as ma import sys from multiprocessing import Pool from losoto._logging import logger as logging has_fitting=True try: import casacore.tables # must be loaded before expion - used in other operations as lofarbeam from lofar.expion import baselinefitting as fitting except: # logging.error('No lofar.expion present, clock/tec separation not active.') logging.debug('No lofar.expion present, clock/tec separation maybe not optimal') has_fitting=False light_speed=299792458. # from pylab import * def ClockTEC_3rdorder_func(xarray, par): """clock tec fitting withextra parameter for third order ionospheric effects at lowest frequencies""" delay = np.array([par[1] * 1e-9]).flatten() # in ns, array has dimension 1, even if scalar delayfact = 2 * np.pi * delay[:, np.newaxis] * xarray TEC = np.array([par[0]]).flatten() # dTEC in TECU drefract = -8.4479745e9 * TEC[:, np.newaxis] / xarray TEC3rd = np.array([par[2]]).flatten() # 3rd_order d3rd = light_speed**3* TEC3rd[:, np.newaxis] / xarray**3 return drefract[:, np.newaxis,np.newaxis, :] + delayfact[np.newaxis,:,np.newaxis, :]+d3rd[np.newaxis,np.newaxis,:, :] def ClockTECfunc(xarray, par):
#!/usr/bin/env python # -*- coding: utf-8 -*- # This is an example operation for LoSoTo from losoto.lib_operations import * from losoto._logging import logger as logging logging.debug('Loading CLOCKTEC module.') def _run_parser(soltab, parser, step): clocksoltabOut = parser.getstr(step, 'clocksoltabOut', 'clock000') tecsoltabOut = parser.getstr(step, 'tecsoltabOut', 'tec000') offsetsoltabOut = parser.getstr(step, 'offsetsoltabOut', 'phase_offset000') tec3rdsoltabOut = parser.getstr(step, 'tec3rdsoltabOut', 'tec3rd000') flagBadChannels = parser.getbool(step, 'flagBadChannels', True) flagCut = parser.getfloat(step, 'flagCut', 5.) chi2cut = parser.getfloat(step, 'chi2cut', 3000.) combinePol = parser.getbool(step, 'combinePol', False) removePhaseWraps = parser.getbool(step, 'removePhaseWraps', True) fit3rdorder = parser.getbool(step, 'fit3rdorder', False) circular = parser.getbool(step, 'circular', False) reverse = parser.getbool(step, 'reverse', False) invertOffset = parser.getbool(step, 'invertOffset', False) nproc = parser.getint(step, 'nproc', 10) parser.checkSpelling(step, soltab, [ 'tecsoltabOut', 'clocksoltabOut', 'offsetsoltabOut', 'tec3rdsoltabOut', 'flagBadChannels', 'flagCut', 'chi2cut', 'combinePol', 'removePhaseWraps', 'fit3rdorder', 'circular', 'reverse',
def run( soltab, soltabOut='phasediff', maxResidual=1., fitOffset=False, average=False, replace=False, minFreq=0, refAnt='' ): """ Estimate polarization misalignment as delay. Parameters ---------- soltabOut : str, optional output table name (same solset), by deault "phasediff". maxResidual : float, optional Maximum acceptable rms of the residuals in radians before flagging, by default 1. If 0: No check. fitOffset : bool, optional Assume that together with a delay each station has also a differential phase offset (important for old LBA observations). By default False. average : bool, optional Mean-average in time the resulting delays/offset. By default False. replace : bool, optional replace using smoothed value instead of flag bad data? Smooth must be active. By default, False. minFreq : float, optional minimum frequency [Hz] to use in estimating the PA. By default, 0 (all freqs). refAnt : str, optional Reference antenna, by default the first. """ import numpy as np import scipy.optimize from scipy import stats from scipy.ndimage import generic_filter logging.info("Finding polarization align for soltab: "+soltab.name) delaycomplex = lambda d, freq, y: abs(np.cos(d[0]*freq) - np.cos(y)) + abs(np.sin(d[0]*freq) - np.sin(y)) #delaycomplex = lambda d, freq, y: abs(d[0]*freq - y) solType = soltab.getType() if solType != 'phase': logging.warning("Soltab type of "+soltab.name+" is of type "+solType+", should be phase. Ignoring.") return 1 if refAnt != '' and refAnt != 'closest' and not refAnt in soltab.getAxisValues('ant', ignoreSelection = True): logging.warning('Reference antenna '+refAnt+' not found. Using: '+soltab.getAxisValues('ant')[1]) refAnt = soltab.getAxisValues('ant')[1] if refAnt == '': refAnt = soltab.getAxisValues('ant')[1] # times and ants needs to be complete or selection is much slower times = soltab.getAxisValues('time') # create new table solset = soltab.getSolset() soltabout = solset.makeSoltab(soltype = soltab.getType(), soltabName = soltabOut, axesNames=soltab.getAxesNames(), axesVals=[soltab.getAxisValues(axisName) for axisName in soltab.getAxesNames()], vals=soltab.getValues(retAxesVals = False), weights=soltab.getValues(weight = True, retAxesVals = False)) soltabout.addHistory('Created by POLALIGN operation from %s.' % soltab.name) if 'XX' in soltab.getAxisValues('pol'): pol = 'XX' elif 'RR' in soltab.getAxisValues('pol'): pol = 'RR' else: logging.error('Cannot reference to known polarisation.') return 1 for vals, weights, coord, selection in soltab.getValuesIter(returnAxes=['freq','pol','time'], weight=True, refAnt=refAnt): # reorder axes vals = reorderAxes( vals, soltab.getAxesNames(), ['pol','freq','time'] ) weights = reorderAxes( weights, soltab.getAxesNames(), ['pol','freq','time'] ) if 'RR' in coord['pol'] and 'LL' in coord['pol']: coord1 = np.where(coord['pol'] == 'RR')[0][0] coord2 = np.where(coord['pol'] == 'LL')[0][0] elif 'XX' in coord['pol'] and 'YY' in coord['pol']: coord1 = np.where(coord['pol'] == 'XX')[0][0] coord2 = np.where(coord['pol'] == 'YY')[0][0] if (weights == 0.).all() == True: logging.warning('Skipping flagged antenna: '+coord['ant']) weights[:] = 0 else: fit_delays=[]; fit_offset=[]; fit_weights=[] for t, time in enumerate(times): # apply flags idx = ( (weights[coord1,:,t] != 0.) & (weights[coord2,:,t] != 0.) & (coord['freq'] > minFreq) ) freq = np.copy(coord['freq'])[idx] phase1 = vals[coord1,:,t][idx] phase2 = vals[coord2,:,t][idx] if len(freq) < 30: fit_weights.append(0.) fit_delays.append(0.) fit_offset.append(0.) logging.debug('Not enough unflagged point for the timeslot '+str(t)) continue # if more than 1/2 of chans are flagged if (len(idx) - len(freq))/float(len(idx)) > 1/2.: logging.debug('High number of filtered out data points for the timeslot %i: %i/%i' % (t, len(idx) - len(freq), len(idx)) ) phase_diff = phase1 - phase2 phase_diff = np.mod(phase_diff + np.pi, 2.*np.pi) - np.pi phase_diff = np.unwrap(phase_diff) A = np.vstack([freq, np.ones(len(freq))]).T fitresultdelay = np.linalg.lstsq(A, phase_diff.T)[0] # get the closest n*(2pi) to the intercept and refit with only 1 parameter if not fitOffset: numjumps = np.around(fitresultdelay[1]/(2*np.pi)) A = np.reshape(freq, (-1,1)) # no b phase_diff -= numjumps * 2 * np.pi fitresultdelay = np.linalg.lstsq(A, phase_diff.T)[0] fitresultdelay = [fitresultdelay[0],0.] # set offset to 0 to keep the rest of the script equal # fractional residual residual = np.mean(np.abs( fitresultdelay[0]*freq + fitresultdelay[1] - phase_diff )) fit_delays.append(fitresultdelay[0]) fit_offset.append(fitresultdelay[1]) if maxResidual == 0 or residual < maxResidual: fit_weights.append(1.) else: # high residual, flag logging.debug('Bad solution for ant: '+coord['ant']+' (time: '+str(t)+', residual: '+str(residual)+') -> ignoring.') fit_weights.append(0.) # Debug plot doplot = False #if doplot and t%100==0 and (coord['ant'] == 'RS310LBA' or coord['ant'] == 'CS301LBA'): if doplot and t%10==0 and (coord['ant'] == 'W04'): if not 'matplotlib' in sys.modules: import matplotlib as mpl mpl.rc('figure.subplot',left=0.05, bottom=0.05, right=0.95, top=0.95,wspace=0.22, hspace=0.22 ) mpl.use("Agg") import matplotlib.pyplot as plt fig = plt.figure() fig.subplots_adjust(wspace=0) ax = fig.add_subplot(111) # plot rm fit plotdelay = lambda delay, offset, freq: np.mod( delay*freq + offset + np.pi, 2.*np.pi) - np.pi ax.plot(freq, fitresultdelay[0]*freq + fitresultdelay[1], "-", color='purple', label=r'delay:%f$\nu$ (ns) + %f ' % (fitresultdelay[0]*1e9,fitresultdelay[1]) ) ax.plot(freq, np.mod(phase1 + np.pi, 2.*np.pi) - np.pi, 'ob' ) ax.plot(freq, np.mod(phase2 + np.pi, 2.*np.pi) - np.pi, 'og' ) #ax.plot(freq, np.mod(phase_diff + np.pi, 2.*np.pi) - np.pi, '.', color='purple' ) ax.plot(freq, phase_diff, '.', color='purple' ) residual = np.mod(plotdelay(fitresultdelay[0], fitresultdelay[1], freq)-phase_diff + np.pi,2.*np.pi)-np.pi ax.plot(freq, residual, '.', color='yellow') ax.set_xlabel('freq') ax.set_ylabel('phase') #ax.set_ylim(ymin=-np.pi, ymax=np.pi) logging.warning('Save pic: '+str(t)+'_'+coord['ant']+'.png') fig.legend(loc='upper left') plt.savefig(coord['ant']+'_'+str(t)+'.png', bbox_inches='tight') del fig # end cycle in time fit_weights = np.array(fit_weights) fit_delays = np.array(fit_delays) fit_offset = np.array(fit_offset) # avg in time if average: fit_delays_bkp = fit_delays[ fit_weights == 0 ] fit_offset_bkp = fit_offset[ fit_weights == 0 ] np.putmask(fit_delays, fit_weights == 0, np.nan) np.putmask(fit_offset, fit_weights == 0, np.nan) fit_delays[:] = np.nanmean(fit_delays) # angle mean fit_offset[:] = np.angle( np.nansum( np.exp(1j*fit_offset) ) / np.count_nonzero(~np.isnan(fit_offset)) ) if replace: fit_weights[ fit_weights == 0 ] = 1. fit_weights[ np.isnan(fit_delays) ] = 0. # all the size was flagged cannot estrapolate value else: fit_delays[ fit_weights == 0 ] = fit_delays_bkp fit_offset[ fit_weights == 0 ] = fit_offset_bkp logging.info('%s: average delay: %f ns (offset: %f)' % ( coord['ant'], np.mean(fit_delays)*1e9, np.mean(fit_offset))) for t, time in enumerate(times): #vals[:,:,t] = 0. #vals[coord1,:,t] = fit_delays[t]*np.array(coord['freq'])/2. vals[coord1,:,t] = 0 phase = np.mod(fit_delays[t]*coord['freq'] + fit_offset[t] + np.pi, 2.*np.pi) - np.pi vals[coord2,:,t] = -1.*phase#/2. #weights[:,:,t] = 0. weights[coord1,:,t] = fit_weights[t] weights[coord2,:,t] = fit_weights[t] # reorder axes back to the original order, needed for setValues vals = reorderAxes( vals, ['pol','freq','time'], [ax for ax in soltab.getAxesNames() if ax in ['pol','freq','time']] ) weights = reorderAxes( weights, ['pol','freq','time'], [ax for ax in soltab.getAxesNames() if ax in ['pol','freq','time']] ) soltabout.setSelection(**coord) soltabout.setValues( vals ) soltabout.setValues( weights, weight=True ) return 0
def run(soltab, axesInPlot, axisInTable='', axisInCol='', axisDiff='', NColFig=0, figSize=[0,0], markerSize=2, minmax=[0,0], log='', \ plotFlag=False, doUnwrap=False, refAnt='', soltabsToAdd='', makeAntPlot=False, makeMovie=False, prefix='', ncpu=0): """ This operation for LoSoTo implements basic plotting WEIGHT: flag-only compliant, no need for weight Parameters ---------- axesInPlot : array of str 1- or 2-element array which says the coordinates to plot (2 for 3D plots). axisInTable : str, optional the axis to plot on a page - e.g. ant to get all antenna's on one file. By default ''. axisInCol : str, optional The axis to plot in different colours - e.g. pol to get correlations with different colors. By default ''. axisDiff : str, optional This must be a len=2 axis and the plot will have the differential value - e.g. 'pol' to plot XX-YY. By default ''. NColFig : int, optional Number of columns in a multi-table image. By default is automatically chosen. figSize : array of int, optional Size of the image [x,y], if one of the values is 0, then it is automatically chosen. By default automatic set. markerSize : int, optional Size of the markers in the 2D plot. By default 2. minmax : array of float, optional Min max value for the independent variable (0 means automatic). By default 0. log : bool, optional Use Log='XYZ' to set which axes to put in Log. By default ''. plotFlag : bool, optional Whether to plot also flags as red points in 2D plots. By default False. doUnwrap : bool, optional Unwrap phases. By default False. refAnt : str, optional Reference antenna for phases. By default None. soltabsToAdd : str, optional Tables to "add" (e.g. 'sol000/tec000'), it works only for tec and clock to be added to phases. By default None. makeAntPlot : bool, optional Make a plot containing antenna coordinates in x,y and in color the value to plot, axesInPlot must be [ant]. By default False. makeMovie : bool, optional Make a movie summing up all the produced plots, by default False. prefix : str, optional Prefix to add before the self-generated filename, by default None. ncpu : int, optional Number of cpus, by default all available. """ import os, random import numpy as np from losoto.lib_unwrap import unwrap, unwrap_2d logging.info("Plotting soltab: " + soltab.name) # input check # str2list if axisInTable == '': axisInTable = [] else: axisInTable = [axisInTable] if axisInCol == '': axisInCol = [] else: axisInCol = [axisInCol] if axisDiff == '': axisDiff = [] else: axisDiff = [axisDiff] if len(set(axisInTable + axesInPlot + axisInCol + axisDiff)) != len(axisInTable + axesInPlot + axisInCol + axisDiff): logging.error('Axis defined multiple times.') return 1 # just because we use lists, check that they are 1-d if len(axisInTable) > 1 or len(axisInCol) > 1 or len(axisDiff) > 1: logging.error( 'Too many TableAxis/ColAxis/DiffAxis, they must be at most one each.' ) return 1 for axis in axesInPlot + axisInCol + axisDiff: if axis not in soltab.getAxesNames(): logging.error('Axis \"' + axis + '\" not found.') return 1 if makeMovie: prefix = prefix + '__tmp__' if os.path.dirname(prefix) != '' and not os.path.exists( os.path.dirname(prefix)): logging.debug('Creating ' + os.path.dirname(prefix) + '.') os.makedirs(os.path.dirname(prefix)) if refAnt == '': refAnt = None elif refAnt != 'closest' and not refAnt in soltab.getAxisValues( 'ant', ignoreSelection=True): logging.error('Reference antenna ' + refAnt + ' not found. Using: ' + soltab.getAxisValues('ant')[1]) refAnt = soltab.getAxisValues('ant')[1] minZ, maxZ = minmax solset = soltab.getSolset() soltabsToAdd = [ solset.getSoltab(soltabName) for soltabName in soltabsToAdd ] cmesh = False if len(axesInPlot) == 2: cmesh = True # not color possible in 3D axisInCol = [] elif len(axesInPlot) != 1: logging.error('Axes must be a len 1 or 2 array.') return 1 # end input check # all axes that are not iterated by anything else axesInFile = soltab.getAxesNames() for axis in axisInTable + axesInPlot + axisInCol + axisDiff: axesInFile.remove(axis) # set subplots scheme if axisInTable != []: Nplots = soltab.getAxisLen(axisInTable[0]) else: Nplots = 1 # prepare antennas coord in makeAntPlot case if makeAntPlot: if axesInPlot != ['ant']: logging.error( 'If makeAntPlot is selected the "Axes" values must be "ant"') return 1 antCoords = [[], []] for ant in soltab.getAxisValues( 'ant'): # select only user-selected antenna in proper order antCoords[0].append(+1 * soltab.getSolset().getAnt()[ant][1]) antCoords[1].append(-1 * soltab.getSolset().getAnt()[ant][0]) else: antCoords = [] datatype = soltab.getType() # start processes for multi-thread mpm = multiprocManager(ncpu, _plot) # compute dataCube size shape = [] if axisInTable != []: shape.append(soltab.getAxisLen(axisInTable[0])) else: shape.append(1) if axisInCol != []: shape.append(soltab.getAxisLen(axisInCol[0])) else: shape.append(1) if cmesh: shape.append(soltab.getAxisLen(axesInPlot[1])) shape.append(soltab.getAxisLen(axesInPlot[0])) else: shape.append(soltab.getAxisLen(axesInPlot[0])) # will contain the data to pass to each thread to make 1 image dataCube = np.ma.zeros(shape=shape, fill_value=np.nan) # cycle on files if makeMovie: pngs = [] # store png filenames for vals, coord, selection in soltab.getValuesIter( returnAxes=axisDiff + axisInTable + axisInCol + axesInPlot): # set filename filename = '' for axis in axesInFile: filename += axis + str(coord[axis]) + '_' filename = filename[:-1] # remove last _ if prefix + filename == '': filename = 'plot' # axis vals (they are always the same, regulat arrays) xvals = coord[axesInPlot[0]] # if plotting antenna - convert to number if axesInPlot[0] == 'ant': xvals = np.arange(len(xvals)) # if plotting time - convert in h/min/s xlabelunit = '' if axesInPlot[0] == 'time': if xvals[-1] - xvals[0] > 3600: xvals = (xvals - xvals[0]) / 3600. # hrs xlabelunit = ' [hr]' elif xvals[-1] - xvals[0] > 60: xvals = (xvals - xvals[0]) / 60. # mins xlabelunit = ' [min]' else: xvals = (xvals - xvals[0]) # sec xlabelunit = ' [s]' # if plotting freq convert in MHz elif axesInPlot[0] == 'freq': xvals = xvals / 1.e6 # MHz xlabelunit = ' [MHz]' if cmesh: # axis vals (they are always the same, regular arrays) yvals = coord[axesInPlot[1]] # same as above but for y-axis if axesInPlot[1] == 'ant': yvals = np.arange(len(yvals)) if len(xvals) <= 1 or len(yvals) <= 1: logging.error( '3D plot must have more then one value per axes.') mpm.wait() return 1 ylabelunit = '' if axesInPlot[1] == 'time': if yvals[-1] - yvals[0] > 3600: yvals = (yvals - yvals[0]) / 3600. # hrs ylabelunit = ' [hr]' elif yvals[-1] - yvals[0] > 60: yvals = (yvals - yvals[0]) / 60. # mins ylabelunit = ' [min]' else: yvals = (yvals - yvals[0]) # sec ylabelunit = ' [s]' elif axesInPlot[1] == 'freq': # Mhz yvals = yvals / 1.e6 ylabelunit = ' [MHz]' else: yvals = None if datatype == 'clock': datatype = 'Clock' ylabelunit = ' (s)' elif datatype == 'tec': datatype = 'dTEC' ylabelunit = ' (TECU)' elif datatype == 'rotationmeasure': datatype = 'dRM' ylabelunit = r' (rad m$^{-2}$)' elif datatype == 'tec3rd': datatype = r'dTEC$_3$' ylabelunit = r' (rad m$^{-3}$)' else: ylabelunit = '' # cycle on tables soltab1Selection = soltab.selection # save global selection and subselect only axex to iterate soltab.selection = selection titles = [] for Ntab, (vals, coord, selection) in enumerate( soltab.getValuesIter(returnAxes=axisDiff + axisInCol + axesInPlot)): # set tile titles.append('') for axis in coord: if axis in axesInFile + axesInPlot + axisInCol: continue titles[Ntab] += axis + ':' + str(coord[axis]) + ' ' titles[Ntab] = titles[Ntab][:-1] # remove last ' ' # cycle on colors soltab2Selection = soltab.selection soltab.selection = selection for Ncol, (vals, weight, coord, selection) in enumerate( soltab.getValuesIter(returnAxes=axisDiff + axesInPlot, weight=True, reference=refAnt)): # differential plot if axisDiff != []: # find ordered list of axis names = [ axis for axis in soltab.getAxesNames() if axis in axisDiff + axesInPlot ] if axisDiff[0] not in names: logging.error("Axis to differentiate (%s) not found." % axisDiff[0]) mpm.wait() return 1 if len(coord[axisDiff[0]]) != 2: logging.error( "Axis to differentiate (%s) has too many values, only 2 is allowed." % axisDiff[0]) mpm.wait() return 1 # find position of interesting axis diff_idx = names.index(axisDiff[0]) # roll to first place vals = np.rollaxis(vals, diff_idx, 0) vals = vals[0] - vals[1] weight = np.rollaxis(weight, diff_idx, 0) weight[0][weight[1] == 0] = 0 weight = weight[0] del coord[axisDiff[0]] # add tables if required (e.g. phase/tec) for soltabToAdd in soltabsToAdd: logging.warning('soltabsToAdd not implemented. Ignoring.') # newCoord = {} # for axisName in coord.keys(): # # prepare selected on present axes # if axisName in soltabToAdd.getAxesNames(): # if type(coord[axisName]) is np.ndarray: # newCoord[axisName] = coord[axisName] # else: # newCoord[axisName] = [coord[axisName]] # avoid being interpreted as regexp, faster # # soltabToAdd.setSelection(**newCoord) # valsAdd = np.squeeze(soltabToAdd.getValues(retAxesVals=False, weight=False, reference=refAnt)) # # # add missing axes # print ('shape:', vals.shape) # for axisName in coord.keys(): # if not axisName in soltabToAdd.getAxesNames(): # # find axis positions # axisPos = soltab.getAxesNames().index(axisName) # # create a new axes for the table to add and duplicate the values # valsAdd = np.expand_dims(valsAdd, axisPos) # print ('shape to add:', valsAdd.shape) # # if soltabToAdd.getType() == 'clock': # valsAdd = 2. * np.pi * valsAdd * coord['freq'] # elif soltabToAdd.getType() == 'tec': # valsAdd = -8.44797245e9 * valsAdd / coord['freq'] # else: # logging.warning('Only Clock or TEC can be added to solutions. Ignoring: '+soltabToAdd.getType()+'.') # continue # # if valsAdd.shape != vals.shape: # logging.error('Cannot combine the table '+soltabToAdd.getType()+' with '+soltab.getType()+'. Wrong shape.') # mpm.wait() # return 1 # # vals += valsAdd # normalize if (soltab.getType() == 'phase' or soltab.getType() == 'scalarphase'): vals = normalize_phase(vals) if (soltab.getType() == 'rotation'): vals = np.mod(vals + np.pi / 2., np.pi) - np.pi / 2. # is user requested axis in an order that is different from h5parm, we need to transpose if cmesh: if soltab.getAxesNames().index( axesInPlot[0]) < soltab.getAxesNames().index( axesInPlot[1]): vals = vals.T weight = weight.T # unwrap if required if (soltab.getType() == 'phase' or soltab.getType() == 'scalarphase') and doUnwrap: if len(axesInPlot) == 1: vals = unwrap(vals) else: flags = np.array((weight == 0), dtype=bool) if not (flags == True).all(): vals = unwrap_2d(vals, flags, coord[axesInPlot[0]], coord[axesInPlot[1]]) dataCube[Ntab, Ncol] = vals sel1 = np.where(weight == 0.) sel2 = np.where(np.isnan(vals)) if cmesh: dataCube[Ntab, Ncol, sel1[0], sel1[1]] = np.ma.masked dataCube[Ntab, Ncol, sel2[0], sel2[1]] = np.ma.masked else: dataCube[Ntab, Ncol, sel1[0]] = np.ma.masked dataCube[Ntab, Ncol, sel2[0]] = np.ma.masked soltab.selection = soltab2Selection ### end cycle on colors # if dataCube too large (> 500 MB) do not go parallel if np.array(dataCube).nbytes > 1024 * 1024 * 500: logging.debug('Big plot, parallel not possible.') _plot(Nplots, NColFig, figSize, markerSize, cmesh, axesInPlot, axisInTable, xvals, yvals, xlabelunit, ylabelunit, datatype, prefix + filename, titles, log, dataCube, minZ, maxZ, plotFlag, makeMovie, antCoords, None) else: mpm.put([ Nplots, NColFig, figSize, markerSize, cmesh, axesInPlot, axisInTable, xvals, yvals, xlabelunit, ylabelunit, datatype, prefix + filename, titles, log, np.ma.copy(dataCube), minZ, maxZ, plotFlag, makeMovie, antCoords ]) if makeMovie: pngs.append(prefix + filename + '.png') soltab.selection = soltab1Selection ### end cycle on tables mpm.wait() if makeMovie: def long_substr(strings): """ Find longest common substring """ substr = '' if len(strings) > 1 and len(strings[0]) > 0: for i in range(len(strings[0])): for j in range(len(strings[0]) - i + 1): if j > len(substr) and all(strings[0][i:i + j] in x for x in strings): substr = strings[0][i:i + j] return substr movieName = long_substr(pngs) assert movieName != '' # need a common prefix, use prefix keyword in case logging.info('Making movie: ' + movieName) # make every movie last 20 sec, min one second per slide fps = np.ceil(len(pngs) / 200.) ss="mencoder -ovc lavc -lavcopts vcodec=mpeg4:vpass=1:vbitrate=6160000:mbd=2:keyint=132:v4mv:vqmin=3:lumi_mask=0.07:dark_mask=0.2:"+\ "mpeg_quant:scplx_mask=0.1:tcplx_mask=0.1:naq -mf type=png:fps="+str(fps)+" -nosound -o "+movieName.replace('__tmp__','')+".mpg mf://"+movieName+"* > mencoder.log 2>&1" os.system(ss) #for png in pngs: os.system('rm '+png) return 0
#!/usr/bin/env python # -*- coding: utf-8 -*- from losoto.lib_operations import * from losoto._logging import logger as logging logging.debug('Loading FLAGSTATION module.') def _run_parser(soltab, parser, step): mode = parser.getstr(step, 'mode') # no default maxFlaggedFraction = parser.getfloat(step, 'maxFlaggedFraction', 0.5) nSigma = parser.getfloat(step, 'nSigma', 5.0) maxStddev = parser.getfloat(step, 'maxStddev', -1.0) ampRange = parser.getarrayfloat(step, 'ampRange', [0., 0.]) telescope = parser.getstr(step, 'telescope', 'lofar') skipInternational = parser.getbool(step, 'skipInternational', False) refAnt = parser.getstr(step, 'refAnt', '') soltabExport = parser.getstr(step, 'soltabExport', '') ncpu = parser.getint('_global', 'ncpu', 0) parser.checkSpelling(step, soltab, [ 'mode', 'maxFlaggedFraction', 'nSigma', 'maxStddev', 'ampRange', 'telescope', 'skipInternational', 'refAnt', 'soltabExport' ]) return run(soltab, mode, maxFlaggedFraction, nSigma, maxStddev, ampRange, telescope, skipInternational, refAnt, soltabExport, ncpu) def _flag_resid(vals, weights, soltype, nSigma, maxFlaggedFraction, maxStddev, ants, s, outQueue):
def doFit( phases, mask, freqs, stations, station_positions, axes, refstIdx='superterp', flagBadChannels=True, flagcut=1.5, chi2cut=30000., removePhaseWraps=True, combine_pol=False, fit3rdorder=False, circular=False, initSol=[], initoffsets=[], n_proc=10 ): # make sure order of axes is as expected stidx = axes.index('ant') freqidx = axes.index('freq') timeidx = axes.index('time') try: polidx = axes.index('pol') data = ma.array(phases, mask=mask).transpose((timeidx, freqidx, stidx, polidx)) npol = data.shape[3] except: data = ma.array(phases, mask=mask).transpose((timeidx, freqidx, stidx)) data = data.reshape(data.shape+(1,)) npol = 1 nT = data.shape[0] nF = data.shape[1] nSt = data.shape[2] if npol == 4: data = data[:, :, :, (0, 3)] npol = 2 if refstIdx == 'superterp': superterpstations = [i for i in stations if i[:5] in [ 'CS002', 'CS003', 'CS004', 'CS005', 'CS006', 'CS007' ]] refstIdx = [i for (i, j) in enumerate(stations) if j in superterpstations] if len(refstIdx)==0: refstIdx=0 if not hasattr(refstIdx, '__len__'): refstIdx = [refstIdx] # get phases from reference stations refdata = np.ma.sum(np.cos(data[:, :, refstIdx, :]) + 1.j * np.sin(data[:, :, refstIdx, :]), axis=2) refdata = np.ma.arctan2(np.imag(refdata), np.real(refdata)) # unwrap around mean mymean = np.ma.average(refdata, axis=0) refdata = np.ma.remainder(refdata - mymean[np.newaxis] + np.pi, 2 * np.pi) + mymean[np.newaxis] - np.pi data -= refdata[:, :, np.newaxis] # flag bad channels - test if really nedded if data already flagged indices = np.arange(nF) if flagBadChannels: mymask=np.zeros((nF), dtype=np.bool) for nr_iter in range(2): rms = np.ma.std(np.ma.std(refdata, axis=0), axis=1) freqselect = rms < flagcut * np.average(rms) logging.debug('Iter %d: flagging %d channels' % (nr_iter, np.sum(np.logical_not(np.array(freqselect))))) #freqs = freqs[freqselect] #data = data[:, freqselect] mymask=np.ma.logical_or(mymask,~freqselect) #refdata = refdata[:, freqselect] refdata.mask=np.ma.logical_or(refdata.mask,mymask[np.newaxis,:,np.newaxis]) logging.debug('Flagging: ' + str(indices[np.logical_not(freqselect)])) #indices = indices[freqselect] mask=data.mask data.mask=np.logical_or(mask,mymask[np.newaxis,:,np.newaxis,np.newaxis]) # Remove completely flagged stations selectstations = [st for stationIndices, st in enumerate(stations) if data[:, :, stationIndices].mask.all() != True] logging.debug('%d selected stations: ' % len(selectstations) + str(selectstations)) stationIndices = np.array([idxst in selectstations for idxst in stations]) data = data[:, :, stationIndices] stations_old = stations[:] # keep a copy for putting flagged stations back at the end stations = stations[stationIndices] station_positions = station_positions[stationIndices] RSstations = [i for (i, j) in enumerate(stations) if 'RS' in j] CSstations = [i for (i, j) in enumerate(stations) if 'CS' in j] otherstations = [i for (i, j) in enumerate(stations) if not 'RS' in j and not 'CS' in j] logging.debug('Station indices: ' + str(stationIndices) + ' RS ' + str(RSstations)) nSt = data.shape[2] # combine polarizationsif requested - needed in HBA if combine_pol: if npol == 2 and not circular: cdata = np.ma.cos(data) + 1.j * np.ma.sin(data) data = np.ma.sum(cdata, axis=3).reshape((nT, nF, nSt, 1)) data = np.ma.arctan2(np.imag(data), np.real(data)) # np.angle doesnot yet return masked array!! npol = 1 if circular: data=np.ma.sum(data, axis=3).reshape((nT, nF, nSt, 1)) npol=1 # guess clock, remove from data # not in LBA because TEC dominant if not 'LBA' in stations[0] and len(initSol) < 1: initclock = getInitClock(data[int(nT / 2):int(nT / 2 + 100)][:, :, RSstations + otherstations], freqs) # only on a few timestamps logging.debug('Initial clocks: ' + str(initclock[1])) # init CS clocks to 0 # logging.debug("data before init clock" + str(data[nT/2,:,-1])) data[:, :, RSstations + otherstations] = data[:, :, RSstations + otherstations] - freqs[np.newaxis, :, np.newaxis, np.newaxis] * initclock[1][np.newaxis, np.newaxis, :] * 1e-9 * 2 * np.pi # logging.debug("clock correction" + str(np.remainder(freqs*initclock[1][-1]*-1e-9*2*np.pi+np.pi,2*np.pi)-np.pi)) # logging.debug("data after init clock" + str(np.remainder(data[nT/2,:,-1]+np.pi,2*np.pi)-np.pi)) offset = np.zeros((nSt, npol), dtype=np.float32) if len(initoffsets)>0: # Check if initoffsets is not empty offset = initoffsets data[:, :, :, :] += offset[:][np.newaxis, np.newaxis] # initialize arrays clock = np.zeros((nT, nSt, npol), dtype=np.float32) tec = np.zeros((nT, nSt, npol), dtype=np.float32) if fit3rdorder: tec3rd = np.zeros((nT, nSt, npol), dtype=np.float32) # better not to use fitoffset tecarray = np.zeros((nT, nSt), dtype=np.float32) clockarray = np.zeros((nT, nSt), dtype=np.float32) residualarray = np.zeros((nT, nF, nSt), dtype=np.float32) if fit3rdorder: tec3rdarray= np.zeros((nT, nSt), dtype=np.float32) for pol in range(npol): # get a good guesss without offset # logging.debug("sending masked data "+str(data[:,:,:,pol].count())) initialchi2cut = chi2cut # user defined if removePhaseWraps: initialchi2cut = 30000. # this number is quite arbitrary if fit3rdorder: poolargs=[] for ist in range(nSt): poolargs.append((np.ma.copy(data[:, :, ist, pol]), freqs, stations[ist], [], True, initialchi2cut, True, circular and combine_pol )) p=Pool(processes=n_proc) result=p.map(merge_ct_args,poolargs) for ist,tc in enumerate(result): tecarray[:, ist], clockarray[:, ist],residualarray[:,:,ist],tec3rdarray[:,ist] = tc else: poolargs=[] for ist in range(nSt): poolargs.append((np.ma.copy(data[:, :, ist, pol]), freqs, stations[ist], [], True, initialchi2cut, False, circular and combine_pol )) p=Pool(processes=n_proc) result=p.map(merge_ct_args,poolargs) for ist,tc in enumerate(result): tecarray[:, ist], clockarray[:, ist],residualarray[:,:,ist] = tc if removePhaseWraps: # correctfrist times only,try to make init correct ? #corrects wraps based on spatial correlation (averaged in time), only works for long time observations, not testted for LBA (offset[:, pol], wraps, steps) = correctWraps(tecarray, residualarray, freqs, station_positions) else: #always correct for wraps based on average residuals wraps, steps = correctWrapsFromResiduals(residualarray, tecarray<-5,freqs) #maximum allowed 2pi phase wrap is +-1 #wraps = np.minimum(wraps,2) #wraps = np.maximum(wraps,-2) logging.debug('Residual iter 1, pol %d: ' % pol + str(residualarray[0, 0])) logging.debug('TEC iter 1, pol %d: ' % pol + str(tecarray[0])) logging.debug('Clock iter 1, pol %d: ' % pol + str(clockarray[0])) logging.debug('Wraps: ' + str(wraps)) logging.debug('Offsets: ' + str(offset[:, pol])) # remove completely initialoffset? if len(initoffsets)>0: # Check if initoffsets is not empty offset[:, pol] -= initoffsets[:, pol] data[:, :, :, pol] += offset[:, pol][np.newaxis, np.newaxis] # remove fitoffset if removePhaseWraps: initsol = np.zeros((nSt, 2), dtype=np.float32) initsol[:, 0] = get_first_good(tecarray[:, :]) + wraps * steps[0] initsol[:, 1] = get_first_good(clockarray[:, :]) + wraps * steps[1] logging.debug('Initsol TEC, pol %d: ' % pol + str(initsol[:, 0])) logging.debug('Initsol clock, pol %d: ' % pol + str(initsol[:, 1])) # is it needed to redo the fitting? this is the time bottleneck if fit3rdorder: poolargs=[] for ist in range(nSt): poolargs.append((np.ma.copy(data[:, :, ist, pol]), freqs, stations[ist], initsol[ist], False, chi2cut, True, circular and combine_pol )) p=Pool(processes=n_proc) result=p.map(merge_ct_args,poolargs) for ist,tc in enumerate(result): tec[:, ist, pol], clock[:, ist, pol],tec3rd[:,ist,pol] = tc else: poolargs=[] for ist in range(nSt): poolargs.append((np.ma.copy(data[:, :, ist, pol]), freqs, stations[ist], initsol[ist], False, chi2cut, False, circular and combine_pol )) p=Pool(processes=n_proc) result=p.map(merge_ct_args,poolargs) for ist,tc in enumerate(result): tec[:, ist, pol], clock[:, ist, pol] = tc else: tec[:, :, pol] = tecarray[:, :]+ wraps * steps[0] clock[:, :, pol] = clockarray[:, :]+ wraps * steps[1] if fit3rdorder: tec3rd[:, :, pol] = tec3rdarray[:, :]+ wraps * steps[2] logging.debug('TEC iter 2, pol %d: ' % pol + str(tec[0, :, pol])) logging.debug('Clock iter 2, pol %d: ' % pol + str(clock[0, :, pol])) if not 'LBA' in stations[0] and len(initSol) < 1: clock[:, RSstations + otherstations] += initclock[1][np.newaxis, :, :] if combine_pol and circular: clock/=2;tec/=2;offset/=2 # put flagged stations back for idx, wasUsed in enumerate(stationIndices): if not wasUsed: logging.debug('Adding completely flagged station '+str(idx)) clock = np.insert(clock,idx,0,axis=1) # [time:ant:pol] tec = np.insert(tec,idx,-5,axis=1) # [time:ant:pol] offset = np.insert(offset,idx,0,axis=0) # [ant:pol] if fit3rdorder: tec3rd = np.insert(tec3rd,idx,0,axis=1) # [time:ant:pol] # station = station_old if fit3rdorder: return (clock, tec, offset, tec3rd) else: return (clock, tec, offset)
#!/usr/bin/env python # -*- coding: utf-8 -*- from __future__ import print_function from losoto.lib_operations import * from losoto._logging import logger as logging logging.debug('Loading BANDPASSTEC module.') def _run_parser(soltab, parser, step): soltabOutTec = parser.getstr(step, 'soltabOutTEC', 'tec000') soltabOutBP = parser.getstr(step, 'soltabOutBP', 'phase000') refAnt = parser.getstr(step, 'refAnt', '') parser.checkSpelling( step, soltab, ['soltabOutTEC', 'soltabOutBP', 'refAnt', 'maxResidual']) return run(soltab, soltabOutTEC, soltabOutBP, refAnt) def run(soltab, soltabOutTEC='tec000', soltabOutBP='phase000', refAnt=''): """ Isolate BP from TEC in a calibrator (uGMRT data) Parameters ---------- soltabOutTEC : str, optional output TEC table name (same solset), by deault "tec000". soltabOutBP : str, optional output bandpass table name (same solset), by deault "phase000".
def run(soltab, soltabOut='tec000', refAnt='', maxResidualFlag=2.5, maxResidualProp=1.): """ Bruteforce TEC extraction from phase solutions. Parameters ---------- soltabOut : str, optional output table name (same solset), by deault "tec". refAnt : str, optional Reference antenna, by default the first. maxResidualFlag : float, optional Max average residual in radians before flagging datapoint, by default 2.5 If 0: no check. maxResidualProp : float, optional Max average residual in radians before stop propagating solutions, by default 1. If 0: no check. """ import numpy as np import scipy.optimize from losoto.lib_unwrap import unwrap_2d def mod(d): return np.mod(d + np.pi, 2. * np.pi) - np.pi drealbrute = lambda d, freq, y: np.sum( np.abs(mod(-8.44797245e9 * d / freq) - y)) # bruteforce dreal = lambda d, freq, y: mod(-8.44797245e9 * d[0] / freq) - y #dreal2 = lambda d, freq, y: mod(-8.44797245e9*d[0]/freq + d[1]) - y #dcomplex = lambda d, freq, y: np.sum( ( np.cos(-8.44797245e9*d/freq) - np.cos(y) )**2 ) + np.sum( ( np.sin(-8.44797245e9*d/freq) - np.sin(y) )**2 ) #def dcomplex( d, freq, y, y_pre, y_post): # return np.sum( ( np.absolute( np.exp(-1j*8.44797245e9*d/freq) - np.exp(1j*y) ) )**2 ) + \ # .5*np.sum( ( np.absolute( np.exp(-1j*8.44797245e9*d/freq) - np.exp(1j*y_pre) ) )**2 ) + \ # .5*np.sum( ( np.absolute( np.exp(-1j*8.44797245e9*d/freq) - np.exp(1j*y_post) ) )**2 ) #dcomplex2 = lambda d, freq, y: abs(np.cos(-8.44797245e9*d[0]/freq + d[1]) - np.cos(y)) + abs(np.sin(-8.44797245e9*d[0]/freq + d[1]) - np.sin(y)) logging.info("Find TEC for soltab: " + soltab.name) # input check solType = soltab.getType() if solType != 'phase': logging.warning("Soltab type of " + soltab._v_name + " is of type " + solType + ", should be phase. Ignoring.") return 1 ants = soltab.getAxisValues('ant') if refAnt != '' and refAnt != 'closest' and not refAnt in soltab.getAxisValues( 'ant', ignoreSelection=True): logging.error('Reference antenna ' + refAnt + ' not found. Using: ' + ants[1]) refAnt = ants[0] if refAnt == '': refAnt = ants[0] # times and ants needs to be complete or selection is much slower times = soltab.getAxisValues('time') # create new table solset = soltab.getSolset() soltabout = solset.makeSoltab(soltype = 'tec', soltabName = soltabOut, axesNames=['ant','time'], \ axesVals=[soltab.getAxisValues(axisName) for axisName in ['ant','time']], \ vals=np.zeros(shape=(soltab.getAxisLen('ant'),soltab.getAxisLen('time'))), \ weights=np.ones(shape=(soltab.getAxisLen('ant'),soltab.getAxisLen('time'))) ) soltabout.addHistory('Created by TEC operation from %s.' % soltab.name) for vals, weights, coord, selection in soltab.getValuesIter( returnAxes=['freq', 'time'], weight=True, reference=refAnt): if len(coord['freq']) < 10: logging.error( 'Delay estimation needs at least 10 frequency channels, preferably distributed over a wide range.' ) return 1 # reorder axes vals = reorderAxes(vals, soltab.getAxesNames(), ['freq', 'time']) weights = reorderAxes(weights, soltab.getAxesNames(), ['freq', 'time']) fitd = np.zeros(len(times)) fitweights = np.ones(len(times)) # all unflagged to start #fitdguess = 0.01 # good guess ranges = (-0.5, 0.5) Ns = 1000 if not coord['ant'] == refAnt: if (weights == 0.).all() == True: logging.warning('Skipping flagged antenna: ' + coord['ant']) fitweights[:] = 0 else: # unwrap 2d timexfreq #flags = np.array((weights == 0), dtype=bool) #vals = unwrap_2d(vals, flags, coord['freq'], coord['time']) for t, time in enumerate(times): # apply flags idx = (weights[:, t] != 0.) freq = np.copy(coord['freq'])[idx] if t == 0: phaseComb_pre = vals[idx, 0] else: phaseComb_pre = vals[idx, t - 1] if t == len(times) - 1: phaseComb_post = vals[idx, -1] else: phaseComb_post = vals[idx, t + 1] phaseComb = vals[idx, t] if len(freq) < 10: fitweights[t] = 0 logging.warning( 'No valid data found for delay fitting for antenna: ' + coord['ant'] + ' at timestamp ' + str(t)) continue # if more than 1/4 of chans are flagged if (len(idx) - len(freq)) / float(len(idx)) > 1 / 3.: logging.debug( 'High number of filtered out data points for the timeslot %i: %i/%i' % (t, len(idx) - len(freq), len(idx))) # least square 2 #fitresultd2, success = scipy.optimize.leastsq(dreal2, [fitdguess,0.], args=(freq, phaseComb)) #numjumps = np.around(fitresultd2[1]/(2*np.pi)) #print 'best jumps:', numjumps #phaseComb -= numjumps * 2*np.pi #fitresultd, success = scipy.optimize.leastsq(dreal, [fitresultd2[0]], args=(freq, phaseComb)) # least square 1 #fitresultd, success = scipy.optimize.leastsq(dreal, [fitdguess], args=(freq, phaseComb)) # hopper #fitresultd = scipy.optimize.basinhopping(dreal, [fitdguess], T=1, minimizer_kwargs={'args':(freq, phaseComb)}) #fitresultd = [fitresultd.x] #best_residual = np.nanmean(np.abs( mod(-8.44797245e9*fitresultd[0]/freq) - phaseComb ) ) #best_residual = np.inf #for jump in [-2,-1,0,1,2]: # fitresultd, success = scipy.optimize.leastsq(dreal, [fitdguess], args=(freq, phaseComb - jump * 2*np.pi)) # print fitresultd # # fractional residual # residual = np.nanmean(np.abs( (-8.44797245e9*fitresultd[0]/freq) - phaseComb - jump * 2*np.pi ) ) # if residual < best_residual: # best_residual = residual # fitd[t] = fitresultd[0] # best_jump = jump # brute force fitresultd = scipy.optimize.brute(drealbrute, ranges=(ranges, ), Ns=Ns, args=(freq, phaseComb)) fitresultd, success = scipy.optimize.leastsq( dreal, fitresultd, args=(freq, phaseComb)) best_residual = np.nanmean( np.abs( mod(-8.44797245e9 * fitresultd[0] / freq) - phaseComb)) fitd[t] = fitresultd[0] if maxResidualFlag == 0 or best_residual < maxResidualFlag: fitweights[t] = 1 if maxResidualProp == 0 or best_residual < maxResidualProp: ranges = (fitresultd[0] - 0.05, fitresultd[0] + 0.05) Ns = 100 else: ranges = (-0.5, 0.5) Ns = 1000 else: # high residual, flag and reset initial guess logging.warning('Bad solution for ant: ' + coord['ant'] + ' (time: ' + str(t) + ', resdiual: ' + str(best_residual) + ').') fitweights[t] = 0 ranges = (-0.5, 0.5) Ns = 1000 # Debug plot doplot = False if doplot and (coord['ant'] == 'RS509LBA' or coord['ant'] == 'RS210LBA') and t % 50 == 0: print("Plotting") if not 'matplotlib' in sys.modules: import matplotlib as mpl mpl.rc('figure.subplot', left=0.05, bottom=0.05, right=0.95, top=0.95, wspace=0.22, hspace=0.22) mpl.use("Agg") import matplotlib.pyplot as plt fig = plt.figure() fig.subplots_adjust(wspace=0) ax = fig.add_subplot(111) # plot rm fit plotd = lambda d, freq: -8.44797245e9 * d / freq ax.plot(freq, plotd(fitresultd[0], freq[:]), "-", color='purple') ax.plot(freq, mod(plotd(fitresultd[0], freq[:])), ":", color='purple') #ax.plot(freq, vals[idx,t], '.b' ) #ax.plot(freq, phaseComb + numjumps * 2*np.pi, 'x', color='purple' ) ax.plot(freq, phaseComb, 'o', color='purple') residual = mod(plotd(fitd[t], freq[:]) - phaseComb) ax.plot(freq, residual, '.', color='orange') ax.set_xlabel('freq') ax.set_ylabel('phase') #ax.set_ylim(ymin=-np.pi, ymax=np.pi) logging.warning('Save pic: ' + str(t) + '_' + coord['ant'] + '.png') plt.savefig(str(t) + '_' + coord['ant'] + '.png', bbox_inches='tight') del fig logging.info('%s: average tec: %f TECU' % (coord['ant'], np.mean(2 * fitd))) # reorder axes back to the original order, needed for setValues soltabout.setSelection(ant=coord['ant']) soltabout.setValues(fitd) soltabout.setValues(fitweights, weight=True) return 0
#!/usr/bin/env python # -*- coding: utf-8 -*- from losoto.lib_operations import * from losoto._logging import logger as logging logging.debug('Loading POLALIGN module.') def _run_parser(soltab, parser, step): soltabOut = parser.getstr( step, 'soltabOut', 'phasediff' ) maxResidual = parser.getfloat( step, 'maxResidual', 1. ) fitOffset = parser.getbool( step, 'fitOffset', False ) average = parser.getbool( step, 'average', False ) replace = parser.getbool( step, 'replace', False ) minFreq = parser.getfloat( step, 'minFreq', 0 ) refAnt = parser.getstr( step, 'refAnt', '' ) parser.checkSpelling( step, soltab, ['soltabOut', 'maxResidual', 'fitOffset', 'average', 'replace', 'minFreq', 'refAnt']) return run(soltab, soltabOut, maxResidual, fitOffset, average, replace, minFreq, refAnt) def run( soltab, soltabOut='phasediff', maxResidual=1., fitOffset=False, average=False, replace=False, minFreq=0, refAnt='' ): """ Estimate polarization misalignment as delay. Parameters ---------- soltabOut : str, optional output table name (same solset), by deault "phasediff". maxResidual : float, optional