def run(soltab, axesToNorm, normVal=1., log=False): """ Normalize the solutions to a given value WEIGHT: Weights compliant Parameters ---------- axesToNorm : array of str Axes along which compute the normalization. normVal : float, optional Number to normalize to vals = vals * (normVal/valsMean), by default 1. log : bool, optional clip is done in log10 space, by default False. """ import numpy as np logging.info("Normalizing soltab: " + soltab.name) # input check axesNames = soltab.getAxesNames() for normAxis in axesToNorm: if normAxis not in axesNames: logging.error('Normalization axis ' + normAxis + ' not found.') return 1 if soltab.getType() == 'amplitude' and not log: logging.warning( 'Amplitude solution tab detected and log=False. Amplitude solution tables should be treated in log space.' ) for vals, weights, coord, selection in soltab.getValuesIter( returnAxes=axesToNorm, weight=True): if log: vals = np.log10(vals) # rescale solutions if np.all(weights == 0): continue # skip flagged selections valsMean = np.nanmean(vals[weights != 0]) if log: vals[weights != 0] += np.log10(normVal) - valsMean else: vals[weights != 0] *= normVal / valsMean logging.debug("Rescaling by: " + str(normVal / valsMean)) # writing back the solutions if log: vals = 10**vals soltab.setValues(vals, selection) soltab.flush() soltab.addHistory('NORM (on axis %s)' % (axesToNorm)) return 0
def getStepSoltabs(parser, step, H): """ Return a list of soltabs object for a step and apply selection creteria Parameters ---------- parser : parser obj configuration file step : str current step H : h5parm obj the h5parm object Returns ------- list list of soltab obj with applied selection """ # selection on soltabs if parser.has_option(step, 'soltab'): stsel = parser.getarraystr(step, 'soltab') elif parser.has_option('_global', 'soltab'): stsel = parser.getarraystr('_global', 'soltab') else: stsel = ['.*/.*'] # select all #if not type(stsel) is list: stsel = [stsel] soltabs = [] for solset in H.getSolsets(): for soltabName in solset.getSoltabNames(): if any( re.compile(this_stsel).match(solset.name + '/' + soltabName) for this_stsel in stsel): if parser.getstr(step, 'operation').lower() in cacheSteps: soltabs.append(solset.getSoltab(soltabName, useCache=True)) else: soltabs.append(solset.getSoltab(soltabName, useCache=False)) if soltabs == []: logging.warning('No soltabs selected for step %s.' % step) # axes selection for soltab in soltabs: userSel = {} for axisName in soltab.getAxesNames(): userSel[axisName] = getParAxis(parser, step, axisName) soltab.setSelection(**userSel) return soltabs
def checkSpelling(self, s, soltab, availValues=[]): """ check if any value in the step is missing from a value list and return a warning """ entries = [x.lower() for x in list(dict(self.items(s)).keys())] availValues = ['soltab','operation'] + availValues + \ soltab.getAxesNames() + [a+'.minmaxstep' for a in soltab.getAxesNames()] + [a+'.regexp' for a in soltab.getAxesNames()] availValues = [x.lower() for x in availValues] for e in entries: if e not in availValues: logging.warning('Mispelled option: %s - Ignoring!' % e)
def _run_timestep(t,coord_rr,coord_ll,weights,vals,solType,coord,maxResidual): c = 2.99792458e8 if solType == 'phase': idx = ((weights[coord_rr,:] != 0.) & (weights[coord_ll,:] != 0.)) freq = np.copy(coord['freq'])[idx] phase_rr = vals[coord_rr,:][idx] phase_ll = vals[coord_ll,:][idx] # RR-LL to be consistent with BBS/NDPPP phase_diff = (phase_rr - phase_ll) # not divide by 2 otherwise jump problem, then later fix this else: # rotation table idx = ((weights[:] != 0.) & (weights[:] != 0.)) freq = np.copy(coord['freq'])[idx] phase_diff = 2.*vals[:][idx] # a rotation is between -pi and +pi if len(freq) < 20: fitresultrm_wav = [0] weight = 0 logging.warning('No valid data found for Faraday fitting for antenna: '+coord['ant']+' at timestamp '+str(t)) else: # if more than 1/4 of chans are flagged if (len(idx) - len(freq))/float(len(idx)) > 1/4.: logging.debug('High number of filtered out data points for the timeslot %i: %i/%i' % (t, len(idx) - len(freq), len(idx)) ) wav = c/freq #fitresultrm_wav, success = scipy.optimize.leastsq(rmwavcomplex, [fitrmguess], args=(wav, phase_diff)) ranges = slice(-0.1, 0.1, 2e-4) fitresultrm_wav = scipy.optimize.brute(costfunctionRM, (ranges,), finish=scipy.optimize.leastsq, args=(wav, phase_diff)) # fractional residual residual = np.nanmean(np.abs(np.mod((2.*fitresultrm_wav*wav*wav)-phase_diff + np.pi, 2.*np.pi) - np.pi)) if maxResidual == 0 or residual < maxResidual: fitrmguess = fitresultrm_wav[0] # Don't know what this is for... weight = 1 else: # high residual, flag logging.warning('Bad solution for ant: '+coord['ant']+' (time: '+str(t)+', resdiaul: '+str(residual)+').') weight = 0 return fitresultrm_wav[0],weight
def run(soltab, soltabsToSub, ratio=False): """ Subtract/divide two tables or a clock/tec/tec3rd/rm from a phase. Parameters ---------- soltabsToSub : list of str List of soltabs to subtract ratio : bool, optional Return the ratio instead of subtracting, by default False. """ import numpy as np logging.info("Subtract soltab: " + soltab.name) solset = soltab.getSolset() for soltabToSub in soltabsToSub: soltabsub = solset.getSoltab(soltabToSub) if soltab.getType() != 'phase' and ( soltabsub.getType() == 'tec' or soltabsub.getType() == 'clock' or soltabsub.getType() == 'rotationmeasure' or soltabsub.getType() == 'tec3rd'): logging.warning( soltabToSub + ' is of type clock/tec/rm and should be subtracted from a phase. Skipping it.' ) return 1 logging.info('Subtracting table: ' + soltabToSub) # a major speed up if tables are assumed with same axes, check that (should be the case in almost any case) for i, axisName in enumerate(soltabsub.getAxesNames()): # also armonise selection by copying only the axes present in the outtable and in the right order soltabsub.selection[i] = soltab.selection[ soltab.getAxesNames().index(axisName)] assert (soltabsub.getAxisValues(axisName) == soltab.getAxisValues( axisName)).all() # table not conform if soltab.getValues(retAxesVals=False, weight=False).shape != soltabsub.getValues( retAxesVals=False, weight=False).shape: hasMissingAxes = True else: hasMissingAxes = False if soltabsub.getType() == 'clock' or soltabsub.getType( ) == 'tec' or soltabsub.getType() == 'tec3rd' or soltabsub.getType( ) == 'rotationmeasure' or hasMissingAxes: freq = soltab.getAxisValues('freq') vals = soltab.getValues(retAxesVals=False, weight=False) weights = soltab.getValues(retAxesVals=False, weight=True) #print 'vals', vals.shape # valsSub doesn't have freq valsSub = soltabsub.getValues(retAxesVals=False, weight=False) weightsSub = soltabsub.getValues(retAxesVals=False, weight=True) #print 'valsSub', valsSub.shape # add missing axes and move it to the last position expand = [ soltab.getAxisLen(ax) for ax in soltab.getAxesNames() if ax not in soltabsub.getAxesNames() ] #print "expand:", expand valsSub = np.resize(valsSub, expand + list(valsSub.shape)) weightsSub = np.resize(weightsSub, expand + list(weightsSub.shape)) #print 'valsSub missing axes', valsSub.shape # reorder axes to match soltab names = [ ax for ax in soltab.getAxesNames() if ax not in soltabsub.getAxesNames() ] + soltabsub.getAxesNames() #print names, soltab.getAxesNames() valsSub = reorderAxes(valsSub, names, soltab.getAxesNames()) weightsSub = reorderAxes(weightsSub, names, soltab.getAxesNames()) weights[weightsSub == 0] = 0 # propagate flags #print 'valsSub reorder', valsSub.shape # put freq axis at the end idxFreq = soltab.getAxesNames().index('freq') vals = np.swapaxes(vals, idxFreq, len(vals.shape) - 1) valsSub = np.swapaxes(valsSub, idxFreq, len(valsSub.shape) - 1) #print 'vals reshaped', valsSub.shape # a multiplication will go along the last axis of the array if soltabsub.getType() == 'clock': vals -= 2. * np.pi * valsSub * freq elif soltabsub.getType() == 'tec': vals -= -8.44797245e9 * valsSub / freq elif soltabsub.getType() == 'tec3rd': vals -= -1.e21 * valsSub / np.power(freq, 3) elif soltabsub.getType() == 'rotationmeasure': # put pol axis at the beginning idxPol = soltab.getAxesNames().index('pol') if idxPol == len(vals.shape) - 1: idxPol = idxFreq # maybe freq swapped with pol vals = np.swapaxes(vals, idxPol, 0) valsSub = np.swapaxes(valsSub, idxPol, 0) #print 'vals reshaped 2', valsSub.shape wav = 2.99792458e8 / freq ph = wav * wav * valsSub #if coord['pol'] == 'XX' or coord['pol'] == 'RR': pols = soltab.getAxisValues('pol') assert len(pols) == 2 # full jons not supported if (pols[0] == 'XX' and pols[1] == 'YY') or \ (pols[0] == 'RR' and pols[1] == 'LL'): vals[0] -= ph[0] vals[1] += ph[1] else: vals[0] += ph[0] vals[1] -= ph[1] vals = np.swapaxes(vals, 0, idxPol) else: if ratio: vals = (vals - valsSub) / valsSub else: vals -= valsSub # move freq axis back vals = np.swapaxes(vals, len(vals.shape) - 1, idxFreq) soltab.setValues(vals) soltab.setValues(weights, weight=True) else: if ratio: soltab.setValues((soltab.getValues(retAxesVals=False) - soltabsub.getValues(retAxesVals=False)) / soltabsub.getValues(retAxesVals=False)) else: soltab.setValues( soltab.getValues(retAxesVals=False) - soltabsub.getValues(retAxesVals=False)) weight = soltab.getValues(retAxesVals=False, weight=True) weight[soltabsub.getValues(retAxesVals=False, weight=True) == 0] = 0 soltab.setValues(weight, weight=True) soltab.addHistory('RESIDUALS by subtracting tables ' + ' '.join(soltabsToSub)) return 0
def run(soltab, refAnt='', soltabError='', ncpu=0): """ Remove jumps from TEC solutions. WEIGHT: uses the errors. Parameters ---------- soltabError : str, optional The table name with solution errors. By default it has the same name of soltab with "error" in place of "tec". refAnt : str, optional Reference antenna for phases. By default None. """ import scipy.ndimage.filters import numpy as np from scipy.optimize import minimize from scipy.interpolate import griddata import scipy.cluster.vq as vq def getPhaseWrapBase(freqs): """ freqs: frequency grid of the data return the step size from a local minima (2pi phase wrap) to the others [0]: TEC, [1]: clock """ freqs = np.array(freqs) nF = freqs.shape[0] A = np.zeros((nF, 2), dtype=np.float) A[:, 1] = freqs * 2 * np.pi * 1e-9 A[:, 0] = -8.44797245e9 / freqs steps = np.dot(np.dot(np.linalg.inv(np.dot(A.T, A)), A.T), 2 * np.pi * np.ones((nF, ), dtype=np.float)) return steps if soltab.getType() != 'tec': logging.error('TECJUMP works only on tec solutions.') return 1 if soltabError == '': soltabError = soltab.name.replace('tec', 'error') solset = soltab.getSolset() soltab_e = solset.getSoltab(soltabError) try: seltab_e = solset.getSoltab(soltabError) except: logging.error('Cannot fine error solution table %s.' % soltabError) return 1 vals_e_all = soltab_e.getValues(retAxesVals=False, weight=False) logging.info("Removing TEC jumps from soltab: " + soltab.name) ants = soltab.getAxisValues('ant') if refAnt != '' and refAnt != 'closest' and not refAnt in soltab.getAxisValues( 'ant', ignoreSelection=True): logging.warning('Reference antenna ' + refAnt + ' not found. Using: ' + ants[1]) refAnt = ants[0] if refAnt == '': refAnt = None # Get the theoretical tec jump tec_jump_theory = abs(getPhaseWrapBase([42.308e6, 42.308e6 + 23828e6])[0]) # Find the average jump on all antennas by averaging all jumps found to be withing 0.5 and 2 times the tec_jump_theory vals = soltab.getValues(retAxesVals=False, refAnt=refAnt) timeAxis = soltab.getAxesNames().index('time') vals = np.swapaxes(vals, 0, timeAxis) vals = vals[1, ...] - vals[:-1, ...] vals = vals[(vals > tec_jump_theory * 1) & (vals < tec_jump_theory * 1.5)] if len(vals) == 0: logging.info('TEC jump - theoretical: %.5f TECU - NO JUMP FOUND' % (tec_jump_theory)) return 0 tec_jump = np.nanmedian(vals) logging.info('TEC jump - theoretical: %.5f TECU - estimated: %.5f TECU' % (tec_jump_theory, tec_jump)) mpm = multiprocManager(ncpu, _run_antenna) for vals, weights, coord, selection in soltab.getValuesIter( returnAxes='time', weight=True, refAnt=refAnt): # skip all flagged if (weights == 0).all(): continue # skip reference if (vals[(weights != 0)] == 0).all(): continue logging.info('Working on ant: %s' % (coord['ant'])) # 1d linear interp to fill flagged data vals[np.where(weights == 0)] = np.interp( np.where(weights == 0)[0], np.where(weights != 0)[0], vals[np.where(weights != 0)]) vals_init = np.copy(vals) # backup for final check vals_e = np.squeeze(vals_e_all[selection]) vals_e[np.where(weights == 0)] = 1. mpm.put([ vals, vals_e, vals_init, weights, selection, tec_jump, coord['ant'] ]) mpm.wait() for (vals, weights, selection) in mpm.get(): # set back to 0 the values for flagged data vals[weights == 0] = 0 soltab.setValues(vals, selection) soltab.addHistory('TECJUMP') return 0
def run(soltab, chanWidth='', outSoltabName='bandpass', BadSBList='', interpolate=True, removeTimeAxis=True, autoFlag=False, nSigma=5.0, maxFlaggedFraction=0.5, maxStddev=0.01, ncpu=0): """ This operation for LoSoTo implements the Prefactor bandpass operation WEIGHT: flag-only compliant, no need for weight Parameters ---------- chanWidth : str or float, optional the width of each channel in the data from which solutions were obtained. Can be either a string like "48kHz" or a float in Hz. If interpolate = True, chanWidth must be specified BadSBList : str, optional a list of bad subbands that will be flagged outSoltabName : str, optional Name of the output bandpass soltab. An existing soltab with this name will be overwritten interpolate : bool, optional If True, interpolate to a regular frequency grid and then smooth, ignoring bad subbands. If False, neither interpolation nor smoothing is done and the output frequency grid is the same as the input one. If interpolate = True, chanWidth must be specified removeTimeAxis : bool, optional If True, the time axis of the output bandpass soltab is removed by doing a median over time. If False, the output time grid is the same as the input one autoFlag : bool, optional If True, automatically flag bad frequencies and stations nSigma : float, optional Number of sigma for autoFlagging. Amplitudes outside of nSigma*stddev are flagged maxFlaggedFraction : float, optional Maximum allowable fraction of flagged frequencies for autoFlagging. Stations with higher fractions will be completely flagged maxStddev : float, optional Maximum allowable standard deviation for autoFlagging ncpu : int, optional Number of CPUs to use during autoFlagging (0 = all) """ import numpy as np import scipy import scipy.ndimage logging.info("Running prefactor_bandpass on: " + soltab.name) solset = soltab.getSolset() solType = soltab.getType() if solType != 'amplitude': logging.warning("Soltab type of " + soltab.name + " is: " + solType + " should be amplitude. Ignoring.") return 1 if soltab.name == outSoltabName and (removeTimeAxis or interpolate): logging.error( "If removeTimeAxis = True or interpolate = True, outSoltabName must specify a new soltab." ) raise ValueError( "If removeTimeAxis = True or interpolate = True, outSoltabName must specify a new soltab." ) if BadSBList == '': bad_sblist = [] else: bad_sblist = [int(SB) for SB in BadSBList.strip('\"\'').split(';')] for vals, weights, coord, selection in soltab.getValuesIter( returnAxes=['pol', 'ant', 'freq', 'time'], weight=True): vals = reorderAxes(vals, soltab.getAxesNames(), ['time', 'ant', 'freq', 'pol']) weights = reorderAxes(weights, soltab.getAxesNames(), ['time', 'ant', 'freq', 'pol']) pass amplitude_arraytmp = vals # axes are [time, ant, freq, pol] weights_arraytmp = weights # axes are [time, ant, freq, pol] flagged = np.where(amplitude_arraytmp == 1.0) weights_arraytmp[flagged] = 0.0 nfreqs = len(soltab.freq[:]) ntimes = len(soltab.time[:]) nants = len(soltab.ant[:]) subbandHz = 195.3125e3 if interpolate: if chanWidth == '': logging.error( "If interpolate = True, chanWidth must be specified.") raise ValueError( "If interpolate = True, chanWidth must be specified.") if type(chanWidth) is str: letters = [1 for s in chanWidth[::-1] if s.isalpha()] indx = len(chanWidth) - sum(letters) unit = chanWidth[indx:] if unit.strip().lower() == 'hz': conversion = 1.0 elif unit.strip().lower() == 'khz': conversion = 1e3 elif unit.strip().lower() == 'mhz': conversion = 1e6 else: logging.error("The unit on chanWidth was not understood.") raise ValueError("The unit on chanWidth was not understood.") chanWidthHz = float(chanWidth[:indx]) * conversion else: chanWidthHz = chanWidth offsetHz = subbandHz / 2.0 - 0.5 * chanWidthHz freqmin = np.min( soltab.freq[:]) + offsetHz # central frequency of first subband freqmax = np.max( soltab.freq[:]) + offsetHz # central frequency of last subband SBgrid = np.floor( (soltab.freq[:] - np.min(soltab.freq[:])) / subbandHz) freqs_new = np.arange(freqmin, freqmax + 100e3, subbandHz) amps_array_flagged = np.zeros((nants, ntimes, len(freqs_new), 2), dtype='float') amps_array = np.zeros((nants, ntimes, len(freqs_new), 2), dtype='float') weights_array = np.ones((nants, ntimes, len(freqs_new), 2), dtype='float') logging.info("Have " + str(max(SBgrid)) + " subbands.") if len(freqs_new) < 20: logging.error( "Frequency span is less than 20 subbands! The filtering will not work!" ) logging.error( "Please run the calibrator pipeline on the full calibrator bandwidth." ) raise ValueError( "Frequency span is less than 20 subbands! Amplitude filtering will not work!" ) # make a mapping of new frequencies to old ones freq_mapping = {} for fn in freqs_new: ind = np.where( np.logical_and(soltab.freq < fn + subbandHz / 2.0, soltab.freq >= fn - subbandHz / 2.0)) freq_mapping['{}'.format(fn)] = ind # remove bad subbands specified by user for bad_sb in bad_sblist: logging.info('Removing user-specified subband: ' + str(bad_sb)) weights_arraytmp[:, :, bad_sb, :] = 0.0 # remove bad solutions relative to the model bandpass if autoFlag: if ncpu == 0: import multiprocessing ncpu = multiprocessing.cpu_count() mpm = multiprocManager(ncpu, _flag_amplitudes) for s in range(nants): mpm.put([ soltab.freq[:], amplitude_arraytmp[:, s, :, :], weights_arraytmp[:, s, :, :], nSigma, maxFlaggedFraction, maxStddev, False, s ]) mpm.wait() for (s, w) in mpm.get(): weights_arraytmp[:, s, :, :] = w # Now interpolate over flagged values and smooth over frequency and time axes if interpolate: for antenna_id in range(len(soltab.ant[:])): for time in range(len(soltab.time[:])): amp_xx_tmp = np.copy(amplitude_arraytmp[time, antenna_id, :, 0]) amp_yy_tmp = np.copy(amplitude_arraytmp[time, antenna_id, :, 1]) freq_tmp = soltab.freq[:] assert len(amp_xx_tmp[:]) == len(freq_tmp[:]) mask_xx = np.not_equal( weights_arraytmp[time, antenna_id, :, 0], 0.0) if np.sum(mask_xx) > 2: amps_xx_tointer = amp_xx_tmp[mask_xx] freq_xx_tointer = freq_tmp[mask_xx] amps_array_flagged[antenna_id, time, :, 0] = np.interp(freqs_new, freq_xx_tointer, amps_xx_tointer) elif time > 0: amps_array_flagged[antenna_id, time, :, 0] = amps_array_flagged[antenna_id, (time - 1), :, 0] mask_yy = np.not_equal( weights_arraytmp[time, antenna_id, :, 1], 0.0) if np.sum(mask_yy) > 2: amps_yy_tointer = amp_yy_tmp[mask_yy] freq_yy_tointer = freq_tmp[mask_yy] amps_array_flagged[antenna_id, time, :, 1] = np.interp(freqs_new, freq_yy_tointer, amps_yy_tointer) elif time > 0: amps_array_flagged[antenna_id, time, :, 1] = amps_array_flagged[antenna_id, (time - 1), :, 1] ampsoutfile = open('calibrator_amplitude_array.txt', 'w') ampsoutfile.write( '# Antenna name, Antenna ID, subband, XXamp, YYamp, frequency\n') for antenna_id in range(len(soltab.ant[:])): if np.all(weights_arraytmp[:, antenna_id, :, :] == 0.0): weights_array[antenna_id, :, :, :] = 0.0 else: amp_xx = np.copy(amps_array_flagged[antenna_id, :, :, 0]) amp_yy = np.copy(amps_array_flagged[antenna_id, :, :, 1]) amp_xx = scipy.ndimage.filters.median_filter(amp_xx, (3, 3)) amp_xx = scipy.ndimage.filters.median_filter(amp_xx, (7, 1)) amp_yy = scipy.ndimage.filters.median_filter(amp_yy, (3, 3)) amp_yy = scipy.ndimage.filters.median_filter(amp_yy, (7, 1)) for i in range(len(freqs_new)): ampsoutfile.write( '%s %s %s %s %s %s\n' % (soltab.ant[antenna_id], antenna_id, i, np.median(amp_xx[:, i], axis=0), np.median(amp_yy[:, i], axis=0), freqs_new[i])) for time in range(len(soltab.time[:])): amps_array[antenna_id, time, :, 0] = np.copy( _savitzky_golay(amp_xx[time, :], 17, 2)) amps_array[antenna_id, time, :, 1] = np.copy( _savitzky_golay(amp_yy[time, :], 17, 2)) for i in range(len(freqs_new)): amps_array[antenna_id, :, i, 0] = np.median(amps_array[antenna_id, :, i, 0]) amps_array[antenna_id, :, i, 1] = np.median(amps_array[antenna_id, :, i, 1]) ind = freq_mapping['{}'.format(freqs_new[i])] for p in range(2): # If half or more of original frequencies are flagged, flag the # output frequency as well nflagged = len( np.where(weights_arraytmp[:, antenna_id, ind, p] == 0.0)[0]) ntot = weights_arraytmp.shape[0] * len(ind[0]) if ntot > 0: if float(nflagged) / float(ntot) >= 0.5: weights_array[antenna_id, :, i, p] = 0.0 amps_array = amps_array.swapaxes(0, 1) weights_array = weights_array.swapaxes(0, 1) else: amps_array = amplitude_arraytmp weights_array = weights_arraytmp freqs_new = soltab.freq[:] # delete existing bandpass soltab if needed and write solutions if soltab.name != outSoltabName: try: new_soltab = solset.getSoltab(outSoltabName) new_soltab.delete() except: pass if removeTimeAxis: # Write bandpass, taking median over the time axis new_soltab = solset.makeSoltab( soltype='amplitude', soltabName=outSoltabName, axesNames=['ant', 'freq', 'pol'], axesVals=[soltab.ant, freqs_new, ['XX', 'YY']], vals=np.median(amps_array, axis=0), weights=np.median(weights_array, axis=0)) new_soltab.addHistory( 'CREATE (by PREFACTOR_BANDPASS operation) with BadSBList = {0}, ' 'interpolate={1}, removeTimeAxis={2}, autoFlag={3}, nSigma={4}, ' 'maxFlaggedFraction={5}, maxStddev={6}'.format( BadSBList, interpolate, removeTimeAxis, autoFlag, nSigma, maxFlaggedFraction, maxStddev)) else: # Write bandpass, preserving the time axis if soltab.name == outSoltabName: soltab.setValues(amps_array) soltab.setValues(weights_array, weight=True) soltab.addHistory( 'BANDPASS processed with BadSBList = {0}, interpolate={1}, ' 'removeTimeAxis={2}, autoFlag={3}, nSigma={4}, ' 'maxFlaggedFraction={5}, maxStddev={6}'.format( BadSBList, interpolate, removeTimeAxis, autoFlag, nSigma, maxFlaggedFraction, maxStddev)) else: new_soltab = solset.makeSoltab( soltype='amplitude', soltabName=outSoltabName, axesNames=['time', 'ant', 'freq', 'pol'], axesVals=[soltab.time, soltab.ant, freqs_new, ['XX', 'YY']], vals=amps_array, weights=weights_array) new_soltab.addHistory( 'CREATE (by PREFACTOR_BANDPASS operation) with BadSBList = {0}, ' 'interpolate={1}, removeTimeAxis={2}, autoFlag={3}, nSigma={4}, ' 'maxFlaggedFraction={5}, maxStddev={6}'.format( BadSBList, interpolate, removeTimeAxis, autoFlag, nSigma, maxFlaggedFraction, maxStddev)) return 0
def run(soltab, tecsoltabOut='tec000', clocksoltabOut='clock000', offsetsoltabOut='phase_offset000', tec3rdsoltabOut='tec3rd000', flagBadChannels=True, flagCut=5., chi2cut=3000., combinePol=False, removePhaseWraps=True, fit3rdorder=False, circular=False, reverse=False, invertOffset=False, nproc=10): """ Separate phase solutions into Clock and TEC. The Clock and TEC values are stored in the specified output soltab with type 'clock', 'tec', 'tec3rd'. Parameters ---------- flagBadChannels : bool, optional Detect and remove bad channel before fitting, by default True. flagCut : float, optional chi2cut : float, optional combinePol : bool, optional Find a combined polarization solution, by default False. removePhaseWraps : bool, optional Detect and remove phase wraps, by default True. fit3rdorder : bool, optional Fit a 3rd order ionospheric ocmponent (usefult <40 MHz). By default False. circular : bool, optional Assume circular polarization with FR not removed. By default False. reverse : bool, optional Reverse the time axis. By default False. invertOffset : bool, optional Invert (reverse the sign of) the phase offsets. By default False. Set to True if you want to use them with the residuals operation. """ import numpy as np from ._fitClockTEC import doFit logging.info("Clock/TEC separation on soltab: " + soltab.name) # some checks solType = soltab.getType() if solType != 'phase': logging.warning("Soltab type of " + soltab.name + " is: " + solType + " should be phase. Ignoring.") return 1 # Collect station properties solset = soltab.getSolset() station_dict = solset.getAnt() stations = soltab.getAxisValues('ant') station_positions = np.zeros((len(stations), 3), dtype=np.float) for i, station_name in enumerate(stations): station_positions[i, 0] = station_dict[station_name][0] station_positions[i, 1] = station_dict[station_name][1] station_positions[i, 2] = station_dict[station_name][2] returnAxes = ['ant', 'freq', 'pol', 'time'] for vals, flags, coord, selection in soltab.getValuesIter( returnAxes=returnAxes, weight=True): if len(coord['ant']) < 2: logging.error( 'Clock/TEC separation needs at least 2 antennas selected.') return 1 if len(coord['freq']) < 10: logging.error( 'Clock/TEC separation needs at least 10 frequency channels, preferably distributed over a wide range' ) return 1 freqs = coord['freq'] stations = coord['ant'] times = coord['time'] # get axes index axes = [i for i in soltab.getAxesNames() if i in returnAxes] # reverse time axes if reverse: vals = np.swapaxes( np.swapaxes(vals, 0, axes.index('time'))[::-1], 0, axes.index('time')) flags = np.swapaxes( np.swapaxes(flags, 0, axes.index('time'))[::-1], 0, axes.index('time')) result=doFit(vals,flags==0,freqs,stations,station_positions,axes,\ flagBadChannels=flagBadChannels,flagcut=flagCut,chi2cut=chi2cut,combine_pol=combinePol,removePhaseWraps=removePhaseWraps,fit3rdorder=fit3rdorder,circular=circular,n_proc=nproc) if fit3rdorder: clock, tec, offset, tec3rd = result if reverse: clock = clock[::-1, :] tec = tec[::-1, :] tec3rd = tec3rd[::-1, :] else: clock, tec, offset = result if reverse: clock = clock[::-1, :] tec = tec[::-1, :] if invertOffset: offset *= -1.0 weights = tec > -5 tec[np.logical_not(weights)] = 0 clock[np.logical_not(weights)] = 0 weights = np.float16(weights) if combinePol or not 'pol' in soltab.getAxesNames(): tf_st = solset.makeSoltab('tec', soltabName=tecsoltabOut, axesNames=['time', 'ant'], axesVals=[times, stations], vals=tec[:, :, 0], weights=weights[:, :, 0]) tf_st.addHistory('CREATE (by CLOCKTECFIT operation)') tf_st = solset.makeSoltab('clock', soltabName=clocksoltabOut, axesNames=['time', 'ant'], axesVals=[times, stations], vals=clock[:, :, 0] * 1e-9, weights=weights[:, :, 0]) tf_st.addHistory('CREATE (by CLOCKTECFIT operation)') tf_st = solset.makeSoltab('phase', soltabName=offsetsoltabOut, axesNames=['ant'], axesVals=[stations], vals=offset[:, 0], weights=np.ones_like(offset[:, 0], dtype=np.float16)) tf_st.addHistory('CREATE (by CLOCKTECFIT operation)') if fit3rdorder: tf_st = solset.makeSoltab('tec3rd', soltabName=tec3rdsoltabOut, axesNames=['time', 'ant'], axesVals=[times, stations], vals=tec3rd[:, :, 0], weights=weights[:, :, 0]) else: tf_st = solset.makeSoltab('tec', soltabName=tecsoltabOut, axesNames=['time', 'ant', 'pol'], axesVals=[times, stations, ['XX', 'YY']], vals=tec, weights=weights) tf_st.addHistory('CREATE (by CLOCKTECFIT operation)') tf_st = solset.makeSoltab('clock', soltabName=clocksoltabOut, axesNames=['time', 'ant', 'pol'], axesVals=[times, stations, ['XX', 'YY']], vals=clock * 1e-9, weights=weights) tf_st.addHistory('CREATE (by CLOCKTECFIT operation)') tf_st = solset.makeSoltab('phase', soltabName=offsetsoltabOut, axesNames=['ant', 'pol'], axesVals=[stations, ['XX', 'YY']], vals=offset, weights=np.ones_like(offset, dtype=np.float16)) tf_st.addHistory('CREATE (by CLOCKTECFIT operation)') if fit3rdorder: tf_st = solset.makeSoltab( 'tec3rd', soltabName=tec3rdsoltabOut, axesNames=['time', 'ant', 'pol'], axesVals=[times, stations, ['XX', 'YY']], vals=tec3rd, weights=weights) return 0
def run(soltab, mode, maxFlaggedFraction=0.5, nSigma=5.0, maxStddev=None, ampRange=None, telescope='lofar', skipInternational=False, refAnt='', soltabExport='', ncpu=0): """ This operation for LoSoTo implements a station-flagging procedure. Flags are time-independent. WEIGHT: compliant Parameters ---------- mode: str Fitting algorithm: bandpass or resid. Bandpass mode clips amplitudes relative to a model bandpass (only LOFAR is currently supported). Resid mode clips residual phases or log(amplitudes). maxFlaggedFraction : float, optional This sets the maximum allowable fraction of flagged solutions above which the entire station is flagged. nSigma : float, optional This sets the number of standard deviations considered when outlier clipping is done. maxStddev : float, optional Maximum allowable standard deviation when outlier clipping is done. For phases, this should value should be in radians, for amplitudes in log(amp). If None (or negative), a value of 0.1 rad is used for phases and 0.01 for amplitudes. ampRange : array, optional 2-element array of the median amplitude level to be acceptable, ampRange[0]: lower limit, ampRange[1]: upper limit. If None or [0, 0], a reasonable range for typical observations is used. telescope : str, optional Specifies the telescope if mode = 'bandpass'. skipInternational : str, optional If True, skip flagging of international LOFAR stations (only used if telescope = 'lofar') refAnt : str, optional If mode = resid, this sets the reference antenna for phase solutions, by default None. soltabExport : str, optional Soltab to export station flags to. Note: exported flags are not time- or frequency-dependent. ncpu : int, optional Number of cpu to use, by default all available. """ logging.info("Flagging on soltab: " + soltab.name) # input check if refAnt == '': refAnt = None if soltabExport == '': soltabExport = None if mode is None or mode.lower() not in ['bandpass', 'resid']: logging.error('Mode must be one of bandpass or resid') return 1 solType = soltab.getType() if maxStddev is None or maxStddev <= 0.0: if solType == 'phase': maxStddev = 0.1 # in radians else: maxStddev = 0.01 # in log10(amp) # Axis order must be [time, ant, freq, pol], so reorder if necessary axis_names = soltab.getAxesNames() if ('freq' not in axis_names or 'pol' not in axis_names or 'time' not in axis_names or 'ant' not in axis_names): logging.error("Currently, flagstation requires the following axes: " "freq, pol, time, and ant.") return 1 freq_ind = axis_names.index('freq') pol_ind = axis_names.index('pol') time_ind = axis_names.index('time') ant_ind = axis_names.index('ant') if 'dir' in axis_names: dir_ind = axis_names.index('dir') vals_arraytmp = soltab.val[:].transpose( [time_ind, ant_ind, freq_ind, pol_ind, dir_ind]) weights_arraytmp = soltab.weight[:].transpose( [time_ind, ant_ind, freq_ind, pol_ind, dir_ind]) else: vals_arraytmp = soltab.val[:].transpose( [time_ind, ant_ind, freq_ind, pol_ind]) weights_arraytmp = soltab.weight[:].transpose( [time_ind, ant_ind, freq_ind, pol_ind]) # Check for NaN solutions and flag flagged = np.where(np.isnan(vals_arraytmp)) weights_arraytmp[flagged] = 0.0 if mode == 'bandpass': if solType != 'amplitude': logging.error( "Soltab must be of type amplitude for bandpass mode.") return 1 # Fill the queue if 'dir' in axis_names: for d, dirname in enumerate(soltab.dir): mpm = multiprocManager(ncpu, _flag_bandpass) for s in range(len(soltab.ant)): if ('CS' not in soltab.ant[s] and 'RS' not in soltab.ant[s] and skipInternational and telescope.lower() == 'lofar'): continue mpm.put([ soltab.freq[:], vals_arraytmp[:, s, :, :, d], weights_arraytmp[:, s, :, :, d], telescope, nSigma, ampRange, maxFlaggedFraction, maxStddev, False, soltab.ant[:], s ]) mpm.wait() for (s, w) in mpm.get(): weights_arraytmp[:, s, :, :, d] = w else: mpm = multiprocManager(ncpu, _flag_bandpass) for s in range(len(soltab.ant)): if ('CS' not in soltab.ant[s] and 'RS' not in soltab.ant[s] and skipInternational and telescope.lower() == 'lofar'): continue mpm.put([ soltab.freq[:], vals_arraytmp[:, s, :, :], weights_arraytmp[:, s, :, :], telescope, nSigma, ampRange, maxFlaggedFraction, maxStddev, False, soltab.ant[:], s ]) mpm.wait() for (s, w) in mpm.get(): weights_arraytmp[:, s, :, :] = w # Make sure that fully flagged stations have all pols flagged for s in range(len(soltab.ant)): for p in range(len(soltab.pol)): if np.all(weights_arraytmp[:, s, :, p] == 0.0): weights_arraytmp[:, s, :, :] = 0.0 break # Write new weights if 'dir' in axis_names: weights_array = weights_arraytmp.transpose( [time_ind, ant_ind, freq_ind, pol_ind, dir_ind]) else: weights_array = weights_arraytmp.transpose( [time_ind, ant_ind, freq_ind, pol_ind]) soltab.setValues(weights_array, weight=True) soltab.addHistory( 'FLAGSTATION (mode=bandpass, telescope={0}, maxFlaggedFraction={1}, ' 'nSigma={2})'.format(telescope, maxFlaggedFraction, nSigma)) else: if solType not in ['phase', 'amplitude']: logging.error( "Soltab must be of type phase or amplitude for resid mode.") return 1 # Subtract reference phases if refAnt is not None: if solType != 'phase': logging.error( 'Reference possible only for phase solution tables. Ignoring referencing.' ) else: if refAnt == 'nearest': for i, antToRef in enumerate(soltab.getAxisValues('ant')): # get the closest antenna antDists = soltab.getSolset().getAntDist( antToRef) # this is a dict for badAnt in soltab._getFullyFlaggedAnts(): del antDists[badAnt] # remove bad ants reference = list(antDists.keys( ))[list(antDists.values()).index( sorted(antDists.values())[1] )] # get the second closest antenna (the first is itself) refInd = soltab.getAxisValues( 'ant', ignoreSelection=True).tolist().index(reference) if 'dir' in axis_names: vals_arrayref = vals_arraytmp[:, refInd, :, :, :].copy( ) else: vals_arrayref = vals_arraytmp[:, refInd, :, :].copy() if 'dir' in axis_names: vals_arraytmp[:, i, :, :, :] -= vals_arrayref else: vals_arraytmp[:, i, :, :] -= vals_arrayref else: ants = soltab.getAxisValues('ant') if refAnt not in ants: logging.warning('Reference antenna ' + refAnt + ' not found. Using: ' + ants[0]) refAnt = ants[0] refInd = ants.tolist().index(refAnt) if 'dir' in axis_names: vals_arrayref = vals_arraytmp[:, refInd, :, :, :].copy() else: vals_arrayref = vals_arraytmp[:, refInd, :, :].copy() for i in range(len(soltab.ant)): if 'dir' in axis_names: vals_arraytmp[:, i, :, :, :] -= vals_arrayref else: vals_arraytmp[:, i, :, :] -= vals_arrayref # Fill the queue if 'dir' in axis_names: for d, dirname in enumerate(soltab.dir): mpm = multiprocManager(ncpu, _flag_resid) for s in range(len(soltab.ant)): mpm.put([ vals_arraytmp[:, s, :, :, d], weights_arraytmp[:, s, :, :, d], solType, nSigma, maxFlaggedFraction, maxStddev, soltab.ant[:], s ]) mpm.wait() for (s, w) in mpm.get(): weights_arraytmp[:, s, :, :, d] = w else: mpm = multiprocManager(ncpu, _flag_resid) for s in range(len(soltab.ant)): mpm.put([ vals_arraytmp[:, s, :, :], weights_arraytmp[:, s, :, :], solType, nSigma, maxFlaggedFraction, maxStddev, soltab.ant[:], s ]) mpm.wait() for (s, w) in mpm.get(): weights_arraytmp[:, s, :, :] = w # Make sure that fully flagged stations have all pols flagged for s in range(len(soltab.ant)): for p in range(len(soltab.pol)): if np.all(weights_arraytmp[:, s, :, p] == 0.0): weights_arraytmp[:, s, :, :] = 0.0 break # Write new weights if 'dir' in axis_names: weights_array = weights_arraytmp.transpose( [time_ind, ant_ind, freq_ind, pol_ind, dir_ind]) else: weights_array = weights_arraytmp.transpose( [time_ind, ant_ind, freq_ind, pol_ind]) soltab.setValues(weights_array, weight=True) soltab.addHistory('FLAGSTATION (mode=resid, maxFlaggedFraction={0}, ' 'nSigma={1})'.format(maxFlaggedFraction, nSigma)) if soltabExport is not None: # Transfer station flags to soltabExport solset = soltab.getSolset() soltabexp = solset.getSoltab(soltabExport) axis_namesexp = soltabexp.getAxesNames() for stat in soltabexp.ant: if stat in soltab.ant: s = soltab.ant[:].tolist().index(stat) if 'pol' in axis_namesexp: for pol in soltabexp.pol: if pol in soltab.pol: soltabexp.setSelection(ant=stat, pol=pol) p = soltab.pol[:].tolist().index(pol) if np.all(weights_arraytmp[:, s, :, p] == 0): soltabexp.setValues(np.zeros( soltabexp.weight.shape), weight=True) else: soltabexp.setSelection(ant=stat) if np.all(weights_arraytmp[:, s, :, :] == 0): soltabexp.setValues(np.zeros(soltabexp.weight.shape), weight=True) soltabexp.addHistory('WEIGHT imported by FLAGSTATION from ' + soltab.name + '.') return 0
def run( soltab, soltabOut='phasediff', maxResidual=1., fitOffset=False, average=False, replace=False, minFreq=0, refAnt='' ): """ Estimate polarization misalignment as delay. Parameters ---------- soltabOut : str, optional output table name (same solset), by deault "phasediff". maxResidual : float, optional Maximum acceptable rms of the residuals in radians before flagging, by default 1. If 0: No check. fitOffset : bool, optional Assume that together with a delay each station has also a differential phase offset (important for old LBA observations). By default False. average : bool, optional Mean-average in time the resulting delays/offset. By default False. replace : bool, optional replace using smoothed value instead of flag bad data? Smooth must be active. By default, False. minFreq : float, optional minimum frequency [Hz] to use in estimating the PA. By default, 0 (all freqs). refAnt : str, optional Reference antenna, by default the first. """ import numpy as np import scipy.optimize from scipy import stats from scipy.ndimage import generic_filter logging.info("Finding polarization align for soltab: "+soltab.name) delaycomplex = lambda d, freq, y: abs(np.cos(d[0]*freq) - np.cos(y)) + abs(np.sin(d[0]*freq) - np.sin(y)) #delaycomplex = lambda d, freq, y: abs(d[0]*freq - y) solType = soltab.getType() if solType != 'phase': logging.warning("Soltab type of "+soltab.name+" is of type "+solType+", should be phase. Ignoring.") return 1 if refAnt != '' and refAnt != 'closest' and not refAnt in soltab.getAxisValues('ant', ignoreSelection = True): logging.warning('Reference antenna '+refAnt+' not found. Using: '+soltab.getAxisValues('ant')[1]) refAnt = soltab.getAxisValues('ant')[1] if refAnt == '': refAnt = soltab.getAxisValues('ant')[1] # times and ants needs to be complete or selection is much slower times = soltab.getAxisValues('time') # create new table solset = soltab.getSolset() soltabout = solset.makeSoltab(soltype = soltab.getType(), soltabName = soltabOut, axesNames=soltab.getAxesNames(), axesVals=[soltab.getAxisValues(axisName) for axisName in soltab.getAxesNames()], vals=soltab.getValues(retAxesVals = False), weights=soltab.getValues(weight = True, retAxesVals = False)) soltabout.addHistory('Created by POLALIGN operation from %s.' % soltab.name) if 'XX' in soltab.getAxisValues('pol'): pol = 'XX' elif 'RR' in soltab.getAxisValues('pol'): pol = 'RR' else: logging.error('Cannot reference to known polarisation.') return 1 for vals, weights, coord, selection in soltab.getValuesIter(returnAxes=['freq','pol','time'], weight=True, refAnt=refAnt): # reorder axes vals = reorderAxes( vals, soltab.getAxesNames(), ['pol','freq','time'] ) weights = reorderAxes( weights, soltab.getAxesNames(), ['pol','freq','time'] ) if 'RR' in coord['pol'] and 'LL' in coord['pol']: coord1 = np.where(coord['pol'] == 'RR')[0][0] coord2 = np.where(coord['pol'] == 'LL')[0][0] elif 'XX' in coord['pol'] and 'YY' in coord['pol']: coord1 = np.where(coord['pol'] == 'XX')[0][0] coord2 = np.where(coord['pol'] == 'YY')[0][0] if (weights == 0.).all() == True: logging.warning('Skipping flagged antenna: '+coord['ant']) weights[:] = 0 else: fit_delays=[]; fit_offset=[]; fit_weights=[] for t, time in enumerate(times): # apply flags idx = ( (weights[coord1,:,t] != 0.) & (weights[coord2,:,t] != 0.) & (coord['freq'] > minFreq) ) freq = np.copy(coord['freq'])[idx] phase1 = vals[coord1,:,t][idx] phase2 = vals[coord2,:,t][idx] if len(freq) < 30: fit_weights.append(0.) fit_delays.append(0.) fit_offset.append(0.) logging.debug('Not enough unflagged point for the timeslot '+str(t)) continue # if more than 1/2 of chans are flagged if (len(idx) - len(freq))/float(len(idx)) > 1/2.: logging.debug('High number of filtered out data points for the timeslot %i: %i/%i' % (t, len(idx) - len(freq), len(idx)) ) phase_diff = phase1 - phase2 phase_diff = np.mod(phase_diff + np.pi, 2.*np.pi) - np.pi phase_diff = np.unwrap(phase_diff) A = np.vstack([freq, np.ones(len(freq))]).T fitresultdelay = np.linalg.lstsq(A, phase_diff.T)[0] # get the closest n*(2pi) to the intercept and refit with only 1 parameter if not fitOffset: numjumps = np.around(fitresultdelay[1]/(2*np.pi)) A = np.reshape(freq, (-1,1)) # no b phase_diff -= numjumps * 2 * np.pi fitresultdelay = np.linalg.lstsq(A, phase_diff.T)[0] fitresultdelay = [fitresultdelay[0],0.] # set offset to 0 to keep the rest of the script equal # fractional residual residual = np.mean(np.abs( fitresultdelay[0]*freq + fitresultdelay[1] - phase_diff )) fit_delays.append(fitresultdelay[0]) fit_offset.append(fitresultdelay[1]) if maxResidual == 0 or residual < maxResidual: fit_weights.append(1.) else: # high residual, flag logging.debug('Bad solution for ant: '+coord['ant']+' (time: '+str(t)+', residual: '+str(residual)+') -> ignoring.') fit_weights.append(0.) # Debug plot doplot = False #if doplot and t%100==0 and (coord['ant'] == 'RS310LBA' or coord['ant'] == 'CS301LBA'): if doplot and t%10==0 and (coord['ant'] == 'W04'): if not 'matplotlib' in sys.modules: import matplotlib as mpl mpl.rc('figure.subplot',left=0.05, bottom=0.05, right=0.95, top=0.95,wspace=0.22, hspace=0.22 ) mpl.use("Agg") import matplotlib.pyplot as plt fig = plt.figure() fig.subplots_adjust(wspace=0) ax = fig.add_subplot(111) # plot rm fit plotdelay = lambda delay, offset, freq: np.mod( delay*freq + offset + np.pi, 2.*np.pi) - np.pi ax.plot(freq, fitresultdelay[0]*freq + fitresultdelay[1], "-", color='purple', label=r'delay:%f$\nu$ (ns) + %f ' % (fitresultdelay[0]*1e9,fitresultdelay[1]) ) ax.plot(freq, np.mod(phase1 + np.pi, 2.*np.pi) - np.pi, 'ob' ) ax.plot(freq, np.mod(phase2 + np.pi, 2.*np.pi) - np.pi, 'og' ) #ax.plot(freq, np.mod(phase_diff + np.pi, 2.*np.pi) - np.pi, '.', color='purple' ) ax.plot(freq, phase_diff, '.', color='purple' ) residual = np.mod(plotdelay(fitresultdelay[0], fitresultdelay[1], freq)-phase_diff + np.pi,2.*np.pi)-np.pi ax.plot(freq, residual, '.', color='yellow') ax.set_xlabel('freq') ax.set_ylabel('phase') #ax.set_ylim(ymin=-np.pi, ymax=np.pi) logging.warning('Save pic: '+str(t)+'_'+coord['ant']+'.png') fig.legend(loc='upper left') plt.savefig(coord['ant']+'_'+str(t)+'.png', bbox_inches='tight') del fig # end cycle in time fit_weights = np.array(fit_weights) fit_delays = np.array(fit_delays) fit_offset = np.array(fit_offset) # avg in time if average: fit_delays_bkp = fit_delays[ fit_weights == 0 ] fit_offset_bkp = fit_offset[ fit_weights == 0 ] np.putmask(fit_delays, fit_weights == 0, np.nan) np.putmask(fit_offset, fit_weights == 0, np.nan) fit_delays[:] = np.nanmean(fit_delays) # angle mean fit_offset[:] = np.angle( np.nansum( np.exp(1j*fit_offset) ) / np.count_nonzero(~np.isnan(fit_offset)) ) if replace: fit_weights[ fit_weights == 0 ] = 1. fit_weights[ np.isnan(fit_delays) ] = 0. # all the size was flagged cannot estrapolate value else: fit_delays[ fit_weights == 0 ] = fit_delays_bkp fit_offset[ fit_weights == 0 ] = fit_offset_bkp logging.info('%s: average delay: %f ns (offset: %f)' % ( coord['ant'], np.mean(fit_delays)*1e9, np.mean(fit_offset))) for t, time in enumerate(times): #vals[:,:,t] = 0. #vals[coord1,:,t] = fit_delays[t]*np.array(coord['freq'])/2. vals[coord1,:,t] = 0 phase = np.mod(fit_delays[t]*coord['freq'] + fit_offset[t] + np.pi, 2.*np.pi) - np.pi vals[coord2,:,t] = -1.*phase#/2. #weights[:,:,t] = 0. weights[coord1,:,t] = fit_weights[t] weights[coord2,:,t] = fit_weights[t] # reorder axes back to the original order, needed for setValues vals = reorderAxes( vals, ['pol','freq','time'], [ax for ax in soltab.getAxesNames() if ax in ['pol','freq','time']] ) weights = reorderAxes( weights, ['pol','freq','time'], [ax for ax in soltab.getAxesNames() if ax in ['pol','freq','time']] ) soltabout.setSelection(**coord) soltabout.setValues( vals ) soltabout.setValues( weights, weight=True ) return 0
def run(soltab, outsoltab, axisToRegrid, newdelta, delta='', maxFlaggedWidth=0, log=False): """ This operation for LoSoTo implements regridding and linear interpolation of data for an axis. WEIGHT: compliant Parameters ---------- outsoltab: str Name of output soltab axisToRegrid : str Name of the axis for which regridding/interpolation will be done newdelta : float or str Fundamental width between samples after regridding. E.g., "100kHz" or "10s" delta : float or str, optional Fundamental width between samples in axisToRegrid. E.g., "100kHz" or "10s". If "", it is calculated from the axisToRegrid values maxFlaggedWidth : int, optional Maximum allowable width in number of samples (after regridding) above which interpolated values are flagged (e.g., maxFlaggedWidth = 5 would allow gaps of 5 samples or less to be interpolated across but gaps of 6 or more would be flagged) log : bool, optional Interpolation is done in log10 space, by default False """ import scipy.ndimage as nd # Check inputs if axisToRegrid not in soltab.getAxesNames(): logging.error('Axis \"' + axisToRegrid + '\" not found.') return 1 if axisToRegrid not in ['freq', 'time']: logging.error('Axis \"' + axisToRegrid + '\" must be either time or freq.') return 1 newdelta = _convert_strval(newdelta) if delta == "": deltas = soltab.getAxisValues(axisToRegrid)[1:] - soltab.getAxisValues( axisToRegrid)[:-1] delta = np.min(deltas) logging.info('Using {} for delta'.format(delta)) else: delta = _convert_strval(delta) if soltab.getType() == 'amplitude' and not log: logging.warning( 'Amplitude solution tab detected and log=False. Amplitude solution tables should be treated in log space.' ) # Regrid axis axisind = soltab.getAxesNames().index(axisToRegrid) orig_axisvals = soltab.getAxisValues(axisToRegrid) new_axisvals = _regrid_axis(orig_axisvals, delta, newdelta) orig_shape = soltab.val.shape new_shape = list(orig_shape) new_shape[axisind] = len(new_axisvals) new_vals = np.zeros(new_shape, dtype='float') new_weights = np.zeros(new_shape, dtype='float') for vals, weights, coord, selection in soltab.getValuesIter( returnAxes=[axisToRegrid], weight=True): flagged = np.logical_or(np.equal(weights, 0.0), np.isnan(vals)) weights[flagged] = 0.0 unflagged = np.not_equal(weights, 0.0) if np.sum(unflagged) > 2: # If there are at least two unflagged points, interpolate with mask if log: vals = np.log10(vals) new_vals[selection] = np.interp(new_axisvals, orig_axisvals[unflagged], vals[unflagged], left=np.nan, right=np.nan) # For the weights, interpolate without the mask new_weights[selection] = np.round( np.interp(new_axisvals, orig_axisvals, weights, left=np.nan, right=np.nan)) # Check for flagged gaps if maxFlaggedWidth > 1: inv_weights = new_weights[selection].astype(bool).squeeze() rank = len(inv_weights.shape) connectivity = nd.generate_binary_structure(rank, rank) mask_labels, count = nd.label(~inv_weights, connectivity) for i in range(count): ind = np.where(mask_labels == i + 1) gapsize = len(ind[0]) if gapsize <= maxFlaggedWidth: # Unflag narrow gaps selection[axisind] = ind[0] new_weights[selection] = 1.0 # Write new soltab solset = soltab.getSolset() axesVals = [] for axisName in soltab.getAxesNames(): if axisName == axisToRegrid: axesVals.append(new_axisvals) else: axesVals.append(soltab.getAxisValues(axisName)) if log: new_vals = 10**new_vals s = solset.makeSoltab(soltab.getType(), outsoltab, axesNames=soltab.getAxesNames(), axesVals=axesVals, vals=new_vals, weights=new_weights) s.addHistory('CREATE by INTERPOLATE operation from ' + soltab.name + '.') return 0
def run(soltab, outSoltab='tecscreen', height=200.0e3, order=12, beta=5.0 / 3.0, ncpu=0): """ Fits a screen to TEC + scalaraphase values. The results of the fit are stored in the soltab parent solset in "outSoltab" and the residual phases (actual-screen) are stored in "outsoltabresid". These values are the screen phase values per station per pierce point per solution interval. The pierce point locations are stored in an auxiliary array in the output soltabs. Screens can be plotted with the PLOTSCREEN operation. Parameters ---------- soltab: solution table Soltab containing phase solutions outSoltab: str, optional Name of output soltab height : float, optional Height in m of screen order : int, optional Order of screen (i.e., number of KL base vectors to keep). beta: float, optional Power-law index for phase structure function (5/3 => pure Kolmogorov turbulence) ncpu: int, optional Number of CPUs to use. If 0, all are used niter: int, optional Number of iterations to do when determining weights nsigma: float, optional Number of sigma above which directions are flagged """ import numpy as np from numpy import newaxis import re import os # Get screen type screen_type = soltab.getType() if screen_type not in ['phase', 'tec']: logging.error( 'Screens can only be fit to soltabs of type "phase" or "tec".') return 1 logging.info('Using solution table {0} to calculate {1} screens'.format( soltab.name, screen_type)) # Load phases axis_names = soltab.getAxesNames() r = np.array(soltab.val) weights = soltab.weight[:] if 'freq' in soltab.getAxesNames(): freqs = soltab.freq[:] if len(freqs) > 1: logging.error('Screens can only be fit at a single frequency') return 1 freq = freqs[0] # remove degenerate freq axis freq_ind = soltab.getAxesNames().index('freq') r = np.squeeze(r, axis=freq_ind) weights = np.squeeze(weights, axis=freq_ind) axis_names.pop(freq_ind) # fix for missing dir axis if not 'dir' in soltab.getAxesNames(): r = np.array([r]) weights = np.array([weights]) dir_ind = len(axis_names) source_names = ['POINTING'] else: dir_ind = axis_names.index('dir') source_names = soltab.dir[:] time_ind = axis_names.index('time') ant_ind = axis_names.index('ant') r = r.transpose([dir_ind, ant_ind, time_ind]) weights = weights.transpose([dir_ind, ant_ind, time_ind]) times = np.array(soltab.time) # Collect station and source names and positions and times, making sure # that they are ordered correctly. solset = soltab.getSolset() source_dict = solset.getSou() source_positions = [] for source in source_names: source_positions.append(source_dict[source]) station_names = soltab.ant[:] station_dict = solset.getAnt() station_positions = [] for station in station_names: station_positions.append(station_dict[station]) N_sources = len(source_names) N_times = len(times) N_stations = len(station_names) logging.info('Using height = {0} m and order = {1}'.format(height, order)) if height < 100e3: logging.warning("Height is less than 100e3 m. This is likely too low.") # Initialize various arrays N_piercepoints = N_sources * N_stations if screen_type == 'phase': real_screen = np.zeros((N_sources, N_stations, N_times)) real_residual = np.zeros((N_sources, N_stations, N_times)) imag_screen = np.zeros((N_sources, N_stations, N_times)) imag_residual = np.zeros((N_sources, N_stations, N_times)) screen = np.zeros((N_sources, N_stations, N_times)) residual = np.zeros((N_sources, N_stations, N_times)) val_amp = 1.0 r_0 = 100.0 # shouldn't matter what we choose rr = np.reshape(r, [N_piercepoints, N_times]) # Find pierce points and airmass values for given screen height pp, airmass, midRA, midDec = _calculate_piercepoints( np.array(station_positions), np.array(source_positions), np.array(times), height) # Fit the screens station_weights = np.reshape(weights, [N_piercepoints, N_times]) if screen_type == 'phase': mpm = multiprocManager(ncpu, _fit_phase_screen) for tindx, t in enumerate(times): w = np.diag(station_weights[:, tindx])[:, :, newaxis] mpm.put([ station_names, source_names, pp[tindx, newaxis, :, :], airmass[tindx, newaxis, :], rr[:, tindx, newaxis], w, [t], height, order, r_0, beta ]) mpm.wait() for (real_scr, real_res, imag_scr, imag_res, phase_scr, phase_res, t) in mpm.get(): i = times.tolist().index(t[0]) real_screen[:, :, i] = real_scr[0, :, :] real_residual[:, :, i] = real_res[0, :, :] imag_screen[:, :, i] = imag_scr[0, :, :] imag_residual[:, :, i] = imag_res[0, :, :] screen[:, :, i] = phase_scr[0, :, :] residual[:, :, i] = phase_res[0, :, :] elif screen_type == 'tec': mpm = multiprocManager(ncpu, _fit_tec_screen) for tindx, t in enumerate(times): w = np.diag(station_weights[:, tindx])[:, :, newaxis] mpm.put([ station_names, source_names, pp[tindx, newaxis, :, :], airmass[tindx, newaxis, :], rr[:, tindx, newaxis], w, [t], height, order, r_0, beta ]) mpm.wait() for (scr, res, t) in mpm.get(): i = times.tolist().index(t[0]) screen[:, :, i] = scr[0, :, :] residual[:, :, i] = res[0, :, :] weights = np.reshape(station_weights, (N_sources, N_stations, N_times)) # Write the results to the output solset dirs_out = source_names times_out = times ants_out = station_names # Store screen values weights = weights.transpose([0, 2, 1]) # order is now [dir, time, ant] vals = screen.transpose([0, 2, 1]) screen_st = solset.makeSoltab('{}screen'.format(screen_type), outSoltab, axesNames=['dir', 'time', 'ant'], axesVals=[dirs_out, times_out, ants_out], vals=vals, weights=weights) vals = residual.transpose([0, 2, 1]) resscreen_st = solset.makeSoltab('{}screenresid'.format(screen_type), outSoltab + 'resid', axesNames=['dir', 'time', 'ant'], axesVals=[dirs_out, times_out, ants_out], vals=vals, weights=weights) # Store beta, r_0, height, and order as attributes of the screen soltabs screen_st.obj._v_attrs['beta'] = beta screen_st.obj._v_attrs['r_0'] = r_0 screen_st.obj._v_attrs['height'] = height screen_st.obj._v_attrs['order'] = order if 'freq' in soltab.getAxesNames(): screen_st.obj._v_attrs['freq'] = freq # Store piercepoint table. Note that it does not conform to the axis # shapes, so we cannot use makeSoltab() solset.obj._v_file.create_array('/' + solset.name + '/' + screen_st.obj._v_name, 'piercepoint', obj=pp) screen_st.addHistory('CREATE (by DIRECTIONSCREEN operation)') resscreen_st.addHistory('CREATE (by DIRECTIONSCREEN operation)') return 0
def create_h5parm(instrumentdbFiles, antennaFile, fieldFile, skydbFile, h5parmFile, complevel, solsetName, globaldbFile=None, verbose=False): """ Create the h5parm file. Input: instrumentdbFiles - list of the finenames of the solutions. antennaFile - file name of the antenna table. fieldFile - file name of the field table. skydbFile - file name of the sky table. h5parmFile - file name of the h5parm file that will be created. complevel - level of compression. It is usually 5. solsetName - Name of the solution set. Usually "sol###". globaldbFile (optional) - Name of the globaldbFile. Used only for logging purposes. """ # open/create the h5parm file and the solution-set h5parm = h5parm_mod(h5parmFile, readonly=False, complevel=complevel) solset = h5parm.makeSolset(solsetName) # Create tables using the first instrumentdb # TODO: all the instrument tables should be checked #pdb = lofar.parmdb.parmdb(instrumentdbFiles[0]) pdb = pt.table(instrumentdbFiles[0]) # solTypes = list(set(x[0] for x in (x.split(":") for x in pdb.getNames()))) names = pt.table(instrumentdbFiles[0] + "/NAMES").getcol("NAME") solTypes = list(set(x[0] for x in (x.split(":") for x in names))) logging.info('Found solution types: ' + ', '.join(solTypes)) # rewrite solTypes in order to put together # Gain <-> DirectionalGain # CommonRotationAngle <-> RotationAngle # CommonScalarPhase <-> ScalarPhase # CommonScalarAmplitude <-> ScalarAmplitude # it also separate Real/Imag/Ampl/Phase into different solTypes # if "Gain" in solTypes: # solTypes.remove('Gain') # solTypes.append('*Gain:*:Real') # solTypes.append('*Gain:*:Imag') # solTypes.append('*Gain:*:Ampl') # solTypes.append('*Gain:*:Phase') # if "DirectionalGain" in solTypes: # solTypes.remove('DirectionalGain') # solTypes.append('*Gain:*:Real') # solTypes.append('*Gain:*:Imag') # solTypes.append('*Gain:*:Ampl') # solTypes.append('*Gain:*:Phase') # if "RotationAngle" in solTypes: # solTypes.remove('RotationAngle') # solTypes.append('*RotationAngle') # if "CommonRotationAngle" in solTypes: # solTypes.remove('CommonRotationAngle') # solTypes.append('*RotationAngle') # if "RotationMeasure" in solTypes: # solTypes.remove('RotationMeasure') # solTypes.append('*RotationMeasure') # if "ScalarPhase" in solTypes: # solTypes.remove('ScalarPhase') # solTypes.append('*ScalarPhase') # if "CommonScalarPhase" in solTypes: # solTypes.remove('CommonScalarPhase') # solTypes.append('*ScalarPhase') # if "CommonScalarAmplitude" in solTypes: # solTypes.remove('CommonScalarAmplitude') # solTypes.append('*ScalarAmplitude') if "Gain" in solTypes: solTypes.remove('Gain') solTypes.append(['Gain', 'Real']) solTypes.append(['Gain', 'Imag']) solTypes.append(['Gain', 'Ampl']) solTypes.append(['Gain', 'Phase']) if "DirectionalGain" in solTypes: solTypes.remove('DirectionalGain') solTypes.append(['Gain', 'Real']) solTypes.append(['Gain', 'Imag']) solTypes.append(['Gain', 'Ampl']) solTypes.append(['Gain', 'Phase']) if "RotationAngle" in solTypes: solTypes.remove('RotationAngle') solTypes.append(['RotationAngle']) if "CommonRotationAngle" in solTypes: solTypes.remove('CommonRotationAngle') solTypes.append(['RotationAngle']) if "RotationMeasure" in solTypes: solTypes.remove('RotationMeasure') solTypes.append(['RotationMeasure']) if "ScalarPhase" in solTypes: solTypes.remove('ScalarPhase') solTypes.append(['ScalarPhase']) if "CommonScalarPhase" in solTypes: solTypes.remove('CommonScalarPhase') solTypes.append(['ScalarPhase']) if "CommonScalarAmplitude" in solTypes: solTypes.remove('CommonScalarAmplitude') solTypes.append(['ScalarAmplitude']) # solTypes = list(set(solTypes)) print(solTypes) # every soltype creates a different solution-table for solType in solTypes: # skip missing solTypes (not all parmdbs have e.g. TEC) #if len(pdb.getNames(solType+':*')) == 0: continue found = False for name in names: if all([True if soli in name else False for soli in solType]): found = True break if not found: continue pols = set() dirs = set() ants = set() freqs = set() times = set() ptype = set() logging.info('Reading ' + ':'.join(solType) + '.') for instrumentdbFile in sorted(instrumentdbFiles): #pdb = lofar.parmdb.parmdb(instrumentdbFile) # create the axes grid, necessary if not all entries have the same axes lenght #data = pdb.getValuesGrid(solType+':*') data = getValuesGrid(instrumentdbFile, solType) # check good instrument table if len(data) == 0: logging.error('Instrument table %s is empty, ignoring.' % instrumentdbFile) for solEntry in data: pol, dir, ant, parm = parmdbToAxes(solEntry) if pol is not None: pols |= set([pol]) if dir is not None: dirs |= set([dir]) if ant is not None: ants |= set([ant]) freqs |= set(data[solEntry]['freqs']) times |= set(data[solEntry]['times']) pols = np.sort(list(pols)) dirs = np.sort(list(dirs)) ants = np.sort(list(ants)) freqs = np.sort(list(freqs)) times = np.sort(list(times)) shape = [ i for i in (len(pols), len(dirs), len(ants), len(freqs), len(times)) if i != 0 ] vals = np.empty(shape) vals[:] = np.nan weights = np.zeros(shape, dtype=np.float16) logging.info('Filling table.') for instrumentdbFile in instrumentdbFiles: #pdb = lofar.parmdb.parmdb(instrumentdbFile) # fill the values #data = pdb.getValuesGrid(solType+':*') data = getValuesGrid(instrumentdbFile, solType) #if 'Real' in solType: dataIm = pdb.getValuesGrid(solType.replace('Real','Imag')+':*') #if 'Imag' in solType: dataRe = pdb.getValuesGrid(solType.replace('Imag','Real')+':*') if 'Real' in solType: dataIm = getValuesGrid(instrumentdbFile, [solType[0], 'Imag']) if 'Imag' in solType: dataRe = getValuesGrid(instrumentdbFile, [solType[0], 'Real']) for solEntry in data: pol, dir, ant, parm = parmdbToAxes(solEntry) ptype |= set([solEntry.split(':')[0] ]) # original parmdb solution type freq = data[solEntry]['freqs'] time = data[solEntry]['times'] val = data[solEntry]['values'] # convert Real and Imag in Amp and Phase respectively if parm == 'Real': solEntryIm = solEntry.replace('Real', 'Imag') valI = dataIm[solEntryIm]['values'] val = np.sqrt((val**2) + (valI**2)) if parm == 'Imag': solEntryRe = solEntry.replace('Imag', 'Real') valR = dataRe[solEntryRe]['values'] val = np.arctan2(val, valR) coords = [] if pol is not None: polCoord = np.searchsorted(pols, pol) coords.append(polCoord) if dir is not None: dirCoord = np.searchsorted(dirs, dir) coords.append(dirCoord) if ant is not None: antCoord = np.searchsorted(ants, ant) coords.append(antCoord) freqCoord = np.searchsorted(freqs, freq) timeCoord = np.searchsorted(times, time) vals[tuple(coords)][np.ix_(freqCoord, timeCoord)] = val.T[:, :, 0] weights[tuple(coords)][np.ix_(freqCoord, timeCoord)] = 1 np.putmask(vals, ~np.isfinite(vals), 0) # put inf and nans to 0 #vals = np.nan_to_num(vals) # replace nans with 0 (flagged later) if solType == '*RotationAngle': np.putmask(weights, vals == 0., 0) # flag where val=0 solset.makeSoltab('rotation', axesNames=['dir','ant','freq','time'], \ axesVals=[dirs,ants,freqs,times], vals=vals, weights=weights, parmdbType=', '.join(list(ptype))) if solType == '*RotationMeasure': np.putmask(weights, vals == 0., 0) # flag where val=0 solset.makeSoltab('rotationmeasure', axesNames=['dir','ant','freq','time'], \ axesVals=[dirs,ants,freqs,times], vals=vals, weights=weights, parmdbType=', '.join(list(ptype))) elif solType == '*ScalarPhase': np.putmask(weights, vals == 0., 0) solset.makeSoltab('scalarphase', axesNames=['dir','ant','freq','time'], \ axesVals=[dirs,ants,freqs,times], vals=vals, weights=weights, parmdbType=', '.join(list(ptype))) elif solType == '*ScalarAmplitude': np.putmask(weights, vals == 0., 0) solset.makeSoltab('scalaramplitude', axesNames=['dir','ant','freq','time'], \ axesVals=[dirs,ants,freqs,times], vals=vals, weights=weights, parmdbType=', '.join(list(ptype))) elif solType == 'Clock': np.putmask(weights, vals == 0., 0) # clock may be diag or scalar if len(pols) == 0: solset.makeSoltab('clock', axesNames=['ant','freq','time'], \ axesVals=[ants,freqs,times], vals=vals, weights=weights, parmdbType=', '.join(list(ptype))) else: solset.makeSoltab('clock', axesNames=['pol','ant','freq','time'], \ axesVals=[pol,ants,freqs,times], vals=vals, weights=weights, parmdbType=', '.join(list(ptype))) elif solType == 'TEC': np.putmask(weights, vals == 0., 0) # tec may be diag or scalar if len(pols) == 0: solset.makeSoltab('tec', axesNames=['dir','ant','freq','time'], \ axesVals=[dirs,ants,freqs,times], vals=vals, weights=weights, parmdbType=', '.join(list(ptype))) else: solset.makeSoltab('tec', axesNames=['pol','dir','ant','freq','time'], \ axesVals=[pols,dirs,ants,freqs,times], vals=vals, weights=weights, parmdbType=', '.join(list(ptype))) elif solType == '*Gain:*:Real' or solType == '*Gain:*:Ampl' or solType == [ 'Gain', 'Real' ] or solType == ['Gain', 'Ampl']: np.putmask(vals, vals == 0, 1) # nans were put to 0 before, set them to 1 np.putmask(weights, vals == 1., 0) # flag where val=1 solset.makeSoltab('amplitude', axesNames=['pol','dir','ant','freq','time'], \ axesVals=[pols,dirs,ants,freqs,times], vals=vals, weights=weights, parmdbType=', '.join(list(ptype))) elif solType == '*Gain:*:Imag' or solType == '*Gain:*:Phase' or solType == [ 'Gain', 'Imag' ] or solType == ['Gain', 'Phase']: np.putmask(weights, vals == 0., 0) # falg where val=0 solset.makeSoltab('phase', axesNames=['pol','dir','ant','freq','time'], \ axesVals=[pols,dirs,ants,freqs,times], vals=vals, weights=weights, parmdbType=', '.join(list(ptype))) print(solType) logging.info('Flagged data: %.3f%%' % (100. * (len(weights.flat) - np.count_nonzero(weights)) / len(weights.flat))) logging.info('Collecting information from the ANTENNA table.') antennaTable = pt.table(antennaFile, ack=False) antennaNames = antennaTable.getcol('NAME') antennaPositions = antennaTable.getcol('POSITION') antennaTable.close() antennaTable = solset.obj._f_get_child('antenna') antennaTable.append(list(zip(*(antennaNames, antennaPositions)))) logging.info('Collecting information from the FIELD table.') fieldTable = pt.table(fieldFile, ack=False) phaseDir = fieldTable.getcol('PHASE_DIR') pointing = phaseDir[0, 0, :] fieldTable.close() sourceTable = solset.obj._f_get_child('source') # add the field centre, that is also the direction for Gain and Common* sourceTable.append([('pointing', pointing)]) dirs = [] for tab in solset.obj._v_children: c = solset.obj._f_get_child(tab) if c._v_name != 'antenna' and c._v_name != 'source': if c.__contains__('dir'): dirs.extend(list(set(c.dir))) # remove duplicates dirs = list(set(dirs)) # remove any pointing (already in the table) if 'pointing' in dirs: dirs.remove('pointing') if not os.path.isdir(skydbFile) and dirs != []: logging.critical('Missing skydb table.') sys.exit(1) if dirs != []: logging.info('Collecting information from the sky table.') sourceFile = skydbFile + '/SOURCES' src_table = pt.table(sourceFile, ack=False) sub_tables = src_table.getsubtables() vals = [] ra = dec = np.nan has_patches_subtable = False for sub_table in sub_tables: if 'PATCHES' in sub_table: has_patches_subtable = True if has_patches_subtable: # Read values from PATCHES subtable src_table.close() sourceFile = skydbFile + '/SOURCES/PATCHES' src_table = pt.table(sourceFile, ack=False) patch_names = src_table.getcol('PATCHNAME') patch_ras = src_table.getcol('RA') patch_decs = src_table.getcol('DEC') for source in dirs: try: patch_indx = patch_names.index(source) ra = patch_ras[patch_indx] dec = patch_decs[patch_indx] except ValueError: ra = np.nan dec = np.nan logging.error('Cannot find the source ' + source + '. I leave NaNs.') vals.append([ra, dec]) src_table.close() else: # Try to read default values from parmdb instead skydb = lofar.parmdb.parmdb(skydbFile) vals = [] ra = dec = np.nan for source in dirs: try: ra = skydb.getDefValues('Ra:' + source)['Ra:' + source][0][0] dec = skydb.getDefValues('Dec:' + source)['Dec:' + source][0][0] except KeyError: # Source not found in skymodel parmdb, try to find components logging.warning('Cannot find the source ' + source + '. Trying components.') ra = np.array( list( skydb.getDefValues('Ra:*' + source + '*').values())) dec = np.array( list( skydb.getDefValues('Dec:*' + source + '*').values())) if len(ra) == 0 or len(dec) == 0: ra = np.nan dec = np.nan logging.error('Cannot find the source ' + source + '. I leave NaNs.') else: ra = ra.mean() dec = dec.mean() logging.info('Found average direction for ' + source + ' at ra:' + str(ra) + ' - dec:' + str(dec)) vals.append([ra, dec]) sourceTable.append(list(zip(*(dirs, vals)))) logging.info("Total file size: " + str(int(h5parm.H.get_filesize() / 1024. / 1024.)) + " M.") # Add CREATE entry to history and print summary of tables if verbose soltabs = solset.getSoltabs() for st in soltabs: if globaldbFile is None: st.addHistory( 'CREATE (by H5parm_importer.py from %s:%s/%s)' % (socket.gethostname(), os.path.abspath(''), "manual list")) else: st.addHistory( 'CREATE (by H5parm_importer.py from %s:%s/%s)' % (socket.gethostname(), os.path.abspath(''), globaldbFile)) if verbose: logging.info(str(h5parm)) del h5parm logging.info('Done.')
def run(soltab, interp_dirs, soltabOut=None, prefix='interp_', ncpu=0): """ Add interpolated directions to h5parm Parameters ---------- interp_dirs : 2d array of floats Shape n x 2, contains ra/dec in degree. For example: [[ra1,dec1],[ra2,dec2],...] soltabOut : string, optional Default: Guess from soltype. If specifically set to input soltab, the input soltab will be overwritten. prefix : string, optional, default = "interp_". Name prefix of interpolated directions. """ import multiprocessing as mp # need scipy commit f0a478c4a4172c4d2225910216de6b62721db161 for multidimensional interp. otherwise slow... if scipy.__version__ < '1.4.0': raise ImportError( 'SciPy version >= 1.4.0 is required to support multidimensional rbf interpolation.' ) logging.info("Working on soltab: " + soltab.name) soltype = soltab.getType() solset = soltab.getSolset() soltabOut = None if soltabOut == '' else soltabOut # check input if soltype == 'phase': interp_kind = 'wrap' elif soltype == 'amplitude': interp_kind = 'amp' elif soltype in ['tec', 'rotationmeasure', 'tec3rd']: interp_kind = 'lin' else: logging.error('Soltab type {} not supported.'.format(soltype)) return 1 if not 'dir' in soltab.getAxesNames(): logging.error( 'Data without dir axis cannot be interpolated in directions...') return 1 if not 'ant' in soltab.getAxesNames(): logging.error('Data without ant axis not supported...') return 1 if len(soltab.dir) < 10: logging.error( 'Not enough directions. Use at least ten for interpolation.') return 1 ax_ord = soltab.getAxesNames() # original order dir_ax = ax_ord.index('dir') # convert to rad if interp_dirs.shape == (2, ): # if only one direction, add dir axis interp_dirs = interp_dirs[np.newaxis] interp_dirs = np.deg2rad(interp_dirs) # prepare array for interpolated values - concatenate new directions val_shape_inpt = list(soltab.val.shape) val_shape_interp = val_shape_inpt.copy() val_shape_interp[0] = len( interp_dirs) # match direction axis with no of interp dirs interp_vals = np.zeros(val_shape_interp) interp_weights = np.ones(val_shape_interp) # ra/dec of calibrator dirs. Make sure order is the same as in soltab. cal_dirs = np.array([soltab.getSolset().getSou()[k] for k in soltab.dir]) # Collect arguments for parallelization on antennas args, selections = [], [] returnAxes = ax_ord.copy() returnAxes.remove('ant') for i, (vals, weights, coord, selection) in enumerate( soltab.getValuesIter(returnAxes=returnAxes, weight=True)): dir_ax_sel = returnAxes.index('dir') args.append( [vals, weights, cal_dirs, dir_ax_sel, interp_dirs, interp_kind]) selections.append(selection) #################### DEBUG PLOT ###################### if False: # What to plot? antidx = 32 sel = [100, 201] import matplotlib.pyplot as plt minra, maxra = np.min(cal_dirs[:, 0]) - 0.005, np.max( cal_dirs[:, 0]) + 0.005 ra_range = np.linspace(minra, maxra, 500) mindec, maxdec = np.min(cal_dirs[:, 1]) - 0.005, np.max( cal_dirs[:, 1]) + 0.005 dec_range = np.linspace(mindec, maxdec, 500) plotdirs = np.meshgrid(ra_range, dec_range) plotdirs = np.array((np.array(plotdirs[0]).flatten(), np.array(plotdirs[1]).flatten())).T pvals, pweights = args[antidx][0:2] pvals, pweights = np.array(pvals)[:, sel], np.array(pweights)[:, sel] plotvals = np.array( interpolate_directions3d(pvals, pweights, cal_dirs, dir_ax, plotdirs, interp_kind, smooth=1.e-3))[0, :, 0, 0, 0] plotvals = plotvals.reshape((500, 500)) fig = plt.figure(dpi=200) plt.xlabel('RA', labelpad=13, fontsize=7.5) plt.ylabel('Dec', labelpad=13, fontsize=7.5) cmap = plt.set_cmap('jet') if soltype == 'phase': label = 'phase [rad]' vmin, vmax = -np.pi, np.pi elif soltype == 'tec': label = 'dTEC [TECU]' vmin, vmax = None, None elif soltype == 'amplitude': label = 'amplitude' # vmin, vmax = 0, 1 vmin, vmax = np.min(plotvals), np.max(plotvals) # vmin, vmax = np.min(pvals[:,0,0,0]), np.max(pvals[:,0,0,0]) else: label = 'unknown' vmin, vmax = None, None im = plt.imshow(plotvals, cmap=cmap, extent=[minra, maxra, maxdec, mindec], vmin=vmin, vmax=vmax) cb = fig.colorbar(im) plt.scatter(*cal_dirs.T, c=pvals[:, 0, 0, 0], cmap=cmap, edgecolors='k', vmin=vmin, vmax=vmax) plt.scatter(*cal_dirs.T, c=pvals[:, 0, 0, 0], cmap=cmap, edgecolors='k', vmin=vmin, vmax=vmax) plt.scatter(*interp_dirs.T, facecolor="None", marker='D', edgecolors='k') cb.set_label(label, fontsize=7) cb.ax.tick_params(labelsize=6.5) plt.savefig( 'debug_screen_interp.png', dpi=200, bbox_inches='tight', ) import sys sys.exit() # run the interpolation ncpu = mp.cpu_count() if ncpu == 0 else ncpu # default use all cores with mp.Pool(ncpu) as pool: logging.info('Start interpolation.') results = pool.starmap(interpolate_directions3d, args) # reorder results for selection, result in zip(selections, results): vals, weights = result # fill output arrays - the purpose of the reshape is to get the degenerate dim for the ant interp_vals[tuple(selection)] = vals.reshape( interp_vals[tuple(selection)].shape) interp_weights[tuple(selection)] = weights.reshape( interp_vals[tuple(selection)].shape) # concatenate existing values and interpolated direction values vals = np.concatenate([soltab.val, interp_vals], axis=dir_ax) weights = np.concatenate([soltab.weight, interp_weights], axis=dir_ax) # set names for the interpolated directions interp_dir_names = np.arange(len(interp_dirs)).astype(str) interp_dir_names = [prefix + n.zfill(3) for n in interp_dir_names] # prepare axes values, append directions axes_vals = [soltab.getAxisValues(axisName) for axisName in ax_ord] axes_vals[dir_ax] = np.concatenate([axes_vals[dir_ax], interp_dir_names]) # prepare output - check if soltabOut exists if soltabOut in solset.getSoltabNames(): logging.warning('Soltab {} exists. Overwriting...'.format(soltabOut)) solset.getSoltab(soltabOut).delete() # make soltabOut soltabout = solset.makeSoltab(soltype=soltype, soltabName=soltabOut, axesNames=ax_ord, \ axesVals=axes_vals, vals=vals, weights=weights) newSources = dict(zip(interp_dir_names, interp_dirs)) # append interpolated dirs to solset source table sourceTable = solset.obj._f_get_child('source') for row in sourceTable.iterrows(): _name, _dir = row['name'].decode(), row['dir'] if _name in newSources.keys(): if not np.all(np.isclose(_dir, newSources[_name])): logging.debug('Overwrite direction: {}'.format(_name)) row['dir'] = newSources[_name] else: logging.debug('Source already in soltab: {}'.format(_name)) del newSources[_name] if len(newSources) > 0: logging.debug('Adding to source table: {}'.format(newSources.keys())) sourceTable.append(list(zip(newSources.keys(), newSources.values()))) # sourceTable.append(list(zip(newSources.keys(), newSources.values()))) # Add CREATE entry to history soltabout.addHistory( 'Created by INTERPOLATEDIRECTIONS operation from %s.' % soltab.name) return 0
def run(soltab, soltabOut='tec000', refAnt=''): """ Bruteforce TEC extraction from phase solutions. Parameters ---------- soltabOut : str, optional output table name (same solset), by deault "tec". refAnt : str, optional Reference antenna, by default the first. """ import numpy as np import scipy.optimize def mod(d): return np.mod(d + np.pi, 2. * np.pi) - np.pi def cost_f(d, freq, y): nfreq, ntime = y.shape phase = mod(2 * np.pi * d * freq).repeat(ntime).reshape(nfreq, ntime) dist = np.abs(mod(phase - y)) ngood = np.sum(~np.isnan(dist)) return np.nansum(dist / ngood) logging.info("Find global DELAY for soltab: " + soltab.name) # input check solType = soltab.getType() if solType != 'phase': logging.warning("Soltab type of " + soltab._v_name + " is of type " + solType + ", should be phase. Ignoring.") return 1 ants = soltab.getAxisValues('ant') if refAnt != '' and refAnt != 'closest' and not refAnt in soltab.getAxisValues( 'ant', ignoreSelection=True): logging.warning('Reference antenna ' + refAnt + ' not found. Using: ' + ants[1]) refAnt = ants[0] if refAnt == '': refAnt = ants[0] # times and ants needs to be complete or selection is much slower times = soltab.getAxisValues('time') # create new table solset = soltab.getSolset() soltabout = solset.makeSoltab(soltype = 'clock', soltabName = soltabOut, axesNames=['ant','time'], \ axesVals=[soltab.getAxisValues(axisName) for axisName in ['ant','time']], \ vals=np.zeros(shape=(soltab.getAxisLen('ant'),soltab.getAxisLen('time'))), \ weights=np.ones(shape=(soltab.getAxisLen('ant'),soltab.getAxisLen('time'))) ) soltabout.addHistory('Created by GLOBALDELAY operation from %s.' % soltab.name) for vals, weights, coord, selection in soltab.getValuesIter( returnAxes=['freq', 'time'], weight=True, refAnt=refAnt): if len(coord['freq']) < 5: logging.error( 'Delay estimation needs at least 5 frequency channels, preferably distributed over a wide range.' ) return 1 # reorder axes vals = reorderAxes(vals, soltab.getAxesNames(), ['freq', 'time']) weights = reorderAxes(weights, soltab.getAxesNames(), ['freq', 'time']) ranges = (-1e-7, 1e-7) Ns = 1001 delay_fitresult = np.zeros(len(times)) weights_fitresult = np.ones(len(times)) if not coord['ant'] == refAnt: if (weights == 0.).all() == True: logging.warning('Skipping flagged antenna: ' + coord['ant']) weights_fitresult[:] = 0 else: freq = np.copy(coord['freq']) # brute force fit = scipy.optimize.brute(cost_f, ranges=(ranges, ), Ns=Ns, args=(freq, vals)) delay_fitresult[:] = fit logging.info('%s: average delay: %f ns' % (coord['ant'], fit * 1e9)) # reorder axes back to the original order, needed for setValues soltabout.setSelection(ant=coord['ant']) soltabout.setValues(delay_fitresult) soltabout.setValues(weights_fitresult, weight=True) return 0
def run(soltab, soltabOutTEC='tec000', soltabOutBP='phase000', refAnt=''): """ Isolate BP from TEC in a calibrator (uGMRT data) Parameters ---------- soltabOutTEC : str, optional output TEC table name (same solset), by deault "tec000". soltabOutBP : str, optional output bandpass table name (same solset), by deault "phase000". refAnt : str, optional Reference antenna, by default get the closest for each antenna. """ import numpy as np logging.info("Find BANDPASS+TEC for soltab: " + soltab.name) # input check solType = soltab.getType() if solType != 'phase': logging.warning("Soltab type of " + soltab._v_name + " is of type " + solType + ", should be phase. Ignoring.") return 1 ants = soltab.getAxisValues('ant') if refAnt != '' and refAnt != 'closest' and not refAnt in soltab.getAxisValues( 'ant', ignoreSelection=True): logging.error('Reference antenna ' + refAnt + ' not found. Using: ' + ants[1]) refAnt = ants[0] if refAnt == '': refAnt = ants[0] # create new table solset = soltab.getSolset() soltaboutTEC = solset.makeSoltab(soltype = 'tec', soltabName = soltabOut, axesNames=['ant','time'], \ axesVals=[soltab.getAxisValues(axisName) for axisName in ['ant','time']], \ vals=np.zeros(shape=(soltab.getAxisLen('ant'),soltab.getAxisLen('time'))), \ weights=np.ones(shape=(soltab.getAxisLen('ant'),soltab.getAxisLen('time'))) ) soltaboutTEC.addHistory('Created by BANDPASSTEC operation from %s.' % soltab.name) soltaboutBP = solset.makeSoltab(soltype = 'phase', soltabName = soltabOut, axesNames=['ant','freq','pol'], \ axesVals=[soltab.getAxisValues(axisName) for axisName in ['ant','freq', 'pol']], \ vals=np.zeros(shape=(soltab.getAxisLen('ant'),soltab.getAxisLen('freq'),soltab.getAxisLen('pol'))), \ weights=np.ones(shape=(soltab.getAxisLen('ant'),soltab.getAxisLen('freq'),soltab.getAxisLen('pol'))) ) soltaboutBP.addHistory('Created by BANDPASSTEC operation from %s.' % soltab.name) # get values vals, axesVals = soltab.getValues(retAxesVals=True) # separate bandpass/tec plot = False if plot: pass # write solutions back soltaboutTEC.setValues(fitd) soltaboutBP.setValues(fitd) soltaboutTEC.setValues(fitweights, weight=True) soltaboutBP.setValues(fitweights, weight=True) return 0
#!/usr/bin/env python # -*- coding: utf-8 -*- from losoto.lib_operations import * from losoto._logging import logger as logging from losoto.operations._faraday_timestep import _run_timestep logging.debug('Loading FRjump module.') logging.warning('FRjump module is still experimental - we strongly recommend to check the results carefully') def _run_parser(soltab, parser, step): soltabOut = parser.getstr( step, 'soltabOut', 'rotationmeasure002' ) soltabPhase = parser.getstr( step, 'soltabPhase', 'phase000') clipping = np.array(parser.getarray(step, 'clipping', [0,1e9]),dtype=float) frequencies = np.array(parser.getarray(step, 'frequencies', []),dtype=float) parser.checkSpelling( step, soltab, ['soltabOut','soltabPhase','clipping','frequencies']) return run(soltab, soltabOut,clipping, soltabPhase,frequencies) def costfunctionRM(RM, wav, phase): return np.sum(abs(np.cos(2.*RM[0]*wav*wav) - np.cos(phase)) + abs(np.sin(2.*RM[0]*wav*wav) - np.sin(phase))) def getPhaseWrapBase(wavels): """ freqs: frequency grid of the data return the step size from a local minima (2pi phase wrap) to the others [0]: TEC, [1]: clock """ wavels = np.array(wavels) nF = wavels.shape[0] A = np.zeros((nF, 1), dtype=np.float)
def _flag_bandpass(freqs, amps, weights, telescope, nSigma, ampRange, maxFlaggedFraction, maxStddev, plot, ants, s, outQueue): """ Flags bad amplitude solutions relative to median bandpass (in log space) by setting the corresponding weights to 0.0 Note: A median over the time axis is done before flagging, so the flags are not time- dependent Parameters ---------- freqs : array Array of frequencies amps : array Array of amplitudes as [time, ant, freq, pol] weights : array Array of weights as [time, ant, freq, pol] telescope : str, optional Specifies the telescope for the bandpass model nSigma : float Number of sigma for flagging. Amplitudes outside of nSigma*stddev are flagged maxFlaggedFraction : float Maximum allowable fraction of flagged frequencies. Stations with higher fractions will be completely flagged maxStddev : float Maximum allowable standard deviation plot : bool If True, the bandpass with flags and best-fit line is plotted for each station ants : list List of station names s : int Station index ampRange : array 2-element array of the median amplitude level to be acceptable, ampRange[0]: lower limit, ampRange[1]: upper limit Returns ------- indx, weights : int, array Station index, modified weights array """ def _B(x, k, i, t, extrap, invert): if k == 0: if extrap: if invert: return -1.0 else: return 1.0 else: return 1.0 if t[i] <= x < t[i + 1] else 0.0 if t[i + k] == t[i]: c1 = 0.0 else: c1 = (x - t[i]) / (t[i + k] - t[i]) * _B(x, k - 1, i, t, extrap, invert) if t[i + k + 1] == t[i + 1]: c2 = 0.0 else: c2 = (t[i + k + 1] - x) / (t[i + k + 1] - t[i + 1]) * _B( x, k - 1, i + 1, t, extrap, invert) return c1 + c2 def _bspline(x, t, c, k): n = len(t) - k - 1 assert (n >= k + 1) and (len(c) >= n) invert = False extrap = [False] * n if x >= t[n]: extrap[-1] = True elif x < t[k]: extrap[0] = True invert = False return sum(c[i] * _B(x, k, i, t, e, invert) for i, e in zip(list(range(n)), extrap)) def _bandpass_LBA(freq, c1, c2, c3, c4, c5, c6, c7, c8, c9, c10, c11, c12, c13): """ Defines the functional form of the LBA bandpass in terms of splines of degree 3 The spline fit was done using LSQUnivariateSpline() on the median bandpass between 30 MHz and 78 MHz. The knots were set by hand to acheive a good fit with a minimum number of parameters. Parameters ---------- freq : array Array of frequencies c1-c13 : float Spline coefficients Returns ------- bandpass : list List of bandpass values as function of frequency """ knots = np.array([ 30003357.0, 30003357.0, 30003357.0, 30003357.0, 40000000.0, 50000000.0, 55000000.0, 56000000.0, 60000000.0, 62000000.0, 63000000.0, 64000000.0, 70000000.0, 77610779.0, 77610779.0, 77610779.0, 77610779.0 ]) coeffs = np.array( [c1, c2, c3, c4, c5, c6, c7, c8, c9, c10, c11, c12, c13]) return [_bspline(f, knots, coeffs, 3) for f in freq] def _bandpass_HBA_low(freq, c1, c2, c3, c4, c5, c6, c7, c8, c9, c10): """ Defines the functional form of the HBA-low bandpass in terms of splines of degree 3 The spline fit was done using LSQUnivariateSpline() on the median bandpass between 120 MHz and 188 MHz. The knots were set by hand to acheive a good fit with a minimum number of parameters. Parameters ---------- freq : array Array of frequencies c1-c10 : float Spline coefficients Returns ------- bandpass : list List of bandpass values as function of frequency """ knots = np.array([ 1.15e+08, 1.15e+08, 1.15e+08, 1.15e+08, 1.30e+08, 1.38e+08, 1.48e+08, 1.60e+08, 1.68e+08, 1.78e+08, 1.90e+08, 1.90e+08, 1.9e+08, 1.9e+08 ]) coeffs = np.array([c1, c2, c3, c4, c5, c6, c7, c8, c9, c10]) return [_bspline(f, knots, coeffs, 3) for f in freq] def _fit_bandpass(freq, logamp, sigma, band, do_fit=True): """ Fits amplitudes with one of the bandpass functions The initial coefficients were determined from a LSQUnivariateSpline() fit on the median bandpass of the appropriate band. The allowable fitting ranges were set by hand through testing on a number of observations (to allow the bandpass function to adjust for the differences between stations but not to fit to RFI, etc.). Parameters ---------- freq : array Array of frequencies amps : array Array of log10(amplitudes) sigma : array Array of sigma (1/weights**2) band : str Band name ('hba_low', etc.) do_fit : bool, optional If True, the fitting is done. If False, the unmodified model bandpass is returned Returns ------- fit_parms, bandpass : list, list List of best-fit parameters, List of bandpass values as function of frequency """ from scipy.optimize import curve_fit if band.lower() == 'hba_low': bandpass_function = _bandpass_HBA_low init_coeffs = np.array([ -0.01460369, 0.05062699, 0.02827004, 0.03738518, -0.05729109, 0.02303295, -0.03550487, -0.0803113, -0.2394929, -0.358301 ]) bounds_deltas_lower = [ 0.06, 0.05, 0.04, 0.04, 0.04, 0.04, 0.1, 0.1, 0.2, 0.5 ] bounds_deltas_upper = [ 0.06, 0.1, 0.1, 0.1, 0.04, 0.04, 0.04, 0.04, 0.05, 0.06 ] elif band.lower() == 'lba': bandpass_function = _bandpass_LBA init_coeffs = np.array([ -0.22654016, -0.1950495, -0.07763014, 0.10002095, 0.32797671, 0.46900048, 0.47155583, 0.31945897, 0.29072278, 0.08064795, -0.15761538, -0.36020451, -0.51163338 ]) bounds_deltas_lower = [ 0.25, 0.2, 0.05, 0.05, 0.05, 0.1, 0.1, 0.16, 0.2, 0.15, 0.15, 0.25, 0.3 ] bounds_deltas_upper = [ 0.4, 0.3, 0.15, 0.05, 0.05, 0.05, 0.08, 0.05, 0.08, 0.15, 0.15, 0.25, 0.35 ] else: logging.error('The "{}" band is not supported'.format(band)) return None, None if do_fit: lower = [c - b for c, b in zip(init_coeffs, bounds_deltas_lower)] upper = [c + b for c, b in zip(init_coeffs, bounds_deltas_upper)] param_bounds = (lower, upper) try: popt, pcov = curve_fit(bandpass_function, freq, logamp, sigma=sigma, bounds=param_bounds, method='dogbox', ftol=1e-3, xtol=1e-3, gtol=1e-3) return popt, bandpass_function(freq, *tuple(popt)) except RuntimeError: logging.error('Fitting failed.') return None, bandpass_function(freq, *tuple(init_coeffs)) else: return None, bandpass_function(freq, *tuple(init_coeffs)) # Check that telescope and band is supported. Skip flagging if not if telescope.lower() == 'lofar': # Determine which band we're in if np.median(freqs) < 180e6 and np.median(freqs) > 110e6: band = 'hba_low' elif np.median(freqs) < 90e6: band = 'lba' else: logging.warning( 'The median frequency of {} Hz is outside of the currently supported LOFAR bands ' '(LBA and HBA-low). Flagging will be skipped'.format( np.median(freqs))) outQueue.put([s, weights]) return else: logging.warning( "Only telescope = 'lofar' is currently supported for bandpass mode. " "Flagging will be skipped") outQueue.put([s, weights]) return # Skip fully flagged stations if np.all(weights == 0.0): outQueue.put([s, weights]) return # Build arrays for fitting flagged = np.where(np.logical_or(weights == 0.0, np.isnan(amps))) amps_flagged = amps.copy() amps_flagged[flagged] = np.nan sigma = weights.copy() sigma[flagged] = 1.0 sigma = np.sqrt(1.0 / sigma) sigma[flagged] = 1e8 # Set range of allowed values for the median if ampRange is None or ampRange == [0.0, 0.0]: # Use sensible values depending on correlator if np.nanmedian(amps_flagged) > 1.0: # new correlator ampRange = [50.0, 325.0] else: # old correlator ampRange = [0.0004, 0.0018] median_min = ampRange[0] median_max = ampRange[-1] # Iterate over polarizations npols = amps.shape[2] for pol in range(npols): # Skip fully flagged polarizations if np.all(weights[:, :, pol] == 0.0): continue # Take median over time and divide out the median offset with np.warnings.catch_warnings(): # Filter NaN warnings -- we deal with NaNs below np.warnings.filterwarnings('ignore', r'All-NaN (slice|axis) encountered') amps_div = np.nanmedian(amps_flagged[:, :, pol], axis=0) median_val = np.nanmedian(amps_div) amps_div /= median_val sigma_div = np.median(sigma[:, :, pol], axis=0) sigma_orig = sigma_div.copy() unflagged = np.where(~np.isnan(amps_div)) nsols_unflagged = len(unflagged[0]) median_flagged = np.where(np.isnan(amps_div)) amps_div[median_flagged] = 1.0 sigma_div[median_flagged] = 1e8 median_flagged = np.where(amps_div <= 0.0) amps_div[median_flagged] = 1.0 sigma_div[median_flagged] = 1e8 # Before doing the fitting, renormalize and flag any solutions that deviate from # the model bandpass by a large factor to avoid biasing the first fit _, bp_sp = _fit_bandpass(freqs, np.log10(amps_div), sigma_div, band, do_fit=False) normval = np.median(np.log10(amps_div) - bp_sp) # value to normalize model to data amps_div /= 10**normval bad = np.where(np.abs(np.array(bp_sp) - np.log10(amps_div)) > 0.2) sigma_div[bad] = 1e8 if np.all(sigma_div > 1e7): logging.info('Flagged {0} (pol {1}) due to poor match to ' 'baseline bandpass model'.format(ants[s], pol)) weights[:, :, pol] = 0.0 outQueue.put([s, weights]) return # Iteratively fit and flag maxiter = 5 niter = 0 nflag = 0 nflag_prev = -1 while nflag != nflag_prev and niter < maxiter: p, bp_sp = _fit_bandpass(freqs, np.log10(amps_div), sigma_div, band) stdev_all = np.sqrt( np.average((bp_sp - np.log10(amps_div))**2, weights=(1 / sigma_div)**2)) stdev = min(maxStddev, stdev_all) bad = np.where(np.abs(bp_sp - np.log10(amps_div)) > nSigma * stdev) nflag = len(bad[0]) if nflag == 0 or nflag == nsols_unflagged: break if niter > 0: nflag_prev = nflag sigma_div = sigma_orig.copy() # reset flags to original ones sigma_div[bad] = 1e8 niter += 1 if plot: import matplotlib.pyplot as plt plt.plot(freqs, bp_sp, 'g-', lw=3) plt.plot(freqs, np.log10(amps_div), 'o', c='g') plt.plot(freqs[bad], np.log10(amps_div)[bad], 'o', c='r') plt.show() # Check whether entire station is bad (high stdev or high flagged fraction). If # so, flag all frequencies and polarizations if stdev_all > nSigma * maxStddev: # Station has high stddev relative to median bandpass logging.info('Flagged {0} (pol {1}) due to high stddev ' '({2})'.format(ants[s], pol, stdev_all)) weights[:, :, pol] = 0.0 elif float(len(bad[0])) / float(nsols_unflagged) > maxFlaggedFraction: # Station has high fraction of initially unflagged solutions that are now flagged logging.info('Flagged {0} (pol {1}) due to high flagged fraction ' '({2:.2f})'.format( ants[s], pol, float(len(bad[0])) / float(nsols_unflagged))) weights[:, :, pol] = 0.0 else: flagged = np.where(sigma_div > 1e3) nflagged_orig = len(np.where(weights[:, :, pol] == 0.0)[0]) weights[:, flagged[0], pol] = 0.0 nflagged_new = len(np.where(weights[:, :, pol] == 0.0)[0]) median_val = np.nanmedian(amps[np.where(weights[:, :, pol] > 0.0)]) if median_val < median_min or median_val > median_max: # Station has extreme median value logging.info( 'Flagged {0} (pol {1}) due to extreme median value ' '({2})'.format(ants[s], pol, median_val)) weights[:, :, pol] = 0.0 else: # Station is OK, flag bad points only prcnt = float(nflagged_new - nflagged_orig) / float( np.product(weights.shape[:-1])) * 100.0 logging.info( 'Flagged {0:.1f}% of solutions for {1} (pol {2})'.format( prcnt, ants[s], pol)) outQueue.put([s, weights])
def run(soltab, axesToSmooth, size=[], mode='runningmedian', degree=1, replace=False, log=False, refAnt=''): """ A smoothing function: running-median on an arbitrary number of axes, running polyfit and Savitzky-Golay on one axis, or set all solutions to the mean/median value. WEIGHT: flag ready. Parameters ---------- axesToSmooth : array of str Axes used to compute the smoothing function. size : array of int, optional Window size for the runningmedian, savitzky-golay, and runningpoly (array of same size of axesToSmooth), by default []. mode : {'runningmedian','runningpoly','savitzky-golay','mean','median'}, optional Runningmedian or runningpoly or Savitzky-Golay or mean or median (these last two values set all the solutions to the mean/median), by default "runningmedian". degree : int, optional Degrees of the polynomia for the runningpoly or savitzky-golay modes, by default 1. replace : bool, optional Flagged data are replaced with smoothed value and unflagged, by default False. log : bool, optional clip is done in log10 space, by default False refAnt : str, optional Reference antenna for phases. By default None. """ import numpy as np from scipy.ndimage import generic_filter if refAnt == '': refAnt = None elif not refAnt in soltab.getAxisValues('ant', ignoreSelection=True): logging.error('Reference antenna ' + refAnt + ' not found. Using: ' + soltab.getAxisValues('ant')[1]) refAnt = soltab.getAxisValues('ant')[1] if mode == "runningmedian" and len(axesToSmooth) != len(size): logging.error("Axes and Size lengths must be equal for runningmedian.") return 1 if (mode == "runningpoly" or mode == "savitzky-golay") and (len(axesToSmooth) != 1 or len(size) != 1): logging.error( "Axes and size lengths must be 1 for runningpoly or savitzky-golay." ) return 1 if (mode == "runningpoly" or mode == "savitzky-golay") and soltab.getType() == 'phase': logging.error( "Runningpoly and savitzky-golay modes cannot work on phases.") return 1 for i, s in enumerate(size): if s % 2 == 0: logging.warning('Size should be odd, adding 1.') size[i] += 1 logging.info("Smoothing soltab: " + soltab.name) for i, axis in enumerate(axesToSmooth[:]): if axis not in soltab.getAxesNames(): del axesToSmooth[i] del size[i] logging.warning('Axis \"' + axis + '\" not found. Ignoring.') if soltab.getType() == 'amplitude' and not log: logging.warning( 'Amplitude solution tab detected and log=False. Amplitude solution tables should be treated in log space.' ) if mode == 'median' or mode == 'mean': vals = soltab.getValues(retAxesVals=False, reference=refAnt) if log: vals = np.log10(vals) weights = soltab.getValues(retAxesVals=False, weight=True) np.putmask(vals, weights == 0, np.nan) idx_axes = [ soltab.getAxesNames().index(axisToSmooth) for axisToSmooth in axesToSmooth ] # handle phases by using a complex array if soltab.getType() == 'phase': vals = np.exp(1j * vals) if mode == 'median': vals[:] = np.nanmedian(vals, axis=idx_axes, keepdims=True) if mode == 'mean': logging.warning( 'Mean does not support NaN yet, use median if it is a problem.' ) vals[:] = np.mean( vals, axis=idx_axes, keepdims=True ) # annoying np.nanmean does not accept axis=list! # go back to phases if soltab.getType() == 'phase': vals = np.angle(vals) # write back if log: vals = 10**vals soltab.setValues(vals) if replace: weights[(weights == 0)] = 1 weights[np.isnan( vals )] = 0 # all the slice was flagged, cannot estrapolate value soltab.setValues(weights, weight=True) else: for vals, weights, coord, selection in soltab.getValuesIter( returnAxes=axesToSmooth, weight=True, reference=refAnt): # skip completely flagged selections if (weights == 0).all(): continue if log: vals = np.log10(vals) if mode == 'runningmedian': vals_bkp = vals[weights == 0] # handle phases by using a complex array if soltab.getType() == 'phase': vals = np.exp(1j * vals) valsreal = np.real(vals) valsimag = np.imag(vals) np.putmask(valsreal, weights == 0, np.nan) np.putmask(valsimag, weights == 0, np.nan) # run generic_filter twice, once for real once for imaginary valsrealnew = generic_filter(valsreal, np.nanmedian, size=size, mode='constant', cval=np.nan) valsimagnew = generic_filter(valsimag, np.nanmedian, size=size, mode='constant', cval=np.nan) valsnew = valsrealnew + 1j * valsimagnew # go back to complex valsnew = np.angle(valsnew) # go back to phases else: # other than phases np.putmask(vals, weights == 0, np.nan) valsnew = generic_filter(vals, np.nanmedian, size=size, mode='constant', cval=np.nan) if replace: weights[weights == 0] = 1 weights[np.isnan( valsnew )] = 0 # all the size was flagged cannoth estrapolate value else: valsnew[weights == 0] = vals_bkp elif mode == 'runningpoly': def polyfit(data): if (np.isnan(data)).all(): return np.nan # all size is flagged x = np.arange(len(data))[~np.isnan(data)] y = data[~np.isnan(data)] p = np.polynomial.polynomial.polyfit(x, y, deg=degree) #import matplotlib as mpl #mpl.use("Agg") #import matplotlib.pyplot as plt #plt.plot(x, y, 'ro') #plt.plot(x, np.polyval( p[::-1], x ), 'k-') #plt.savefig('test.png') #sys.exit() return np.polyval( p[::-1], (size[0] - 1) / 2 ) # polyval has opposite convention for polynomial order # flags and at edges pass 0 and then remove them vals_bkp = vals[weights == 0] np.putmask(vals, weights == 0, np.nan) valsnew = generic_filter(vals, polyfit, size=size[0], mode='constant', cval=np.nan) if replace: weights[weights == 0] = 1 weights[np.isnan( valsnew )] = 0 # all the size was flagged cannot extrapolate value else: valsnew[weights == 0] = vals_bkp #print coord['ant'], vals, valsnew elif mode == 'savitzky-golay': vals_bkp = vals[weights == 0] np.putmask(vals, weights == 0, np.nan) valsnew = _savitzky_golay(vals, size[0], degree) if replace: weights[weights == 0] = 1 weights[np.isnan( valsnew )] = 0 # all the size was flagged cannot extrapolate value else: valsnew[weights == 0] = vals_bkp else: logging.error( 'Mode must be: runningmedian, runningpoly, savitzky-golay, median or mean' ) return 1 if log: valsnew = 10**valsnew soltab.setValues(valsnew, selection) if replace: soltab.setValues(weights, selection, weight=True) soltab.flush() soltab.addHistory('SMOOTH (over %s with mode = %s)' % (axesToSmooth, mode)) return 0
def run(soltab, doUnwrap=False, refAnt='', plotName='', ndiv=1): """ Find the structure function from phase solutions of core stations. Parameters ---------- doUnwrap : bool, optional refAnt : str, optional Reference antenna, by default the first. plotName : str, optional Plot file name, by default no plot. ndiv : int, optional """ import numpy as np from losoto.lib_unwrap import unwrap, unwrap_2d logging.info("Find structure function for soltab: " + soltab.name) # input check solType = soltab.getType() if solType != 'phase': logging.warning("Soltab type of " + soltab._v_name + " is of type " + solType + ", should be phase.") return 1 ants = soltab.getAxisValues('ant') if refAnt != '' and refAnt != 'closest' and not refAnt in soltab.getAxisValues( 'ant', ignoreSelection=True): logging.error('Reference antenna ' + refAnt + ' not found. Using: ' + ants[1]) refAnt = ants[1] if refAnt == '' and doUnwrap: logging.error('Unwrap requires reference antenna. Using: ' + ants[1]) refAnt = ants[1] if refAnt == '': refAnt = None soltab.setSelection(ant='CS*', update=True) posAll = soltab.getSolset().getAnt() for vals, weights, coord, selection in soltab.getValuesIter( returnAxes=['freq', 'pol', 'ant', 'time'], weight=True, reference=refAnt): # reorder axes vals = reorderAxes(vals, soltab.getAxesNames(), ['pol', 'ant', 'freq', 'time']) weights = reorderAxes(weights, soltab.getAxesNames(), ['pol', 'ant', 'freq', 'time']) # order positions pos = np.array([list(posAll[ant]) for ant in coord['ant']]) # avg pols vals = np.cos(vals) + 1.j * np.sin(vals) vals = np.nansum(vals, axis=0) vals = np.angle(vals) flags = np.array((weights[0] == 0) | (weights[1] == 0), dtype=bool) # unwrap if doUnwrap: # remove mean to facilitate unwrapping for a, ant in enumerate(coord['ant']): if not (flags[a, :, :] == True).all() and ant != refAnt: logging.debug('Unwrapping: ' + ant) mean = np.angle(np.nanmean(np.exp(1j * vals[a].flatten()))) vals[a] -= mean vals[a] = np.mod(vals[a] + np.pi, 2 * np.pi) - np.pi vals[a, :, :] = unwrap_2d(vals[a, :, :], flags[a, :, :], coord['freq'], coord['time']) logging.debug('Computing differential values...') t1 = np.ma.array(vals, mask=flags) # mask flagged data dph = t1[np.newaxis] - t1[:, np.newaxis] # ant x ant x freq x time D = pos[np.newaxis] - pos[:, np.newaxis] # ant x ant x 3 D2 = np.triu( np.sqrt(np.sum(D**2, axis=-1)) ) # calc distance and keep only uppoer triangle larger than 0 myselect = (D2 > 0) if not doUnwrap: logging.debug('Re-normalising...') dph = np.mod(dph + np.pi, 2 * np.pi) - np.pi avgdph = np.ma.average( dph, axis=2) # avg in freq (can do because is between -pi and pi) #one extra step to remove most(all) phase wraps, phase wraps disturbe the averaging... dph = np.remainder( dph - np.ma.average(avgdph, axis=-1)[:, :, np.newaxis, np.newaxis] + np.pi, 2 * np.pi) + np.ma.average( avgdph, axis=-1)[:, :, np.newaxis, np.newaxis] - np.pi #center around the avg value logging.debug('Computing sructure function...') avgdph = np.ma.average(dph, axis=2) # avg in freq to reduce noise variances = [] pars = [] avgdph = avgdph[ ..., avgdph.shape[-1] % ndiv:] # remove a few timeslots to make the array divisible by np.split for i, avgdphSplit in enumerate(np.split(avgdph, ndiv, axis=-1)): variance = np.ma.var(avgdphSplit, axis=-1) * ( np.average(coord['freq']) / 150.e6)**2 # get time variance and rescale to 150 MHz # linear regression #A = np.ones((2,D2[myselect].shape[0]),dtype=float) #A[1,:] = np.log10(D2[myselect][~variance.mask]) #par = np.dot(np.linalg.inv(np.dot(A,A.T)),np.dot(A,np.log10(variance[myselect]))) mask = variance[myselect].mask A = np.vstack([ np.log10(D2[myselect][~mask]), np.ones(len(D2[myselect][~mask])) ]) par = np.linalg.lstsq(A.T, np.log10(variance[myselect][~mask]))[0] S0 = 10**(-1 * par[1] / par[0]) logging.info(r't%i: beta=%.2f - R_diff=%.2f km' % (i, par[0], S0 / 1.e3)) variances.append(variance) pars.append(par) if plotName != '': if plotName.split('.')[-1] != 'png': plotName += '.png' # add png if not 'matplotlib' in sys.modules: import matplotlib as mpl mpl.use("Agg") import matplotlib.pyplot as plt fig = plt.figure() fig.subplots_adjust(wspace=0) ax = fig.add_subplot(111) ax1 = ax.twinx() for i, variance in enumerate(variances): if len(variances) > 1: color = plt.cm.jet( i / float(len(variances) - 1)) # from 0 to 1 else: color = 'black' ax.plot(D2[myselect] / 1.e3, variance[myselect], marker='o', linestyle='', color=color, markeredgecolor='none', label='T') # regression par = pars[i] x = D2[myselect] S0 = 10**(-1 * par[1] / par[0]) if color == 'black': color = 'red' # in case of single color, use red line that is more visible ax1.plot(x.flatten() / 1.e3, par[0] * np.log10(x.flatten()) + par[1], linestyle='-', color=color, label=r'$\beta=%.2f$ - $R_{\rm diff}=%.2f$ km' % (par[0], S0 / 1.e3)) ax.set_xlabel('Distance (km)') ax.set_ylabel(r'Phase variance @150 MHz (rad$^2$)') ax.set_xscale('log') ax.set_yscale('log') ymin = np.min(variance[myselect]) ymax = np.max(variance[myselect]) ax.set_xlim(xmin=0.1, xmax=3) ax.set_ylim(ymin, ymax) ax1.set_ylim(np.log10(ymin), np.log10(ymax)) ax1.legend(loc='lower right', frameon=False) ax1.set_yticks([]) logging.warning('Save pic: %s' % plotName) plt.savefig(plotName, bbox_inches='tight') return 0
def run(soltab, soltabOut='rotationmeasure000', refAnt='', maxResidual=1.): """ Faraday rotation extraction from either a rotation table or a circular phase (of which the operation get the polarisation difference). Parameters ---------- soltabOut : str, optional output table name (same solset), by deault "rotationmeasure000". refAnt : str, optional Reference antenna, by default the first. maxResidual : float, optional Max average residual in radians before flagging datapoint, by default 1. If 0: no check. """ import numpy as np import scipy.optimize rmwavcomplex = lambda RM, wav, y: abs( np.cos(2. * RM[0] * wav * wav) - np.cos(y)) + abs( np.sin(2. * RM[0] * wav * wav) - np.sin(y)) c = 2.99792458e8 logging.info("Find FR for soltab: " + soltab.name) # input check solType = soltab.getType() if solType == 'phase': returnAxes = ['pol', 'freq', 'time'] if 'RR' in soltab.getAxisValues( 'pol') and 'LL' in soltab.getAxisValues('pol'): coord_rr = np.where(soltab.getAxisValues('pol') == 'RR')[0][0] coord_ll = np.where(soltab.getAxisValues('pol') == 'LL')[0][0] elif 'XX' in soltab.getAxisValues( 'pol') and 'YY' in soltab.getAxisValues('pol'): logging.warning( 'Linear polarization detected, LoSoTo assumes XX->RR and YY->LL.' ) coord_rr = np.where(soltab.getAxisValues('pol') == 'XX')[0][0] coord_ll = np.where(soltab.getAxisValues('pol') == 'YY')[0][0] else: logging.error( "Cannot proceed with Faraday estimation with polarizations: " + str(coord['pol'])) return 1 elif solType == 'rotation': returnAxes = ['freq', 'time'] else: logging.warning("Soltab type of " + soltab._v_name + " is of type " + solType + ", should be phase or rotation. Ignoring.") return 1 ants = soltab.getAxisValues('ant') if refAnt != '' and refAnt != 'closest' and not refAnt in soltab.getAxisValues( 'ant', ignoreSelection=True): logging.error('Reference antenna ' + refAnt + ' not found. Using: ' + ants[1]) refAnt = ants[0] if refAnt == '': refAnt = ants[0] # times and ants needs to be complete or selection is much slower times = soltab.getAxisValues('time') # create new table solset = soltab.getSolset() soltabout = solset.makeSoltab('rotationmeasure', soltabName=soltabOut, axesNames=['ant', 'time'], axesVals=[ants, times], vals=np.zeros((len(ants), len(times))), weights=np.ones((len(ants), len(times)))) soltabout.addHistory('Created by FARADAY operation from %s.' % soltab.name) for vals, weights, coord, selection in soltab.getValuesIter( returnAxes=returnAxes, weight=True, reference=refAnt): if len(coord['freq']) < 10: logging.error( 'Faraday rotation estimation needs at least 10 frequency channels, preferably distributed over a wide range.' ) return 1 # reorder axes vals = reorderAxes(vals, soltab.getAxesNames(), returnAxes) weights = reorderAxes(weights, soltab.getAxesNames(), returnAxes) weights[np.isnan(vals)] = 0. fitrm = np.zeros(len(times)) fitweights = np.ones(len(times)) # all unflagged to start fitrmguess = 0.001 # good guess if not coord['ant'] == refAnt: logging.debug('Working on ant: ' + coord['ant'] + '...') if (weights == 0.).all() == True: logging.warning('Skipping flagged antenna: ' + coord['ant']) fitweights[:] = 0 else: for t, time in enumerate(times): if solType == 'phase': idx = ((weights[coord_rr, :, t] != 0.) & (weights[coord_ll, :, t] != 0.)) freq = np.copy(coord['freq'])[idx] phase_rr = vals[coord_rr, :, t][idx] phase_ll = vals[coord_ll, :, t][idx] # RR-LL to be consistent with BBS/NDPPP phase_diff = ( phase_rr - phase_ll ) # not divide by 2 otherwise jump problem, then later fix this else: # rotation table idx = ((weights[:, t] != 0.) & (weights[:, t] != 0.)) freq = np.copy(coord['freq'])[idx] phase_diff = 2. * vals[:, t][ idx] # a rotation is between -pi and +pi if len(freq) < 20: fitweights[t] = 0 logging.warning( 'No valid data found for Faraday fitting for antenna: ' + coord['ant'] + ' at timestamp ' + str(t)) continue # if more than 1/4 of chans are flagged if (len(idx) - len(freq)) / float(len(idx)) > 1 / 4.: logging.debug( 'High number of filtered out data points for the timeslot %i: %i/%i' % (t, len(idx) - len(freq), len(idx))) wav = c / freq fitresultrm_wav, success = scipy.optimize.leastsq( rmwavcomplex, [fitrmguess], args=(wav, phase_diff)) # fractional residual residual = np.nanmean( np.abs( np.mod((2. * fitresultrm_wav * wav * wav) - phase_diff + np.pi, 2. * np.pi) - np.pi)) # print "t:", t, "result:", fitresultrm_wav, "residual:", residual if maxResidual == 0 or residual < maxResidual: fitrmguess = fitresultrm_wav[0] weight = 1 else: # high residual, flag logging.warning('Bad solution for ant: ' + coord['ant'] + ' (time: ' + str(t) + ', resdiaul: ' + str(residual) + ').') weight = 0 fitrm[t] = fitresultrm_wav[0] fitweights[t] = weight # Debug plot doplot = False if doplot and coord['ant'] == 'RS310LBA' and t % 10 == 0: print("Plotting") if not 'matplotlib' in sys.modules: import matplotlib as mpl mpl.rc('font', size=8) mpl.rc('figure.subplot', left=0.05, bottom=0.05, right=0.95, top=0.95, wspace=0.22, hspace=0.22) mpl.use("Agg") import matplotlib.pyplot as plt fig = plt.figure() fig.subplots_adjust(wspace=0) ax = fig.add_subplot(111) # plot rm fit plotrm = lambda RM, wav: np.mod( (2. * RM * wav * wav) + np.pi, 2. * np.pi ) - np.pi # notice the factor of 2 ax.plot(freq, plotrm(fitresultrm_wav, c / freq[:]), "-", color='purple') if solType == 'phase': ax.plot( freq, np.mod(phase_rr + np.pi, 2. * np.pi) - np.pi, 'ob') ax.plot( freq, np.mod(phase_ll + np.pi, 2. * np.pi) - np.pi, 'og') ax.plot(freq, np.mod(phase_diff + np.pi, 2. * np.pi) - np.pi, '.', color='purple') residual = np.mod( plotrm(fitresultrm_wav, c / freq[:]) - phase_diff + np.pi, 2. * np.pi) - np.pi ax.plot(freq, residual, '.', color='yellow') ax.set_xlabel('freq') ax.set_ylabel('phase') ax.set_ylim(ymin=-np.pi, ymax=np.pi) logging.warning('Save pic: ' + str(t) + '_' + coord['ant'] + '.png') plt.savefig(str(t) + '_' + coord['ant'] + '.png', bbox_inches='tight') del fig soltabout.setSelection(ant=coord['ant'], time=coord['time']) soltabout.setValues(np.expand_dims(fitrm, axis=1)) soltabout.setValues(np.expand_dims(fitweights, axis=1), weight=True) return 0
def run(soltab, axesInPlot, axisInTable='', axisInCol='', axisDiff='', NColFig=0, figSize=[0,0], markerSize=2, minmax=[0,0], log='', \ plotFlag=False, doUnwrap=False, refAnt='', soltabsToAdd='', makeAntPlot=False, makeMovie=False, prefix='', ncpu=0): """ This operation for LoSoTo implements basic plotting WEIGHT: flag-only compliant, no need for weight Parameters ---------- axesInPlot : array of str 1- or 2-element array which says the coordinates to plot (2 for 3D plots). axisInTable : str, optional the axis to plot on a page - e.g. ant to get all antenna's on one file. By default ''. axisInCol : str, optional The axis to plot in different colours - e.g. pol to get correlations with different colors. By default ''. axisDiff : str, optional This must be a len=2 axis and the plot will have the differential value - e.g. 'pol' to plot XX-YY. By default ''. NColFig : int, optional Number of columns in a multi-table image. By default is automatically chosen. figSize : array of int, optional Size of the image [x,y], if one of the values is 0, then it is automatically chosen. By default automatic set. markerSize : int, optional Size of the markers in the 2D plot. By default 2. minmax : array of float, optional Min max value for the independent variable (0 means automatic). By default 0. log : bool, optional Use Log='XYZ' to set which axes to put in Log. By default ''. plotFlag : bool, optional Whether to plot also flags as red points in 2D plots. By default False. doUnwrap : bool, optional Unwrap phases. By default False. refAnt : str, optional Reference antenna for phases. By default None. soltabsToAdd : str, optional Tables to "add" (e.g. 'sol000/tec000'), it works only for tec and clock to be added to phases. By default None. makeAntPlot : bool, optional Make a plot containing antenna coordinates in x,y and in color the value to plot, axesInPlot must be [ant]. By default False. makeMovie : bool, optional Make a movie summing up all the produced plots, by default False. prefix : str, optional Prefix to add before the self-generated filename, by default None. ncpu : int, optional Number of cpus, by default all available. """ import os, random import numpy as np from losoto.lib_unwrap import unwrap, unwrap_2d logging.info("Plotting soltab: " + soltab.name) # input check # str2list if axisInTable == '': axisInTable = [] else: axisInTable = [axisInTable] if axisInCol == '': axisInCol = [] else: axisInCol = [axisInCol] if axisDiff == '': axisDiff = [] else: axisDiff = [axisDiff] if len(set(axisInTable + axesInPlot + axisInCol + axisDiff)) != len(axisInTable + axesInPlot + axisInCol + axisDiff): logging.error('Axis defined multiple times.') return 1 # just because we use lists, check that they are 1-d if len(axisInTable) > 1 or len(axisInCol) > 1 or len(axisDiff) > 1: logging.error( 'Too many TableAxis/ColAxis/DiffAxis, they must be at most one each.' ) return 1 for axis in axesInPlot + axisInCol + axisDiff: if axis not in soltab.getAxesNames(): logging.error('Axis \"' + axis + '\" not found.') return 1 if makeMovie: prefix = prefix + '__tmp__' if os.path.dirname(prefix) != '' and not os.path.exists( os.path.dirname(prefix)): logging.debug('Creating ' + os.path.dirname(prefix) + '.') os.makedirs(os.path.dirname(prefix)) if refAnt == '': refAnt = None elif refAnt != 'closest' and not refAnt in soltab.getAxisValues( 'ant', ignoreSelection=True): logging.error('Reference antenna ' + refAnt + ' not found. Using: ' + soltab.getAxisValues('ant')[1]) refAnt = soltab.getAxisValues('ant')[1] minZ, maxZ = minmax solset = soltab.getSolset() soltabsToAdd = [ solset.getSoltab(soltabName) for soltabName in soltabsToAdd ] cmesh = False if len(axesInPlot) == 2: cmesh = True # not color possible in 3D axisInCol = [] elif len(axesInPlot) != 1: logging.error('Axes must be a len 1 or 2 array.') return 1 # end input check # all axes that are not iterated by anything else axesInFile = soltab.getAxesNames() for axis in axisInTable + axesInPlot + axisInCol + axisDiff: axesInFile.remove(axis) # set subplots scheme if axisInTable != []: Nplots = soltab.getAxisLen(axisInTable[0]) else: Nplots = 1 # prepare antennas coord in makeAntPlot case if makeAntPlot: if axesInPlot != ['ant']: logging.error( 'If makeAntPlot is selected the "Axes" values must be "ant"') return 1 antCoords = [[], []] for ant in soltab.getAxisValues( 'ant'): # select only user-selected antenna in proper order antCoords[0].append(+1 * soltab.getSolset().getAnt()[ant][1]) antCoords[1].append(-1 * soltab.getSolset().getAnt()[ant][0]) else: antCoords = [] datatype = soltab.getType() # start processes for multi-thread mpm = multiprocManager(ncpu, _plot) # compute dataCube size shape = [] if axisInTable != []: shape.append(soltab.getAxisLen(axisInTable[0])) else: shape.append(1) if axisInCol != []: shape.append(soltab.getAxisLen(axisInCol[0])) else: shape.append(1) if cmesh: shape.append(soltab.getAxisLen(axesInPlot[1])) shape.append(soltab.getAxisLen(axesInPlot[0])) else: shape.append(soltab.getAxisLen(axesInPlot[0])) # will contain the data to pass to each thread to make 1 image dataCube = np.ma.zeros(shape=shape, fill_value=np.nan) # cycle on files if makeMovie: pngs = [] # store png filenames for vals, coord, selection in soltab.getValuesIter( returnAxes=axisDiff + axisInTable + axisInCol + axesInPlot): # set filename filename = '' for axis in axesInFile: filename += axis + str(coord[axis]) + '_' filename = filename[:-1] # remove last _ if prefix + filename == '': filename = 'plot' # axis vals (they are always the same, regulat arrays) xvals = coord[axesInPlot[0]] # if plotting antenna - convert to number if axesInPlot[0] == 'ant': xvals = np.arange(len(xvals)) # if plotting time - convert in h/min/s xlabelunit = '' if axesInPlot[0] == 'time': if xvals[-1] - xvals[0] > 3600: xvals = (xvals - xvals[0]) / 3600. # hrs xlabelunit = ' [hr]' elif xvals[-1] - xvals[0] > 60: xvals = (xvals - xvals[0]) / 60. # mins xlabelunit = ' [min]' else: xvals = (xvals - xvals[0]) # sec xlabelunit = ' [s]' # if plotting freq convert in MHz elif axesInPlot[0] == 'freq': xvals = xvals / 1.e6 # MHz xlabelunit = ' [MHz]' if cmesh: # axis vals (they are always the same, regular arrays) yvals = coord[axesInPlot[1]] # same as above but for y-axis if axesInPlot[1] == 'ant': yvals = np.arange(len(yvals)) if len(xvals) <= 1 or len(yvals) <= 1: logging.error( '3D plot must have more then one value per axes.') mpm.wait() return 1 ylabelunit = '' if axesInPlot[1] == 'time': if yvals[-1] - yvals[0] > 3600: yvals = (yvals - yvals[0]) / 3600. # hrs ylabelunit = ' [hr]' elif yvals[-1] - yvals[0] > 60: yvals = (yvals - yvals[0]) / 60. # mins ylabelunit = ' [min]' else: yvals = (yvals - yvals[0]) # sec ylabelunit = ' [s]' elif axesInPlot[1] == 'freq': # Mhz yvals = yvals / 1.e6 ylabelunit = ' [MHz]' else: yvals = None if datatype == 'clock': datatype = 'Clock' ylabelunit = ' (s)' elif datatype == 'tec': datatype = 'dTEC' ylabelunit = ' (TECU)' elif datatype == 'rotationmeasure': datatype = 'dRM' ylabelunit = r' (rad m$^{-2}$)' elif datatype == 'tec3rd': datatype = r'dTEC$_3$' ylabelunit = r' (rad m$^{-3}$)' else: ylabelunit = '' # cycle on tables soltab1Selection = soltab.selection # save global selection and subselect only axex to iterate soltab.selection = selection titles = [] for Ntab, (vals, coord, selection) in enumerate( soltab.getValuesIter(returnAxes=axisDiff + axisInCol + axesInPlot)): # set tile titles.append('') for axis in coord: if axis in axesInFile + axesInPlot + axisInCol: continue titles[Ntab] += axis + ':' + str(coord[axis]) + ' ' titles[Ntab] = titles[Ntab][:-1] # remove last ' ' # cycle on colors soltab2Selection = soltab.selection soltab.selection = selection for Ncol, (vals, weight, coord, selection) in enumerate( soltab.getValuesIter(returnAxes=axisDiff + axesInPlot, weight=True, reference=refAnt)): # differential plot if axisDiff != []: # find ordered list of axis names = [ axis for axis in soltab.getAxesNames() if axis in axisDiff + axesInPlot ] if axisDiff[0] not in names: logging.error("Axis to differentiate (%s) not found." % axisDiff[0]) mpm.wait() return 1 if len(coord[axisDiff[0]]) != 2: logging.error( "Axis to differentiate (%s) has too many values, only 2 is allowed." % axisDiff[0]) mpm.wait() return 1 # find position of interesting axis diff_idx = names.index(axisDiff[0]) # roll to first place vals = np.rollaxis(vals, diff_idx, 0) vals = vals[0] - vals[1] weight = np.rollaxis(weight, diff_idx, 0) weight[0][weight[1] == 0] = 0 weight = weight[0] del coord[axisDiff[0]] # add tables if required (e.g. phase/tec) for soltabToAdd in soltabsToAdd: logging.warning('soltabsToAdd not implemented. Ignoring.') # newCoord = {} # for axisName in coord.keys(): # # prepare selected on present axes # if axisName in soltabToAdd.getAxesNames(): # if type(coord[axisName]) is np.ndarray: # newCoord[axisName] = coord[axisName] # else: # newCoord[axisName] = [coord[axisName]] # avoid being interpreted as regexp, faster # # soltabToAdd.setSelection(**newCoord) # valsAdd = np.squeeze(soltabToAdd.getValues(retAxesVals=False, weight=False, reference=refAnt)) # # # add missing axes # print ('shape:', vals.shape) # for axisName in coord.keys(): # if not axisName in soltabToAdd.getAxesNames(): # # find axis positions # axisPos = soltab.getAxesNames().index(axisName) # # create a new axes for the table to add and duplicate the values # valsAdd = np.expand_dims(valsAdd, axisPos) # print ('shape to add:', valsAdd.shape) # # if soltabToAdd.getType() == 'clock': # valsAdd = 2. * np.pi * valsAdd * coord['freq'] # elif soltabToAdd.getType() == 'tec': # valsAdd = -8.44797245e9 * valsAdd / coord['freq'] # else: # logging.warning('Only Clock or TEC can be added to solutions. Ignoring: '+soltabToAdd.getType()+'.') # continue # # if valsAdd.shape != vals.shape: # logging.error('Cannot combine the table '+soltabToAdd.getType()+' with '+soltab.getType()+'. Wrong shape.') # mpm.wait() # return 1 # # vals += valsAdd # normalize if (soltab.getType() == 'phase' or soltab.getType() == 'scalarphase'): vals = normalize_phase(vals) if (soltab.getType() == 'rotation'): vals = np.mod(vals + np.pi / 2., np.pi) - np.pi / 2. # is user requested axis in an order that is different from h5parm, we need to transpose if cmesh: if soltab.getAxesNames().index( axesInPlot[0]) < soltab.getAxesNames().index( axesInPlot[1]): vals = vals.T weight = weight.T # unwrap if required if (soltab.getType() == 'phase' or soltab.getType() == 'scalarphase') and doUnwrap: if len(axesInPlot) == 1: vals = unwrap(vals) else: flags = np.array((weight == 0), dtype=bool) if not (flags == True).all(): vals = unwrap_2d(vals, flags, coord[axesInPlot[0]], coord[axesInPlot[1]]) dataCube[Ntab, Ncol] = vals sel1 = np.where(weight == 0.) sel2 = np.where(np.isnan(vals)) if cmesh: dataCube[Ntab, Ncol, sel1[0], sel1[1]] = np.ma.masked dataCube[Ntab, Ncol, sel2[0], sel2[1]] = np.ma.masked else: dataCube[Ntab, Ncol, sel1[0]] = np.ma.masked dataCube[Ntab, Ncol, sel2[0]] = np.ma.masked soltab.selection = soltab2Selection ### end cycle on colors # if dataCube too large (> 500 MB) do not go parallel if np.array(dataCube).nbytes > 1024 * 1024 * 500: logging.debug('Big plot, parallel not possible.') _plot(Nplots, NColFig, figSize, markerSize, cmesh, axesInPlot, axisInTable, xvals, yvals, xlabelunit, ylabelunit, datatype, prefix + filename, titles, log, dataCube, minZ, maxZ, plotFlag, makeMovie, antCoords, None) else: mpm.put([ Nplots, NColFig, figSize, markerSize, cmesh, axesInPlot, axisInTable, xvals, yvals, xlabelunit, ylabelunit, datatype, prefix + filename, titles, log, np.ma.copy(dataCube), minZ, maxZ, plotFlag, makeMovie, antCoords ]) if makeMovie: pngs.append(prefix + filename + '.png') soltab.selection = soltab1Selection ### end cycle on tables mpm.wait() if makeMovie: def long_substr(strings): """ Find longest common substring """ substr = '' if len(strings) > 1 and len(strings[0]) > 0: for i in range(len(strings[0])): for j in range(len(strings[0]) - i + 1): if j > len(substr) and all(strings[0][i:i + j] in x for x in strings): substr = strings[0][i:i + j] return substr movieName = long_substr(pngs) assert movieName != '' # need a common prefix, use prefix keyword in case logging.info('Making movie: ' + movieName) # make every movie last 20 sec, min one second per slide fps = np.ceil(len(pngs) / 200.) ss="mencoder -ovc lavc -lavcopts vcodec=mpeg4:vpass=1:vbitrate=6160000:mbd=2:keyint=132:v4mv:vqmin=3:lumi_mask=0.07:dark_mask=0.2:"+\ "mpeg_quant:scplx_mask=0.1:tcplx_mask=0.1:naq -mf type=png:fps="+str(fps)+" -nosound -o "+movieName.replace('__tmp__','')+".mpg mf://"+movieName+"* > mencoder.log 2>&1" os.system(ss) #for png in pngs: os.system('rm '+png) return 0
def run(soltab, axesToClip=None, clipLevel=5., log=False, mode='median'): """ Clip solutions around the median by a factor specified by the user. WEIGHT: flag compliant, putting weights into median is tricky Parameters ---------- axesToClip : list of str axes along which to calculate the median (e.g. [time,freq]). clipLevel : float, optional factor above/below median at which to clip, by default 5. log : bool, optional clip is done in log10 space, by default False. mode : str, optional if "median" then flag at rms*clipLevel times away from the median, if it is "above", then flag all values above clipLevel, if it is "below" then flag all values below clipLevel. By default median. """ import numpy as np def percentFlagged(w): return 100. * (weights.size - np.count_nonzero(weights)) / float( weights.size) logging.info("Clipping soltab: " + soltab.name) # input check if len(axesToClip) == 1 and mode == 'median': logging.error("Please specify axes to clip.") return 1 elif len(axesToClip) == 0: axesToClip = soltab.getAxesNames() if mode != 'median' and mode != 'above' and mode != 'below': logging.error("Mode can be only: median, above or below.") return 1 if mode == 'median' and soltab.getType() == 'amplitude' and not log: logging.warning( 'Amplitude solution tab detected and log=False. Amplitude solution tables should be treated in log space.' ) # some checks for i, axis in enumerate(axesToClip[:]): if axis not in soltab.getAxesNames(): del axesToClip[i] logging.warning('Axis \"' + axis + '\" not found. Ignoring.') for vals, weights, coord, selection in soltab.getValuesIter( returnAxes=axesToClip, weight=True): initPercent = percentFlagged(weights) # skip all flagged if (weights == 0).all(): continue if log: vals = np.log10(vals) if mode == 'median': valmedian = np.nanmedian(vals[(weights != 0)]) rms = np.nanstd(vals[(weights != 0)]) np.putmask(weights, np.abs(vals - valmedian) > rms * clipLevel, 0) elif mode == 'above': np.putmask(weights, vals > clipLevel, 0) elif mode == 'below': np.putmask(weights, vals < clipLevel, 0) # writing back the solutions if log: vals = 10**vals soltab.setValues(weights, selection, weight=True) #print('max', np.max(vals[(weights != 0)])) #print('median', np.nanmedian(vals[(weights != 0)])) logging.debug('Percentage of data flagged (%s): %.3f%% -> %.3f%%' \ % (removeKeys(coord, axesToClip), initPercent, percentFlagged(weights))) soltab.addHistory('CLIP (over %s with %s sigma cut)' % (axesToClip, clipLevel)) soltab.flush() return 0
def run(soltab, soltabOut='tec000', refAnt='', maxResidualFlag=2.5, maxResidualProp=1.): """ Bruteforce TEC extraction from phase solutions. Parameters ---------- soltabOut : str, optional output table name (same solset), by deault "tec". refAnt : str, optional Reference antenna, by default the first. maxResidualFlag : float, optional Max average residual in radians before flagging datapoint, by default 2.5 If 0: no check. maxResidualProp : float, optional Max average residual in radians before stop propagating solutions, by default 1. If 0: no check. """ import numpy as np import scipy.optimize from losoto.lib_unwrap import unwrap_2d def mod(d): return np.mod(d + np.pi, 2. * np.pi) - np.pi drealbrute = lambda d, freq, y: np.sum( np.abs(mod(-8.44797245e9 * d / freq) - y)) # bruteforce dreal = lambda d, freq, y: mod(-8.44797245e9 * d[0] / freq) - y #dreal2 = lambda d, freq, y: mod(-8.44797245e9*d[0]/freq + d[1]) - y #dcomplex = lambda d, freq, y: np.sum( ( np.cos(-8.44797245e9*d/freq) - np.cos(y) )**2 ) + np.sum( ( np.sin(-8.44797245e9*d/freq) - np.sin(y) )**2 ) #def dcomplex( d, freq, y, y_pre, y_post): # return np.sum( ( np.absolute( np.exp(-1j*8.44797245e9*d/freq) - np.exp(1j*y) ) )**2 ) + \ # .5*np.sum( ( np.absolute( np.exp(-1j*8.44797245e9*d/freq) - np.exp(1j*y_pre) ) )**2 ) + \ # .5*np.sum( ( np.absolute( np.exp(-1j*8.44797245e9*d/freq) - np.exp(1j*y_post) ) )**2 ) #dcomplex2 = lambda d, freq, y: abs(np.cos(-8.44797245e9*d[0]/freq + d[1]) - np.cos(y)) + abs(np.sin(-8.44797245e9*d[0]/freq + d[1]) - np.sin(y)) logging.info("Find TEC for soltab: " + soltab.name) # input check solType = soltab.getType() if solType != 'phase': logging.warning("Soltab type of " + soltab._v_name + " is of type " + solType + ", should be phase. Ignoring.") return 1 ants = soltab.getAxisValues('ant') if refAnt != '' and refAnt != 'closest' and not refAnt in soltab.getAxisValues( 'ant', ignoreSelection=True): logging.error('Reference antenna ' + refAnt + ' not found. Using: ' + ants[1]) refAnt = ants[0] if refAnt == '': refAnt = ants[0] # times and ants needs to be complete or selection is much slower times = soltab.getAxisValues('time') # create new table solset = soltab.getSolset() soltabout = solset.makeSoltab(soltype = 'tec', soltabName = soltabOut, axesNames=['ant','time'], \ axesVals=[soltab.getAxisValues(axisName) for axisName in ['ant','time']], \ vals=np.zeros(shape=(soltab.getAxisLen('ant'),soltab.getAxisLen('time'))), \ weights=np.ones(shape=(soltab.getAxisLen('ant'),soltab.getAxisLen('time'))) ) soltabout.addHistory('Created by TEC operation from %s.' % soltab.name) for vals, weights, coord, selection in soltab.getValuesIter( returnAxes=['freq', 'time'], weight=True, reference=refAnt): if len(coord['freq']) < 10: logging.error( 'Delay estimation needs at least 10 frequency channels, preferably distributed over a wide range.' ) return 1 # reorder axes vals = reorderAxes(vals, soltab.getAxesNames(), ['freq', 'time']) weights = reorderAxes(weights, soltab.getAxesNames(), ['freq', 'time']) fitd = np.zeros(len(times)) fitweights = np.ones(len(times)) # all unflagged to start #fitdguess = 0.01 # good guess ranges = (-0.5, 0.5) Ns = 1000 if not coord['ant'] == refAnt: if (weights == 0.).all() == True: logging.warning('Skipping flagged antenna: ' + coord['ant']) fitweights[:] = 0 else: # unwrap 2d timexfreq #flags = np.array((weights == 0), dtype=bool) #vals = unwrap_2d(vals, flags, coord['freq'], coord['time']) for t, time in enumerate(times): # apply flags idx = (weights[:, t] != 0.) freq = np.copy(coord['freq'])[idx] if t == 0: phaseComb_pre = vals[idx, 0] else: phaseComb_pre = vals[idx, t - 1] if t == len(times) - 1: phaseComb_post = vals[idx, -1] else: phaseComb_post = vals[idx, t + 1] phaseComb = vals[idx, t] if len(freq) < 10: fitweights[t] = 0 logging.warning( 'No valid data found for delay fitting for antenna: ' + coord['ant'] + ' at timestamp ' + str(t)) continue # if more than 1/4 of chans are flagged if (len(idx) - len(freq)) / float(len(idx)) > 1 / 3.: logging.debug( 'High number of filtered out data points for the timeslot %i: %i/%i' % (t, len(idx) - len(freq), len(idx))) # least square 2 #fitresultd2, success = scipy.optimize.leastsq(dreal2, [fitdguess,0.], args=(freq, phaseComb)) #numjumps = np.around(fitresultd2[1]/(2*np.pi)) #print 'best jumps:', numjumps #phaseComb -= numjumps * 2*np.pi #fitresultd, success = scipy.optimize.leastsq(dreal, [fitresultd2[0]], args=(freq, phaseComb)) # least square 1 #fitresultd, success = scipy.optimize.leastsq(dreal, [fitdguess], args=(freq, phaseComb)) # hopper #fitresultd = scipy.optimize.basinhopping(dreal, [fitdguess], T=1, minimizer_kwargs={'args':(freq, phaseComb)}) #fitresultd = [fitresultd.x] #best_residual = np.nanmean(np.abs( mod(-8.44797245e9*fitresultd[0]/freq) - phaseComb ) ) #best_residual = np.inf #for jump in [-2,-1,0,1,2]: # fitresultd, success = scipy.optimize.leastsq(dreal, [fitdguess], args=(freq, phaseComb - jump * 2*np.pi)) # print fitresultd # # fractional residual # residual = np.nanmean(np.abs( (-8.44797245e9*fitresultd[0]/freq) - phaseComb - jump * 2*np.pi ) ) # if residual < best_residual: # best_residual = residual # fitd[t] = fitresultd[0] # best_jump = jump # brute force fitresultd = scipy.optimize.brute(drealbrute, ranges=(ranges, ), Ns=Ns, args=(freq, phaseComb)) fitresultd, success = scipy.optimize.leastsq( dreal, fitresultd, args=(freq, phaseComb)) best_residual = np.nanmean( np.abs( mod(-8.44797245e9 * fitresultd[0] / freq) - phaseComb)) fitd[t] = fitresultd[0] if maxResidualFlag == 0 or best_residual < maxResidualFlag: fitweights[t] = 1 if maxResidualProp == 0 or best_residual < maxResidualProp: ranges = (fitresultd[0] - 0.05, fitresultd[0] + 0.05) Ns = 100 else: ranges = (-0.5, 0.5) Ns = 1000 else: # high residual, flag and reset initial guess logging.warning('Bad solution for ant: ' + coord['ant'] + ' (time: ' + str(t) + ', resdiual: ' + str(best_residual) + ').') fitweights[t] = 0 ranges = (-0.5, 0.5) Ns = 1000 # Debug plot doplot = False if doplot and (coord['ant'] == 'RS509LBA' or coord['ant'] == 'RS210LBA') and t % 50 == 0: print("Plotting") if not 'matplotlib' in sys.modules: import matplotlib as mpl mpl.rc('figure.subplot', left=0.05, bottom=0.05, right=0.95, top=0.95, wspace=0.22, hspace=0.22) mpl.use("Agg") import matplotlib.pyplot as plt fig = plt.figure() fig.subplots_adjust(wspace=0) ax = fig.add_subplot(111) # plot rm fit plotd = lambda d, freq: -8.44797245e9 * d / freq ax.plot(freq, plotd(fitresultd[0], freq[:]), "-", color='purple') ax.plot(freq, mod(plotd(fitresultd[0], freq[:])), ":", color='purple') #ax.plot(freq, vals[idx,t], '.b' ) #ax.plot(freq, phaseComb + numjumps * 2*np.pi, 'x', color='purple' ) ax.plot(freq, phaseComb, 'o', color='purple') residual = mod(plotd(fitd[t], freq[:]) - phaseComb) ax.plot(freq, residual, '.', color='orange') ax.set_xlabel('freq') ax.set_ylabel('phase') #ax.set_ylim(ymin=-np.pi, ymax=np.pi) logging.warning('Save pic: ' + str(t) + '_' + coord['ant'] + '.png') plt.savefig(str(t) + '_' + coord['ant'] + '.png', bbox_inches='tight') del fig logging.info('%s: average tec: %f TECU' % (coord['ant'], np.mean(2 * fitd))) # reorder axes back to the original order, needed for setValues soltabout.setSelection(ant=coord['ant']) soltabout.setValues(fitd) soltabout.setValues(fitweights, weight=True) return 0
def run(soltab, soltabOut='tec000', refAnt=''): """ Estimate TEC from ph solutions assuming no wrap for solution at t=0 Parameters ---------- soltabOut : str, optional output table name (same solset), by deault "tec". refAnt : str, optional Reference antenna, by default the first. """ import numpy as np import scipy.optimize from losoto.lib_unwrap import unwrap logging.info("Find TEC for soltab: " + soltab.name) # input check solType = soltab.getType() if solType != 'phase': logging.warning("Soltab type of " + soltab._v_name + " is of type " + solType + ", should be phase. Ignoring.") return 1 ants = soltab.getAxisValues('ant') if refAnt != '' and refAnt != 'closest' and not refAnt in soltab.getAxisValues( 'ant', ignoreSelection=True): logging.warning('Reference antenna ' + refAnt + ' not found. Using: ' + ants[1]) refAnt = ants[0] if refAnt == '': refAnt = ants[0] # times and ants needs to be complete or selection is much slower times = soltab.getAxisValues('time') # create new table solset = soltab.getSolset() soltabout = solset.makeSoltab(soltype = 'tec', soltabName = soltabOut, axesNames=['ant','time','dir'], \ axesVals=[soltab.getAxisValues(axisName) for axisName in ['ant','time','dir']], \ vals=np.zeros(shape=(soltab.getAxisLen('ant'),soltab.getAxisLen('time'),soltab.getAxisLen('dir'))), \ weights=np.ones(shape=(soltab.getAxisLen('ant'),soltab.getAxisLen('time'),soltab.getAxisLen('dir'))) ) soltabout.addHistory('Created by TEC operation from %s.' % soltab.name) for vals, weights, coord, selection in soltab.getValuesIter( returnAxes=['freq', 'time'], weight=True, refAnt=refAnt): assert len(coord['freq']) == 1 # it works with phase at only 1 freq if not coord['ant'] == refAnt: if (weights == 0.).all() == True: logging.warning('Skipping flagged antenna: ' + coord['ant']) fitweights[:] = 0 else: # unwrap vals = np.reshape(unwrap(np.squeeze(vals)), vals.shape) vals *= coord['freq'] / (-8.44797245e9) logging.info('%s: average tec: %f TECU' % (coord['ant'], np.mean(vals))) # reorder axes back to the original order, needed for setValues soltabout.setSelection(ant=coord['ant'], dir=coord['dir']) soltabout.setValues(vals) soltabout.setValues(weights, weight=True) return 0