def RepairObs(D, args=['L1', 'L2', 'L5']): obs = {} for arg in args: argc = arg.replace('L', 'P') if ('2' in arg) else arg.replace('L', 'C') argi = arg + 'lli' try: C = D[argc].values[idel] if np.nansum(np.isfinite(C)) < 600: continue L = D[arg].values[idel] if arg == 'L1': L1 = L Li = D[argi].values[idel] % 2 idlli = np.where(Li == 1)[0] cycle_slip_idx = cycleSlipIdx(L, idlli) # Convert to meters Lmeter = L * c0 / freq[arg[-1]] # Correct cycle slips if cycle_slip_idx.shape[0] > 0: if arg == 'L1': mask = np.isfinite(C) x = np.arange(C.shape[0]) CS = CubicSpline(x[mask], C[mask]) C1 = CS(x) L1meter = cycleSlipRepair(C1, Lmeter, cycle_slip_idx, freq=1, verbose=False, units='m') L = L1meter / c0 * f1 L1 = L else: L = CSnorm(L1, L, cycle_slip_idx, plot=False, verbose=False, frequency=arg[-1]) # Convert back to cycles Ld, Lp = phaseDetrend(L, order=12, polynom=True) x = np.arange(Ld.shape[0]) mask = np.isnan(Lp) Ldp = np.copy(Ld) Ldp[mask] = np.interp(x[mask], x[~mask], Ld[~mask]) Lps = gu.hpf(Ldp) Lps[:15] = np.nan obs[arg] = L obs[argc] = C obs[arg + 'd'] = Ldp obs[arg + 'p'] = Lps obs[arg + 'lli'] = cycle_slip_idx except: pass return obs
def phaseScintillation(data, fc=0.1, filt_order=6, polyfit_order=3, fs=1, skip=20): """ Sebastijan Mrak Standard pocedure to obtain detrended carrier phase scintillation. Input is a list of carrier phase measurements, than this list goes to a process of polynominal detrending and finaly detrended data is high-pass filterd (IIR). """ L = np.nan*np.zeros(data.shape[0]) idx = np.where(np.isfinite(data))[0] L1phi = data[idx] L1_d = uf.phaseDetrend(L1phi, polyfit_order) Y = uf.hpf(L1_d, fc=fc, order=filt_order, fs=fs) L[idx[skip:]] = Y[skip:] return L
def _processTEC(obs, sv, frequency=2): stec = slantTEC(obs['C1'], obs['P2'], obs['L1'], obs['L2'], frequency=frequency) stec += satbias[sv] elevation = D.el.values f = getMappingFunction(elevation, H=200) vtec = stec * f tecd = phaseDetrend(vtec, order=12) x = np.arange(tecd.shape[0]) mask = np.isnan(tecd) tecdp = np.copy(tecd) tecdp[mask] = np.interp(x[mask], x[~mask], tecd[~mask]) tecps = gu.hpf(tecdp) tecps[:15] = np.nan return vtec, tecd, tecps
def _partialProcess(dt, r, x, fs=1, fc=0.1, hpf_order=6, plot_ripple=False, plot_outlier=False): idf = np.isfinite(x) # If there are NaNs in the interval, do a cubic spline. # Max gap is 10 seconds set by the "make ranges routine" # 1. dTEC Split if np.sum(np.isnan(x)) > 0: x0 = np.where(idf)[0] x1 = np.arange(x.size) CSp = CubicSpline(x0, x[idf]) x_cont = CSp(x1) else: x_cont = np.copy(x) # 2. Tec/snr scintillation (high-pass) filtering! tec_hpf = gu.hpf(x_cont, fs=fs, order=hpf_order, fc=fc) tec_hpf[~idf] = np.nan # 3. Remove initial ripple on the scintillation time-series sT_exit, eps = _removeRipple(tec_hpf, E=1.5, L=300, eps=True) if plot_ripple: plt.figure() plt.plot(dt[r[0]:r[1]], tec_hpf, 'b') plt.plot([dt[r[0]], dt[r[1]]], [eps, eps], '--r') if sT_exit != -999: plt.plot(dt[r[0]:r[1]][:sT_exit], tec_hpf[:sT_exit], 'xr') if sT_exit != -999: tec_hpf[:sT_exit] = np.nan tec_hpf_original = np.copy(tec_hpf) # 4. Outlier detection and removal. Still on the scintillation time-series. # 4.1 TEC Scintillation envelope = _runningMax(abs(tec_hpf), N=10) median_envelope = _runningMedian(envelope, N=120) outlier_margin = median_envelope + 5 * np.nanstd(tec_hpf) idoutlier = np.nan_to_num(abs(tec_hpf)) > outlier_margin outlier_mask = np.zeros(tec_hpf.size, dtype=bool) if np.nansum(idoutlier) > 0: outlier_intervals = _mkrngs(tec_hpf, idoutlier, max_length=60, gap_length=10, zero_mean=False) if outlier_intervals.size > 0: if len(outlier_intervals.shape) == 3: outlier_intervals = outlier_intervals[0] for out_ran in outlier_intervals: ratio = np.median( envelope[out_ran[0]:out_ran[1] + 1]) / np.median( median_envelope[out_ran[0]:out_ran[1] + 1]) if np.round(ratio, 1) >= 3: backward = 10 if out_ran[0] > 10 else out_ran[0] forward = 10 if tec_hpf.size - out_ran[1] > 10 else -1 outlier_mask[out_ran[0] - backward:out_ran[1] + 1 + forward] = True if plot_outlier: plt.figure(figsize=[8, 5]) # plt.title('2017-5-28 / Rxi: {}, svi: {}'.format(irx, isv)) plt.plot(dt[r[0]:r[1]], tec_hpf, 'b', label='$\delta TEC_{0.1 Hz}$') plt.plot(dt[r[0]:r[1]], median_envelope, 'g', label='env = <$\widehat{\delta TEC}>|_{10s}$') plt.plot( dt[r[0]:r[1]], outlier_margin, '--r', label='$\epsilon$ = env + 4$\cdot \sigma(\delta TEC)|_{60s}$') plt.plot(dt[r[0]:r[1]], -outlier_margin, '--r') plt.plot(dt[r[0]:r[1]][outlier_mask], tec_hpf[outlier_mask], 'xr') plt.ylabel('$\delta$ TEC [TECu]') plt.xlabel('Time [UTC]') plt.grid(axis='both') plt.legend() tec_hpf[outlier_mask] = np.nan return tec_hpf, tec_hpf_original, outlier_mask
def singleRx(obs, nav, sv='G23', args=['L1','S1'], tlim=None,rawplot=False, sp=True,s4=False,polyfit=False,indicator=False, alt=300,el_mask=30,skip=20,porder=8,tec_ch=2, forder=5,fc=0.1,fs=1): def _plot(x,y,title=''): plt.figure() plt.title(title) plt.plot(x,y,'b') D = xarray.open_dataset(obs, group='OBS') rx_xyz = D.position leap_seconds = uf.getLeapSeconds(nav) obstimes64 = D.time.values times = np.array([Timestamp(t).to_pydatetime() for t in obstimes64]) - \ timedelta(seconds = leap_seconds) # define tlim ie. skip if tlim is not None: s = ((times>=tlim[0]) & (times<=tlim[1])) else: s = np.full(times.shape[0], True, dtype=bool) s[:skip] = False times = times[s] # Get satellite position aer = gpsSatPosition(nav, times, sv=sv, rx_position=rx_xyz, coords='aer') # Elevation mask idel = aer[1] >= el_mask times = times[idel] # times to output dict: Y Y = {'times': times} Y['rx_xyz'] = rx_xyz Y['az'] = aer[0][idel] Y['el'] = aer[1][idel] for arg in args: if not arg[1:] == 'TEC': X = D.sel(sv=sv)[arg].values[s][idel] # To dict Y[arg] = X # Indicators? if indicator: try: if arg[0] == 'L' and arg[-1] != '5': argi = arg + 'lli' elif arg[0] == 'C' or arg[0] == 'P' or arg == 'L5': argi = arg + 'ssi' else: argi = '' Xi = D.sel(sv=sv)[argi].values[s:][idel] Y[argi] = Xi except Exception as e: print (e) if rawplot: _plot(times,X,arg) if arg[0] == 'L' or arg[0] == 'C': if arg == 'C1': X = X * f1 / c0 # To cycles Xd = uf.phaseDetrend(X, order=porder) if polyfit: # To dict Y[arg+'polyfit'] = Xd if rawplot: if rawplot: _plot(times,Xd,arg+'_detrend_poly') if sp: Xy = uf.hpf(Xd, order=forder,fc=fc,plot=False,fs=fs) # To dict Y['sp'] = Xy if rawplot: _plot(times[100:],Xy[100:],arg+'_scint') if arg[0] == 'S': if s4: Xy = AmplitudeScintillationIndex(X,60) # To dict Y['s4'] = Xy if rawplot: _plot(times,Xy,arg+'_S4') if arg[1:] == 'TEC': if tec_ch == 2: C1 = D.sel(sv=sv)['C1'].values[s][idel] L1 = D.sel(sv=sv)['L1'].values[s][idel] C2 = D.sel(sv=sv)['P2'].values[s][idel] L2 = D.sel(sv=sv)['L2'].values[s][idel] elif tec_ch == 5: C1 = D.sel(sv=sv)['C1'].values[s][idel] L1 = D.sel(sv=sv)['L1'].values[s][idel] C2 = D.sel(sv=sv)['C5'].values[s][idel] L2 = D.sel(sv=sv)['L5'].values[s][idel] sTEC = getPhaseCorrTEC(L1, L2, C1, C2,channel=tec_ch) sTEC = sTEC - np.nanmin(sTEC) if arg == 'sTEC': # To dict Y[arg] = sTEC if rawplot: _plot(times, sTEC,title=arg) elif arg == 'vTEC': vTEC = getVerticalTEC(sTEC,aer[1][idel],alt) # To dict Y[arg] = vTEC if rawplot: _plot(times, vTEC, title=arg) # return dict with data/args return Y
f = _plotDuo(dt, data['C1polyfit'] * f1 / c0 / 2 + 5, data['L1polyfit'] - 5, xlim=xlim, ylabel1='$\delta \Phi$ [cycle]', ylabel2='$\delta Range$ [cycle]', xlabel='(2017-10-07) time [UT]', ylim1=[-30, 40], ylim2=[-30, 15], formatter='%H:%M', c1='r', c2='b', lw1=2, lw2=2) L1scint = gu.hpf(data['sp'], fc=0.1, order=3) #az = [] #el = [] #for i in range(len(data)): # az.append(data[i]['az']) # el.append(data[i]['el']) #f = _plotOrbit(az,el) #f.savefig(savefolder + 'orbits' + '.png', dpi=300) ################################ PLOT ######################################### #_plot(dt, data['C1']) #_plot(dt, data['L1']) #_plot(dt, data['C1polyfit']) #_plot(dt, data['L1polyfit']) # --------------------------------------------------------------------------- #
S1 = Ds['S1'][skip:].values aer = pyGnss.getSatellitePosition(rx_xyz, sv, obstimes_dt, nav, cs='aer', dtype='georinex') idel = (aer[1] >= el_mask) obstimes = obstimes64[idel] L1 = L1[idel] S1 = S1[idel] L1d = pyGnss.phaseDetrend(L1, order=porder) L1s = gu.hpf(L1d, order=forder, fc=fc, plot=False, fs=fs) L1s[:20] = np.nan S1d = pyGnss.phaseDetrend(S1, order=3) S1dd, Td = gu.lpf(S1d, fc=0.005, group_delay=True) S1hp = gu.hpf(S1d, order=forder, fc=0.01, plot=False, fs=fs) idin = np.isin(obstimes64, obstimes) T[idin] = obstimes T = np.asarray(T, dtype='datetime64[ns]') Ls[idin] = L1s Ss[idin] = S1dd plt.plot(obstimes, S1d) plt.plot(obstimes - int(Td * 1e9), S1dd) plt.plot(obstimes, S1hp, 'r')
def partialProcess(dt, r, x, fs=1, fc=0.1, hpf_order=6, remove_outliers=False, plot_ripple=False, plot_outlier=False): idf = np.isfinite(x) # If there are NaNs in the interval, do a cubic spline. # Max gap is 10 seconds set by the "make ranges routine" # 1. dTEC Split if np.sum(np.isnan(x)) > 0: x0 = np.where(idf)[0] x1 = np.arange(x.size) CSp = CubicSpline(x0, x[idf]) x_cont = CSp(x1) else: x_cont = np.copy(x) # 2. Tec/snr scintillation (high-pass) filtering! tec_hpf = gu.hpf(x_cont, fs=fs, order=hpf_order, fc=fc) tec_hpf[~idf] = np.nan # 3. Remove initial ripple on the scintillation time-series sT_exit, eps = removeRipple(tec_hpf, E=1.5, L=300, eps=True) if plot_ripple: plt.figure() plt.plot(dt[r[0]:r[1]], tec_hpf, 'b') plt.plot([dt[r[0]], dt[r[1]]], [eps, eps], '--r') if sT_exit != -999: plt.plot(dt[r[0]:r[1]][:sT_exit], tec_hpf[:sT_exit], 'xr') if sT_exit != -999: tec_hpf[:sT_exit] = np.nan # 4. Outlier detection and removal. Still on the scintillation time-series. # 4.1 TEC Scintillation if remove_outliers: envelope = gu.runningMax(abs(tec_hpf), N=10) median_envelope = gu.runningMedian(envelope, N=120) outlier_margin = median_envelope + 5 * np.nanstd(tec_hpf) idoutlier = np.nan_to_num(abs(tec_hpf)) > outlier_margin tec_hpf[idoutlier] = np.nan if np.nansum(idoutlier) > 0: if plot_outlier: plt.figure(figsize=[8, 5]) plt.plot(dt[r[0]:r[1]], tec_hpf, 'b', label='$\delta TEC_{0.1 Hz}$') plt.plot(dt[r[0]:r[1]], median_envelope, 'g', label='env = <$\widehat{\delta TEC}>|_{10s}$') plt.plot( dt[r[0]:r[1]], outlier_margin, '--r', label= '$\epsilon$ = env + 4$\cdot \sigma(\delta TEC)|_{60s}$') plt.plot(dt[r[0]:r[1]], -outlier_margin, '--r') plt.plot(dt[r[0]:r[1]][~idoutlier], tec_hpf[~idoutlier], 'xr') plt.ylabel('$\delta$ TEC [TECu]') plt.xlabel('Time [UTC]') plt.grid(axis='both') plt.legend() return tec_hpf