def RepairObs(D, args=['L1', 'L2', 'L5']):
obs = {}
for arg in args:
argc = arg.replace('L', 'P') if ('2' in arg) else arg.replace('L', 'C')
argi = arg + 'lli'
try:
C = D[argc].values[idel]
if np.nansum(np.isfinite(C)) < 600: continue
L = D[arg].values[idel]
if arg == 'L1': L1 = L
Li = D[argi].values[idel] % 2
idlli = np.where(Li == 1)[0]
cycle_slip_idx = cycleSlipIdx(L, idlli)
# Convert to meters
Lmeter = L * c0 / freq[arg[-1]]
# Correct cycle slips
if cycle_slip_idx.shape[0] > 0:
if arg == 'L1':
mask = np.isfinite(C)
x = np.arange(C.shape[0])
CS = CubicSpline(x[mask], C[mask])
C1 = CS(x)
L1meter = cycleSlipRepair(C1,
Lmeter,
cycle_slip_idx,
freq=1,
verbose=False,
units='m')
L = L1meter / c0 * f1
L1 = L
else:
L = CSnorm(L1,
L,
cycle_slip_idx,
plot=False,
verbose=False,
frequency=arg[-1])
# Convert back to cycles
Ld, Lp = phaseDetrend(L, order=12, polynom=True)
x = np.arange(Ld.shape[0])
mask = np.isnan(Lp)
Ldp = np.copy(Ld)
Ldp[mask] = np.interp(x[mask], x[~mask], Ld[~mask])
Lps = gu.hpf(Ldp)
Lps[:15] = np.nan
obs[arg] = L
obs[argc] = C
obs[arg + 'd'] = Ldp
obs[arg + 'p'] = Lps
obs[arg + 'lli'] = cycle_slip_idx
except:
pass
return obs
def phaseScintillation(data, fc=0.1, filt_order=6, polyfit_order=3, fs=1, skip=20):
"""
Sebastijan Mrak
Standard pocedure to obtain detrended carrier phase scintillation. Input
is a list of carrier phase measurements, than this list goes to a process of
polynominal detrending and finaly detrended data is high-pass filterd (IIR).
"""
L = np.nan*np.zeros(data.shape[0])
idx = np.where(np.isfinite(data))[0]
L1phi = data[idx]
L1_d = uf.phaseDetrend(L1phi, polyfit_order)
Y = uf.hpf(L1_d, fc=fc, order=filt_order, fs=fs)
L[idx[skip:]] = Y[skip:]
return L
def _processTEC(obs, sv, frequency=2):
stec = slantTEC(obs['C1'],
obs['P2'],
obs['L1'],
obs['L2'],
frequency=frequency)
stec += satbias[sv]
elevation = D.el.values
f = getMappingFunction(elevation, H=200)
vtec = stec * f
tecd = phaseDetrend(vtec, order=12)
x = np.arange(tecd.shape[0])
mask = np.isnan(tecd)
tecdp = np.copy(tecd)
tecdp[mask] = np.interp(x[mask], x[~mask], tecd[~mask])
tecps = gu.hpf(tecdp)
tecps[:15] = np.nan
return vtec, tecd, tecps
def _partialProcess(dt,
r,
x,
fs=1,
fc=0.1,
hpf_order=6,
plot_ripple=False,
plot_outlier=False):
idf = np.isfinite(x)
# If there are NaNs in the interval, do a cubic spline.
# Max gap is 10 seconds set by the "make ranges routine"
# 1. dTEC Split
if np.sum(np.isnan(x)) > 0:
x0 = np.where(idf)[0]
x1 = np.arange(x.size)
CSp = CubicSpline(x0, x[idf])
x_cont = CSp(x1)
else:
x_cont = np.copy(x)
# 2. Tec/snr scintillation (high-pass) filtering!
tec_hpf = gu.hpf(x_cont, fs=fs, order=hpf_order, fc=fc)
tec_hpf[~idf] = np.nan
# 3. Remove initial ripple on the scintillation time-series
sT_exit, eps = _removeRipple(tec_hpf, E=1.5, L=300, eps=True)
if plot_ripple:
plt.figure()
plt.plot(dt[r[0]:r[1]], tec_hpf, 'b')
plt.plot([dt[r[0]], dt[r[1]]], [eps, eps], '--r')
if sT_exit != -999:
plt.plot(dt[r[0]:r[1]][:sT_exit], tec_hpf[:sT_exit], 'xr')
if sT_exit != -999:
tec_hpf[:sT_exit] = np.nan
tec_hpf_original = np.copy(tec_hpf)
# 4. Outlier detection and removal. Still on the scintillation time-series.
# 4.1 TEC Scintillation
envelope = _runningMax(abs(tec_hpf), N=10)
median_envelope = _runningMedian(envelope, N=120)
outlier_margin = median_envelope + 5 * np.nanstd(tec_hpf)
idoutlier = np.nan_to_num(abs(tec_hpf)) > outlier_margin
outlier_mask = np.zeros(tec_hpf.size, dtype=bool)
if np.nansum(idoutlier) > 0:
outlier_intervals = _mkrngs(tec_hpf,
idoutlier,
max_length=60,
gap_length=10,
zero_mean=False)
if outlier_intervals.size > 0:
if len(outlier_intervals.shape) == 3:
outlier_intervals = outlier_intervals[0]
for out_ran in outlier_intervals:
ratio = np.median(
envelope[out_ran[0]:out_ran[1] + 1]) / np.median(
median_envelope[out_ran[0]:out_ran[1] + 1])
if np.round(ratio, 1) >= 3:
backward = 10 if out_ran[0] > 10 else out_ran[0]
forward = 10 if tec_hpf.size - out_ran[1] > 10 else -1
outlier_mask[out_ran[0] - backward:out_ran[1] + 1 +
forward] = True
if plot_outlier:
plt.figure(figsize=[8, 5])
# plt.title('2017-5-28 / Rxi: {}, svi: {}'.format(irx, isv))
plt.plot(dt[r[0]:r[1]],
tec_hpf,
'b',
label='$\delta TEC_{0.1 Hz}$')
plt.plot(dt[r[0]:r[1]],
median_envelope,
'g',
label='env = <$\widehat{\delta TEC}>|_{10s}$')
plt.plot(
dt[r[0]:r[1]],
outlier_margin,
'--r',
label='$\epsilon$ = env + 4$\cdot \sigma(\delta TEC)|_{60s}$')
plt.plot(dt[r[0]:r[1]], -outlier_margin, '--r')
plt.plot(dt[r[0]:r[1]][outlier_mask], tec_hpf[outlier_mask], 'xr')
plt.ylabel('$\delta$ TEC [TECu]')
plt.xlabel('Time [UTC]')
plt.grid(axis='both')
plt.legend()
tec_hpf[outlier_mask] = np.nan
return tec_hpf, tec_hpf_original, outlier_mask
def singleRx(obs, nav, sv='G23', args=['L1','S1'], tlim=None,rawplot=False,
sp=True,s4=False,polyfit=False,indicator=False,
alt=300,el_mask=30,skip=20,porder=8,tec_ch=2,
forder=5,fc=0.1,fs=1):
def _plot(x,y,title=''):
plt.figure()
plt.title(title)
plt.plot(x,y,'b')
D = xarray.open_dataset(obs, group='OBS')
rx_xyz = D.position
leap_seconds = uf.getLeapSeconds(nav)
obstimes64 = D.time.values
times = np.array([Timestamp(t).to_pydatetime() for t in obstimes64]) - \
timedelta(seconds = leap_seconds)
# define tlim ie. skip
if tlim is not None:
s = ((times>=tlim[0]) & (times<=tlim[1]))
else:
s = np.full(times.shape[0], True, dtype=bool)
s[:skip] = False
times = times[s]
# Get satellite position
aer = gpsSatPosition(nav, times, sv=sv, rx_position=rx_xyz, coords='aer')
# Elevation mask
idel = aer[1] >= el_mask
times = times[idel]
# times to output dict: Y
Y = {'times': times}
Y['rx_xyz'] = rx_xyz
Y['az'] = aer[0][idel]
Y['el'] = aer[1][idel]
for arg in args:
if not arg[1:] == 'TEC':
X = D.sel(sv=sv)[arg].values[s][idel]
# To dict
Y[arg] = X
# Indicators?
if indicator:
try:
if arg[0] == 'L' and arg[-1] != '5':
argi = arg + 'lli'
elif arg[0] == 'C' or arg[0] == 'P' or arg == 'L5':
argi = arg + 'ssi'
else:
argi = ''
Xi = D.sel(sv=sv)[argi].values[s:][idel]
Y[argi] = Xi
except Exception as e:
print (e)
if rawplot:
_plot(times,X,arg)
if arg[0] == 'L' or arg[0] == 'C':
if arg == 'C1':
X = X * f1 / c0 # To cycles
Xd = uf.phaseDetrend(X, order=porder)
if polyfit:
# To dict
Y[arg+'polyfit'] = Xd
if rawplot:
if rawplot:
_plot(times,Xd,arg+'_detrend_poly')
if sp:
Xy = uf.hpf(Xd, order=forder,fc=fc,plot=False,fs=fs)
# To dict
Y['sp'] = Xy
if rawplot:
_plot(times[100:],Xy[100:],arg+'_scint')
if arg[0] == 'S':
if s4:
Xy = AmplitudeScintillationIndex(X,60)
# To dict
Y['s4'] = Xy
if rawplot:
_plot(times,Xy,arg+'_S4')
if arg[1:] == 'TEC':
if tec_ch == 2:
C1 = D.sel(sv=sv)['C1'].values[s][idel]
L1 = D.sel(sv=sv)['L1'].values[s][idel]
C2 = D.sel(sv=sv)['P2'].values[s][idel]
L2 = D.sel(sv=sv)['L2'].values[s][idel]
elif tec_ch == 5:
C1 = D.sel(sv=sv)['C1'].values[s][idel]
L1 = D.sel(sv=sv)['L1'].values[s][idel]
C2 = D.sel(sv=sv)['C5'].values[s][idel]
L2 = D.sel(sv=sv)['L5'].values[s][idel]
sTEC = getPhaseCorrTEC(L1, L2, C1, C2,channel=tec_ch)
sTEC = sTEC - np.nanmin(sTEC)
if arg == 'sTEC':
# To dict
Y[arg] = sTEC
if rawplot:
_plot(times, sTEC,title=arg)
elif arg == 'vTEC':
vTEC = getVerticalTEC(sTEC,aer[1][idel],alt)
# To dict
Y[arg] = vTEC
if rawplot:
_plot(times, vTEC, title=arg)
# return dict with data/args
return Y
f = _plotDuo(dt,
data['C1polyfit'] * f1 / c0 / 2 + 5,
data['L1polyfit'] - 5,
xlim=xlim,
ylabel1='$\delta \Phi$ [cycle]',
ylabel2='$\delta Range$ [cycle]',
xlabel='(2017-10-07) time [UT]',
ylim1=[-30, 40],
ylim2=[-30, 15],
formatter='%H:%M',
c1='r',
c2='b',
lw1=2,
lw2=2)
L1scint = gu.hpf(data['sp'], fc=0.1, order=3)
#az = []
#el = []
#for i in range(len(data)):
# az.append(data[i]['az'])
# el.append(data[i]['el'])
#f = _plotOrbit(az,el)
#f.savefig(savefolder + 'orbits' + '.png', dpi=300)
################################ PLOT #########################################
#_plot(dt, data['C1'])
#_plot(dt, data['L1'])
#_plot(dt, data['C1polyfit'])
#_plot(dt, data['L1polyfit'])
# --------------------------------------------------------------------------- #
S1 = Ds['S1'][skip:].values
aer = pyGnss.getSatellitePosition(rx_xyz,
sv,
obstimes_dt,
nav,
cs='aer',
dtype='georinex')
idel = (aer[1] >= el_mask)
obstimes = obstimes64[idel]
L1 = L1[idel]
S1 = S1[idel]
L1d = pyGnss.phaseDetrend(L1, order=porder)
L1s = gu.hpf(L1d, order=forder, fc=fc, plot=False, fs=fs)
L1s[:20] = np.nan
S1d = pyGnss.phaseDetrend(S1, order=3)
S1dd, Td = gu.lpf(S1d, fc=0.005, group_delay=True)
S1hp = gu.hpf(S1d, order=forder, fc=0.01, plot=False, fs=fs)
idin = np.isin(obstimes64, obstimes)
T[idin] = obstimes
T = np.asarray(T, dtype='datetime64[ns]')
Ls[idin] = L1s
Ss[idin] = S1dd
plt.plot(obstimes, S1d)
plt.plot(obstimes - int(Td * 1e9), S1dd)
plt.plot(obstimes, S1hp, 'r')
def partialProcess(dt,
r,
x,
fs=1,
fc=0.1,
hpf_order=6,
remove_outliers=False,
plot_ripple=False,
plot_outlier=False):
idf = np.isfinite(x)
# If there are NaNs in the interval, do a cubic spline.
# Max gap is 10 seconds set by the "make ranges routine"
# 1. dTEC Split
if np.sum(np.isnan(x)) > 0:
x0 = np.where(idf)[0]
x1 = np.arange(x.size)
CSp = CubicSpline(x0, x[idf])
x_cont = CSp(x1)
else:
x_cont = np.copy(x)
# 2. Tec/snr scintillation (high-pass) filtering!
tec_hpf = gu.hpf(x_cont, fs=fs, order=hpf_order, fc=fc)
tec_hpf[~idf] = np.nan
# 3. Remove initial ripple on the scintillation time-series
sT_exit, eps = removeRipple(tec_hpf, E=1.5, L=300, eps=True)
if plot_ripple:
plt.figure()
plt.plot(dt[r[0]:r[1]], tec_hpf, 'b')
plt.plot([dt[r[0]], dt[r[1]]], [eps, eps], '--r')
if sT_exit != -999:
plt.plot(dt[r[0]:r[1]][:sT_exit], tec_hpf[:sT_exit], 'xr')
if sT_exit != -999:
tec_hpf[:sT_exit] = np.nan
# 4. Outlier detection and removal. Still on the scintillation time-series.
# 4.1 TEC Scintillation
if remove_outliers:
envelope = gu.runningMax(abs(tec_hpf), N=10)
median_envelope = gu.runningMedian(envelope, N=120)
outlier_margin = median_envelope + 5 * np.nanstd(tec_hpf)
idoutlier = np.nan_to_num(abs(tec_hpf)) > outlier_margin
tec_hpf[idoutlier] = np.nan
if np.nansum(idoutlier) > 0:
if plot_outlier:
plt.figure(figsize=[8, 5])
plt.plot(dt[r[0]:r[1]],
tec_hpf,
'b',
label='$\delta TEC_{0.1 Hz}$')
plt.plot(dt[r[0]:r[1]],
median_envelope,
'g',
label='env = <$\widehat{\delta TEC}>|_{10s}$')
plt.plot(
dt[r[0]:r[1]],
outlier_margin,
'--r',
label=
'$\epsilon$ = env + 4$\cdot \sigma(\delta TEC)|_{60s}$')
plt.plot(dt[r[0]:r[1]], -outlier_margin, '--r')
plt.plot(dt[r[0]:r[1]][~idoutlier], tec_hpf[~idoutlier], 'xr')
plt.ylabel('$\delta$ TEC [TECu]')
plt.xlabel('Time [UTC]')
plt.grid(axis='both')
plt.legend()
return tec_hpf
