def RxbiasEstimator(RxBtime=0, TECref=0, t=0, sTEC=0, sat_bias=0, el=[], IPPalt=350):
Fm = pyGnss.getMappingFunction(el, IPPalt)
idt = abs(t - RxBtime).argmin()
Fmtt = Fm[idt]
sTECt = sTEC[idt]
rx_bias = TECref/Fmtt - sTECt - sat_bias
return rx_bias
def process(fn, odir=None, cfg=None, log=None, irxforce=None, TH=0):
############################### Open data ##################################
if irxforce is not None:
irxforce = int(irxforce)
if odir is None:
odir = os.path.split(fn)[0] + separator
if cfg is None:
plot_ripple = 0
plot_outlier = 0
savefig = 1
figfolder = os.path.join(odir, 'scint_plots' + separator)
plot = 0
fs = 1
fc = 0.1
hpf_order = 6
H = 350
else:
assert os.path.splitext(cfg)[1] in ('.yml', '.yaml')
stream = yaml.load(open(cfg, 'r'))
plot_ripple = stream.get('plot_ripple')
plot_outlier = stream.get('plot_outlier')
plot = stream.get('plot')
savefig = stream.get('savefig')
figfolder = stream.get('figfolder')
if figfolder is None:
figfolder = os.path.join(odir, 'scint_plots' + separator)
fs = stream.get('fs')
fc = stream.get('fc')
hpf_order = stream.get('hpf_order')
H = stream.get('alt_km')
# Output file
if odir is None:
odir = os.path.split(fn)[0] + separator
ofn = odir + 'ix_' + '_'.join(
os.path.split(fn)[1].split('.')[:2]) + '_{}km_{}th.h5'.format(
H,
str(TH).replace('.', ''))
# Dealing with duplicate file names
if os.path.exists(ofn):
head = os.path.splitext(ofn)[0]
c = 0
while os.path.exists(ofn):
try:
c = int(os.path.splitext(ofn)[0].split('_')[-1])
c += 1
except:
c += 1
ofn = head + '_' + str(c) + '.h5'
if log:
logfn = os.path.splitext(ofn)[0] + '.log'
LOG = open(logfn, 'w')
LOG.close()
# Open data file
f = h5py.File(fn, 'r')
time = f['obstimes'][:]
dt = np.array([datetime.utcfromtimestamp(t) for t in time])
if irxforce is None:
rnx = f['el'].shape[2]
else:
rnx = 1
#rnx = 5
svx = f['el'].shape[1]
rxpall = f['rx_positions'][:]
# New arrays
ipp = np.nan * np.ones(
(dt.size, svx, rnx, 2)) # [time, SV, Rx, [lat, lon]]
sigma_tec = np.nan * np.ones((dt.size, svx, rnx))
snr4 = np.nan * np.ones((dt.size, svx, rnx))
s4 = np.nan * np.ones((dt.size, svx, rnx))
roti = np.nan * np.ones((dt.size, svx, rnx))
if plot:
rot = np.nan * np.ones((dt.size, svx, rnx))
# tec_hpf = np.nan * np.ones((dt.size, svx, rnx))
# Bookkeeping
scint_limits = np.nan * np.zeros((rnx, 2))
receiver_std = np.nan * np.zeros((rnx, 2))
receiver_std_median = np.nan * np.zeros((rnx, 2))
for irx in range(rnx):
if log:
with open(logfn, 'a') as LOG:
LOG.write('Processing Rx/all #{}/{}\n'.format(irx + 1, rnx))
LOG.close()
else:
print('Processing Rx/all #{}/{}'.format(irx + 1, rnx))
if plot:
tec_hpf_all = np.nan * np.ones((dt.size, svx))
snr_hpf_all = np.nan * np.ones((dt.size, svx))
sigma_tec_all = np.nan * np.ones((dt.size, svx))
snr4_all = np.nan * np.ones((dt.size, svx))
# Reset to zero for each iteration
tec_outliers = np.zeros((dt.size, svx), dtype=bool)
snr_outliers = np.zeros((dt.size, svx), dtype=bool)
try:
for isv in range(svx):
try:
if irxforce is not None:
el = f['el'][:, isv, irxforce]
az = f['az'][:, isv, irxforce]
res = f['res'][:, isv, irxforce]
snr = f['snr'][:, isv, irxforce]
rxp = rxpall[irxforce]
else:
el = f['el'][:, isv, irx]
az = f['az'][:, isv, irx]
res = f['res'][:, isv, irx]
snr = f['snr'][:, isv, irx]
rxp = rxpall[irx]
# Compute location of the IPP
lat, lon = _toLLT(rxp, az=az, el=el, H=H)
# Get Mapping Function
F = pyGnss.getMappingFunction(el, h=350)
# Stack into the output array
ipp[:, isv, irx, 0] = lat
ipp[:, isv, irx, 1] = lon
except Exception as e:
if log:
with open(logfn, 'a') as LOG:
LOG.write('{}\n'.format(e))
LOG.close()
else:
print(e)
# Check for a minum length of valid observables == 30 min
if np.nansum(np.isfinite(res)) < 30 * 60:
continue
tec_hpf_copy = np.nan * np.copy(res)
rot_copy = np.nan * np.copy(res)
roti_copy = np.nan * np.copy(res)
sigma_tec_copy = np.nan * np.copy(res)
snr4_copy = np.nan * np.copy(snr)
s4_copy = np.nan * np.copy(snr)
tec_hpf_original = np.nan * np.copy(res)
snr_hpf_original = np.nan * np.copy(res)
# 0.0 To ranges: Multipe visits of a satellite per day.
# New interval for a gap bigger than 10 samples.
# Minimum length of interval is 30 minutes
# Create empty arrays
idf_tec = np.isfinite(res)
idf_snr = np.isfinite(snr)
# 0.1 Do for TEC
try:
tec_ranges = ranges(res,
idf_tec,
min_gap=10,
gap_length=10,
min_length=30 * 60,
zero_mean=True)
except:
tec_ranges = np.array([])
try:
snr_ranges = ranges(snr,
idf_snr,
min_gap=10,
gap_length=10,
min_length=30 * 60)
except:
snr_ranges = np.array([])
# Process TEC per intervals
if tec_ranges.size > 0:
for ith_range, r in enumerate(tec_ranges):
# Remove to short ranges if accidentaly do occur
if np.diff(r) < 10: continue
try:
chunk = res[r[0]:r[1]]
tec_hpf, tec_hpf_original[
r[0]:r[1]], tec_mask = _partialProcess(
dt,
r,
chunk,
fs=fs,
fc=fc,
hpf_order=hpf_order,
plot_ripple=plot_ripple,
plot_outlier=plot_outlier)
tec_outliers[r[0]:r[1], isv] = tec_mask
sigma_tec_copy[r[0]:r[1]] = scint.sigmaTEC(tec_hpf,
N=60)
tec_hpf_copy[r[0]:r[1]] = tec_hpf
tmp_diff = np.diff(chunk)
tmp_diff[tec_mask[1:]] = np.nan
rot_copy[r[0] + 1:r[1]] = tmp_diff
roti_copy[r[0] + 1:r[1]] = scint.sigmaTEC(
np.diff(chunk), N=60)
except Exception as e:
if log:
with open(logfn, 'a') as LOG:
LOG.write('{}\n'.format(e))
LOG.close()
else:
print(e)
if snr_ranges.size > 0:
for ith_range, r in enumerate(snr_ranges):
# Remove to short ranges if accidentaly do occur
if np.diff(r) < 60: continue
try:
Schunk = snr[r[0]:r[1]].astype(np.float64)
snr_hpf, snr_hpf_original[
r[0]:r[1]], snr_mask = _partialProcess(
dt,
r,
Schunk,
fs=fs,
fc=fc,
hpf_order=hpf_order,
plot_ripple=plot_ripple,
plot_outlier=plot_outlier)
snr_outliers[r[0]:r[1], isv] = snr_mask
snr4_copy[r[0]:r[1]] = scint.sigmaTEC(snr_hpf,
N=60)
s4_copy[
r[0]:r[1]] = scint.AmplitudeScintillationIndex(
10**(Schunk / 10), 60)
except Exception as e:
if log:
with open(logfn, 'a') as LOG:
LOG.write('{}\n'.format(e))
LOG.close()
else:
print(e)
# Save scintillation indices
sigma_tec[:, isv, irx] = sigma_tec_copy
snr4[:, isv, irx] = (snr4_copy * (F**0.9))
s4[:, isv, irx] = (s4_copy * (F**0.9))
roti[:, isv, irx] = roti_copy
if plot:
rot[:, isv, irx] = rot_copy
tec_hpf_all[:, isv] = tec_hpf_original
snr_hpf_all[:, isv] = snr_hpf_original
sigma_tec_all[:, isv] = sigma_tec_copy
snr4_all[:, isv] = snr4_copy
# 4. Define the scintillation event masks per receiver
# 4.1 Define limits
# sigma_tec: limit ------------------------------------------------------ #
st_std = np.nanstd(sigma_tec[:, :, irx])
st_std_tec = np.nanstd(sigma_tec[:, :, irx])
st_hat = np.nanmedian(sigma_tec[:, :, irx])
st_eps = TH * st_hat # + st_std
# SNR4 limit
s4_std = np.nanstd(snr4[:, :, irx])
s4_hat = np.nanmedian(snr4[:, :, irx])
s4_eps = TH * s4_hat # + st_std
# 4.2 Store the limits ----------------------------------------------- #
scint_limits[irx, 0] = st_eps
receiver_std[irx, 0] = st_std
receiver_std_median[irx, 0] = st_std_tec
# ----------------------------------------------------------------------- #
scint_limits[irx, 1] = s4_eps
receiver_std[irx, 1] = s4_std
receiver_std_median[irx, 1] = s4_std
# ----------------------------------------------------------------------- #
for isv in range(svx):
if log:
with open(logfn, 'a') as LOG:
LOG.write(
'Processing scintillation sv/all {}/{}\n'.format(
isv + 1, svx))
LOG.close()
else:
print('Processing scintillation sv/all {}/{}'.format(
isv + 1, svx))
sigma_tec[:, isv, irx] = _scintillationMask(sigma_tec[:, isv,
irx],
X_hat=st_hat,
X_eps=st_eps,
extend=0,
N_median=60,
min_length=120,
gap_close=5 * 60)
snr4[:, isv, irx] = _scintillationMask(snr4[:, isv, irx],
X_hat=s4_hat,
X_eps=s4_eps,
extend=0,
min_length=120,
gap_close=5 * 60)
#######################################################################
# Plot for refernce
if plot:
try:
if np.nansum(np.isfinite(sigma_tec_all[:, isv])) > 1:
print("Plotting PRN:{}".format(isv + 1))
fig = plt.figure(figsize=[15, 12])
ax1 = fig.add_subplot(421)
ax12 = ax1.twinx()
if irxforce is None:
ax1.plot(dt,
f['res'][:, isv, irx],
'b',
label='RXi {}; PRN {}'.format(
irx, isv + 1))
ax12.plot(dt, f['el'][:, isv, irx], 'g')
else:
ax1.plot(dt,
f['res'][:, isv, irxforce],
'b',
label='RXi {}; PRN {}'.format(
irx, isv + 1))
ax12.plot(dt,
f['el'][:, isv, irxforce],
'g',
lw=0.5)
ax1.set_ylabel('$\Delta$ TEC')
ax1.grid(axis='both')
ax12.set_ylabel('Elevation', color='g')
ax12.tick_params(axis='y', colors='green')
ax1.legend()
ax1.set_xticklabels([])
# Second
ax2 = fig.add_subplot(423, sharex=ax1)
ax2.plot(dt, tec_hpf_all[:, isv], 'b')
ax2.plot(dt[tec_outliers[:, isv]],
tec_hpf_all[:, isv][tec_outliers[:, isv]],
'xr')
ax2.set_ylabel('$\delta TEC_{0.1 Hz}$')
ax2.grid(axis='both')
# Third
ax3 = fig.add_subplot(427, sharex=ax1)
ax3.plot(dt, sigma_tec_all[:, isv], '.b')
i0 = np.argwhere(np.isfinite(
sigma_tec_all[:, isv]))[0]
i1 = np.argwhere(np.isfinite(
sigma_tec_all[:, isv]))[-1]
ax3.plot([dt[i0], dt[i1]], [st_eps, st_eps], '--r')
if sum(np.isfinite(sigma_tec[:, isv, irx])) > 0:
ax3.plot(dt, sigma_tec[:, isv, irx], '.g')
ax3.set_ylabel('$\sigma_{TEC}$ [TECu]')
# ax3.set_xlim([datetime(2017,9,8,0), datetime(2017,9,8,5)])
ax3.grid(axis='both')
ax3.xaxis.set_major_formatter(
mdates.DateFormatter('%H:%M'))
######################### SNR
ax11 = fig.add_subplot(422, sharex=ax1)
if irxforce is None:
ax11.plot(dt,
f['snr'][:, isv, irx],
'b',
label='RXi {}; PRN {}'.format(
irx, isv + 1))
else:
ax11.plot(dt,
f['snr'][:, isv, irxforce],
'b',
label='RXi {}; PRN {}'.format(
irx, isv + 1))
ax11.set_ylabel('SNR')
ax11.grid(axis='both')
ax11.legend()
# Second
ax21 = fig.add_subplot(424, sharex=ax1)
ax21.plot(dt, snr_hpf_all[:, isv], 'b')
ax2.plot(dt[snr_outliers[:, isv]],
tec_hpf_all[:, isv][snr_outliers[:, isv]],
'xr')
ax21.set_ylabel('$SNR4_{0.1 Hz}$')
ax21.grid(axis='both')
# Third
ax31 = fig.add_subplot(426, sharex=ax1)
ax31.plot(dt, snr4_all[:, isv], '.b')
i0 = np.argwhere(np.isfinite(snr4_all[:, isv]))[0]
i1 = np.argwhere(np.isfinite(snr4_all[:, isv]))[-1]
ax31.plot([dt[i0], dt[i1]], [s4_eps, s4_eps],
'--r')
if sum(np.isfinite(snr4[:, isv, irx])) > 0:
ax31.plot(dt, snr4[:, isv, irx], '.g')
ax31.plot(dt, s4[:, isv, irx], 'k', lw=0.5)
ax31.set_ylabel('SNR$_4$ [dB]')
ax31.grid(axis='both')
ax31.xaxis.set_major_formatter(
mdates.DateFormatter('%H:%M'))
prefix = dt[0].strftime("%Y%m%d")
svf = '{}_rxi{}_prni{}'.format(prefix, irx, isv)
ax1.set_title('E($\sigma_T$) = {}'.format(st_eps))
ax11.set_title('E(SNR$_4$) = {}'.format(s4_eps))
ax41 = fig.add_subplot(428, sharex=ax1)
ax41.plot(dt, roti[:, isv, irx], '.b')
ax41.set_ylabel('ROTI [TECu]')
ax41.grid(axis='both')
ax41.xaxis.set_major_formatter(
mdates.DateFormatter('%H:%M'))
ax42 = fig.add_subplot(425, sharex=ax1)
ax42.plot(dt, rot[:, isv, irx], 'b')
ax42.set_ylabel('ROT [TECu]')
ax42.grid(axis='both')
ax42.xaxis.set_major_formatter(
mdates.DateFormatter('%H:%M'))
if savefig:
if not os.path.exists(figfolder):
import subprocess
if platform.system() == 'Linux':
subprocess.call(
'mkdir -p {}'.format(figfolder),
shell=True,
timeout=5)
else:
subprocess.call(
'mkdir "{}"'.format(figfolder),
shell=True,
timeout=5)
plt.savefig(figfolder + '{}.png'.format(svf),
dpi=100)
plt.close(fig)
else:
print("Not enoughd data from PRN:{}".format(isv +
1))
except Exception as e:
print(e)
except Exception as e:
print(e)
if irxforce is not None:
break
rxn = f['rx_name'][:]
rxm = f['rx_model'][:]
f.close()
# Save to new hdf5 file
if irxforce is None:
if log:
with open(logfn, 'a') as LOG:
LOG.write('Saving data to : \n {}'.format(ofn))
LOG.close()
else:
print('Saving data to : \n {}'.format(ofn))
f = h5py.File(ofn, 'w')
gr = f.create_group('data')
gr.create_dataset('rx_name', data=rxn, dtype='S10')
gr.create_dataset('rx_model', data=rxm, dtype='S25')
gr.create_dataset('time',
data=time,
compression='gzip',
compression_opts=9)
gr.create_dataset('sigma_tec',
data=sigma_tec,
compression='gzip',
compression_opts=9)
gr.create_dataset('snr4',
data=snr4,
compression='gzip',
compression_opts=9)
gr.create_dataset('s4',
data=s4,
compression='gzip',
compression_opts=9)
gr.create_dataset('roti',
data=roti,
compression='gzip',
compression_opts=9)
gr.create_dataset('ipp',
data=ipp,
compression='gzip',
compression_opts=9)
gr.create_dataset('rxp',
data=rxpall,
compression='gzip',
compression_opts=9)
gr.create_dataset('scint_limits',
data=scint_limits,
compression='gzip',
compression_opts=9)
gr.create_dataset('rxstd',
data=receiver_std,
compression='gzip',
compression_opts=9)
gr.create_dataset('rxstdmedian',
data=receiver_std_median,
compression='gzip',
compression_opts=9)
gr.attrs[u'altitude_km'] = H
gr.attrs[u'hpf_fc'] = fc
gr.attrs[u'hpf_order'] = hpf_order
f.close()
if log:
with open(logfn, 'a') as LOG:
LOG.write('Successfully saved!')
LOG.close()
else:
print('Successfully saved!')
mask0[:3] = False
# Merge with the elevation mask
idel = np.logical_and(D['idel'].values, mask0)
# sb = satbias[sv]
dt = D.time.values
tsps = np.diff(dt.astype('datetime64[s]'))[0].astype(int)
eps = 1 * np.sqrt(30 / tsps)
elv = D.el.values
mask = (np.nan_to_num(elv) >= el_mask)
stec, intervals = tecPerLOS(D, maxjump=1, maxgap=10)
F = np.nan * np.copy(stec)
F[np.isfinite(elv)] = pyGnss.getMappingFunction(
elv[np.isfinite(elv)], 350)
stec *= F
if np.sum(np.isfinite(stec)) < (15 * (60 / tsps)):
# If shorter than 15 minutes, skip
continue
tecd, err_list = tecdPerLOS(stec,
intervals,
mask,
polynom_list=polynom_list,
eps=eps)
# Print the shit
tecd[~mask] = np.nan
stec[~mask] = np.nan
elv[np.isnan(tecd)] = np.nan
if PLOT:
plots(dt, stec, elv, tecd, polynom_list, err_list, odir=None)
L2 = D['L2'].values
L2[~idel] = np.nan
# obs = {'C1': C1, 'P2': C2, 'L1': L1, 'L2': L2}
# tec, tecd, F = pyGnss.processTEC(obs, sv, frequency = 2, Ts=tsps,
# H=H, elevation = el, sat_bias=sb)
# Intervals
ixin, intervals = pyGnss.getIntervals(L1,L2,C1,C2, maxgap=1)
tec = np.nan * stec[0]
for r in intervals:
tec[r[0]:r[-1]] = pyGnss.slantTEC(C1[r[0]:r[-1]], C2[r[0]:r[-1]],
L1[r[0]:r[-1]], L2[r[0]:r[-1]])
stec[i, :] = tec + sb
F[i, :] = pyGnss.getMappingFunction(el, H)
vtec = stec * F
rxb = -5
eps = 1e6
#cost_column0 = np.nanmean(vtec[:,0])
#cost0 = np.nanmean(cost_column0)
#cost = np.nan * np.ones(stec.shape[1])
#for it in range(stec.shape[1]):
# idx = np.where(np.isfinite(vtec[:,it]))[0]
# d = []
# for i in range(idx.shape[0]):
# for j in np.arange(i+1, idx.shape[0]):
# d.append(abs(vtec[idx[i], it] - vtec[idx[j], it]))
# cost[it] = np.nanmean(d)