def RxbiasEstimator(RxBtime=0, TECref=0, t=0, sTEC=0, sat_bias=0, el=[], IPPalt=350): Fm = pyGnss.getMappingFunction(el, IPPalt) idt = abs(t - RxBtime).argmin() Fmtt = Fm[idt] sTECt = sTEC[idt] rx_bias = TECref/Fmtt - sTECt - sat_bias return rx_bias
def process(fn, odir=None, cfg=None, log=None, irxforce=None, TH=0): ############################### Open data ################################## if irxforce is not None: irxforce = int(irxforce) if odir is None: odir = os.path.split(fn)[0] + separator if cfg is None: plot_ripple = 0 plot_outlier = 0 savefig = 1 figfolder = os.path.join(odir, 'scint_plots' + separator) plot = 0 fs = 1 fc = 0.1 hpf_order = 6 H = 350 else: assert os.path.splitext(cfg)[1] in ('.yml', '.yaml') stream = yaml.load(open(cfg, 'r')) plot_ripple = stream.get('plot_ripple') plot_outlier = stream.get('plot_outlier') plot = stream.get('plot') savefig = stream.get('savefig') figfolder = stream.get('figfolder') if figfolder is None: figfolder = os.path.join(odir, 'scint_plots' + separator) fs = stream.get('fs') fc = stream.get('fc') hpf_order = stream.get('hpf_order') H = stream.get('alt_km') # Output file if odir is None: odir = os.path.split(fn)[0] + separator ofn = odir + 'ix_' + '_'.join( os.path.split(fn)[1].split('.')[:2]) + '_{}km_{}th.h5'.format( H, str(TH).replace('.', '')) # Dealing with duplicate file names if os.path.exists(ofn): head = os.path.splitext(ofn)[0] c = 0 while os.path.exists(ofn): try: c = int(os.path.splitext(ofn)[0].split('_')[-1]) c += 1 except: c += 1 ofn = head + '_' + str(c) + '.h5' if log: logfn = os.path.splitext(ofn)[0] + '.log' LOG = open(logfn, 'w') LOG.close() # Open data file f = h5py.File(fn, 'r') time = f['obstimes'][:] dt = np.array([datetime.utcfromtimestamp(t) for t in time]) if irxforce is None: rnx = f['el'].shape[2] else: rnx = 1 #rnx = 5 svx = f['el'].shape[1] rxpall = f['rx_positions'][:] # New arrays ipp = np.nan * np.ones( (dt.size, svx, rnx, 2)) # [time, SV, Rx, [lat, lon]] sigma_tec = np.nan * np.ones((dt.size, svx, rnx)) snr4 = np.nan * np.ones((dt.size, svx, rnx)) s4 = np.nan * np.ones((dt.size, svx, rnx)) roti = np.nan * np.ones((dt.size, svx, rnx)) if plot: rot = np.nan * np.ones((dt.size, svx, rnx)) # tec_hpf = np.nan * np.ones((dt.size, svx, rnx)) # Bookkeeping scint_limits = np.nan * np.zeros((rnx, 2)) receiver_std = np.nan * np.zeros((rnx, 2)) receiver_std_median = np.nan * np.zeros((rnx, 2)) for irx in range(rnx): if log: with open(logfn, 'a') as LOG: LOG.write('Processing Rx/all #{}/{}\n'.format(irx + 1, rnx)) LOG.close() else: print('Processing Rx/all #{}/{}'.format(irx + 1, rnx)) if plot: tec_hpf_all = np.nan * np.ones((dt.size, svx)) snr_hpf_all = np.nan * np.ones((dt.size, svx)) sigma_tec_all = np.nan * np.ones((dt.size, svx)) snr4_all = np.nan * np.ones((dt.size, svx)) # Reset to zero for each iteration tec_outliers = np.zeros((dt.size, svx), dtype=bool) snr_outliers = np.zeros((dt.size, svx), dtype=bool) try: for isv in range(svx): try: if irxforce is not None: el = f['el'][:, isv, irxforce] az = f['az'][:, isv, irxforce] res = f['res'][:, isv, irxforce] snr = f['snr'][:, isv, irxforce] rxp = rxpall[irxforce] else: el = f['el'][:, isv, irx] az = f['az'][:, isv, irx] res = f['res'][:, isv, irx] snr = f['snr'][:, isv, irx] rxp = rxpall[irx] # Compute location of the IPP lat, lon = _toLLT(rxp, az=az, el=el, H=H) # Get Mapping Function F = pyGnss.getMappingFunction(el, h=350) # Stack into the output array ipp[:, isv, irx, 0] = lat ipp[:, isv, irx, 1] = lon except Exception as e: if log: with open(logfn, 'a') as LOG: LOG.write('{}\n'.format(e)) LOG.close() else: print(e) # Check for a minum length of valid observables == 30 min if np.nansum(np.isfinite(res)) < 30 * 60: continue tec_hpf_copy = np.nan * np.copy(res) rot_copy = np.nan * np.copy(res) roti_copy = np.nan * np.copy(res) sigma_tec_copy = np.nan * np.copy(res) snr4_copy = np.nan * np.copy(snr) s4_copy = np.nan * np.copy(snr) tec_hpf_original = np.nan * np.copy(res) snr_hpf_original = np.nan * np.copy(res) # 0.0 To ranges: Multipe visits of a satellite per day. # New interval for a gap bigger than 10 samples. # Minimum length of interval is 30 minutes # Create empty arrays idf_tec = np.isfinite(res) idf_snr = np.isfinite(snr) # 0.1 Do for TEC try: tec_ranges = ranges(res, idf_tec, min_gap=10, gap_length=10, min_length=30 * 60, zero_mean=True) except: tec_ranges = np.array([]) try: snr_ranges = ranges(snr, idf_snr, min_gap=10, gap_length=10, min_length=30 * 60) except: snr_ranges = np.array([]) # Process TEC per intervals if tec_ranges.size > 0: for ith_range, r in enumerate(tec_ranges): # Remove to short ranges if accidentaly do occur if np.diff(r) < 10: continue try: chunk = res[r[0]:r[1]] tec_hpf, tec_hpf_original[ r[0]:r[1]], tec_mask = _partialProcess( dt, r, chunk, fs=fs, fc=fc, hpf_order=hpf_order, plot_ripple=plot_ripple, plot_outlier=plot_outlier) tec_outliers[r[0]:r[1], isv] = tec_mask sigma_tec_copy[r[0]:r[1]] = scint.sigmaTEC(tec_hpf, N=60) tec_hpf_copy[r[0]:r[1]] = tec_hpf tmp_diff = np.diff(chunk) tmp_diff[tec_mask[1:]] = np.nan rot_copy[r[0] + 1:r[1]] = tmp_diff roti_copy[r[0] + 1:r[1]] = scint.sigmaTEC( np.diff(chunk), N=60) except Exception as e: if log: with open(logfn, 'a') as LOG: LOG.write('{}\n'.format(e)) LOG.close() else: print(e) if snr_ranges.size > 0: for ith_range, r in enumerate(snr_ranges): # Remove to short ranges if accidentaly do occur if np.diff(r) < 60: continue try: Schunk = snr[r[0]:r[1]].astype(np.float64) snr_hpf, snr_hpf_original[ r[0]:r[1]], snr_mask = _partialProcess( dt, r, Schunk, fs=fs, fc=fc, hpf_order=hpf_order, plot_ripple=plot_ripple, plot_outlier=plot_outlier) snr_outliers[r[0]:r[1], isv] = snr_mask snr4_copy[r[0]:r[1]] = scint.sigmaTEC(snr_hpf, N=60) s4_copy[ r[0]:r[1]] = scint.AmplitudeScintillationIndex( 10**(Schunk / 10), 60) except Exception as e: if log: with open(logfn, 'a') as LOG: LOG.write('{}\n'.format(e)) LOG.close() else: print(e) # Save scintillation indices sigma_tec[:, isv, irx] = sigma_tec_copy snr4[:, isv, irx] = (snr4_copy * (F**0.9)) s4[:, isv, irx] = (s4_copy * (F**0.9)) roti[:, isv, irx] = roti_copy if plot: rot[:, isv, irx] = rot_copy tec_hpf_all[:, isv] = tec_hpf_original snr_hpf_all[:, isv] = snr_hpf_original sigma_tec_all[:, isv] = sigma_tec_copy snr4_all[:, isv] = snr4_copy # 4. Define the scintillation event masks per receiver # 4.1 Define limits # sigma_tec: limit ------------------------------------------------------ # st_std = np.nanstd(sigma_tec[:, :, irx]) st_std_tec = np.nanstd(sigma_tec[:, :, irx]) st_hat = np.nanmedian(sigma_tec[:, :, irx]) st_eps = TH * st_hat # + st_std # SNR4 limit s4_std = np.nanstd(snr4[:, :, irx]) s4_hat = np.nanmedian(snr4[:, :, irx]) s4_eps = TH * s4_hat # + st_std # 4.2 Store the limits ----------------------------------------------- # scint_limits[irx, 0] = st_eps receiver_std[irx, 0] = st_std receiver_std_median[irx, 0] = st_std_tec # ----------------------------------------------------------------------- # scint_limits[irx, 1] = s4_eps receiver_std[irx, 1] = s4_std receiver_std_median[irx, 1] = s4_std # ----------------------------------------------------------------------- # for isv in range(svx): if log: with open(logfn, 'a') as LOG: LOG.write( 'Processing scintillation sv/all {}/{}\n'.format( isv + 1, svx)) LOG.close() else: print('Processing scintillation sv/all {}/{}'.format( isv + 1, svx)) sigma_tec[:, isv, irx] = _scintillationMask(sigma_tec[:, isv, irx], X_hat=st_hat, X_eps=st_eps, extend=0, N_median=60, min_length=120, gap_close=5 * 60) snr4[:, isv, irx] = _scintillationMask(snr4[:, isv, irx], X_hat=s4_hat, X_eps=s4_eps, extend=0, min_length=120, gap_close=5 * 60) ####################################################################### # Plot for refernce if plot: try: if np.nansum(np.isfinite(sigma_tec_all[:, isv])) > 1: print("Plotting PRN:{}".format(isv + 1)) fig = plt.figure(figsize=[15, 12]) ax1 = fig.add_subplot(421) ax12 = ax1.twinx() if irxforce is None: ax1.plot(dt, f['res'][:, isv, irx], 'b', label='RXi {}; PRN {}'.format( irx, isv + 1)) ax12.plot(dt, f['el'][:, isv, irx], 'g') else: ax1.plot(dt, f['res'][:, isv, irxforce], 'b', label='RXi {}; PRN {}'.format( irx, isv + 1)) ax12.plot(dt, f['el'][:, isv, irxforce], 'g', lw=0.5) ax1.set_ylabel('$\Delta$ TEC') ax1.grid(axis='both') ax12.set_ylabel('Elevation', color='g') ax12.tick_params(axis='y', colors='green') ax1.legend() ax1.set_xticklabels([]) # Second ax2 = fig.add_subplot(423, sharex=ax1) ax2.plot(dt, tec_hpf_all[:, isv], 'b') ax2.plot(dt[tec_outliers[:, isv]], tec_hpf_all[:, isv][tec_outliers[:, isv]], 'xr') ax2.set_ylabel('$\delta TEC_{0.1 Hz}$') ax2.grid(axis='both') # Third ax3 = fig.add_subplot(427, sharex=ax1) ax3.plot(dt, sigma_tec_all[:, isv], '.b') i0 = np.argwhere(np.isfinite( sigma_tec_all[:, isv]))[0] i1 = np.argwhere(np.isfinite( sigma_tec_all[:, isv]))[-1] ax3.plot([dt[i0], dt[i1]], [st_eps, st_eps], '--r') if sum(np.isfinite(sigma_tec[:, isv, irx])) > 0: ax3.plot(dt, sigma_tec[:, isv, irx], '.g') ax3.set_ylabel('$\sigma_{TEC}$ [TECu]') # ax3.set_xlim([datetime(2017,9,8,0), datetime(2017,9,8,5)]) ax3.grid(axis='both') ax3.xaxis.set_major_formatter( mdates.DateFormatter('%H:%M')) ######################### SNR ax11 = fig.add_subplot(422, sharex=ax1) if irxforce is None: ax11.plot(dt, f['snr'][:, isv, irx], 'b', label='RXi {}; PRN {}'.format( irx, isv + 1)) else: ax11.plot(dt, f['snr'][:, isv, irxforce], 'b', label='RXi {}; PRN {}'.format( irx, isv + 1)) ax11.set_ylabel('SNR') ax11.grid(axis='both') ax11.legend() # Second ax21 = fig.add_subplot(424, sharex=ax1) ax21.plot(dt, snr_hpf_all[:, isv], 'b') ax2.plot(dt[snr_outliers[:, isv]], tec_hpf_all[:, isv][snr_outliers[:, isv]], 'xr') ax21.set_ylabel('$SNR4_{0.1 Hz}$') ax21.grid(axis='both') # Third ax31 = fig.add_subplot(426, sharex=ax1) ax31.plot(dt, snr4_all[:, isv], '.b') i0 = np.argwhere(np.isfinite(snr4_all[:, isv]))[0] i1 = np.argwhere(np.isfinite(snr4_all[:, isv]))[-1] ax31.plot([dt[i0], dt[i1]], [s4_eps, s4_eps], '--r') if sum(np.isfinite(snr4[:, isv, irx])) > 0: ax31.plot(dt, snr4[:, isv, irx], '.g') ax31.plot(dt, s4[:, isv, irx], 'k', lw=0.5) ax31.set_ylabel('SNR$_4$ [dB]') ax31.grid(axis='both') ax31.xaxis.set_major_formatter( mdates.DateFormatter('%H:%M')) prefix = dt[0].strftime("%Y%m%d") svf = '{}_rxi{}_prni{}'.format(prefix, irx, isv) ax1.set_title('E($\sigma_T$) = {}'.format(st_eps)) ax11.set_title('E(SNR$_4$) = {}'.format(s4_eps)) ax41 = fig.add_subplot(428, sharex=ax1) ax41.plot(dt, roti[:, isv, irx], '.b') ax41.set_ylabel('ROTI [TECu]') ax41.grid(axis='both') ax41.xaxis.set_major_formatter( mdates.DateFormatter('%H:%M')) ax42 = fig.add_subplot(425, sharex=ax1) ax42.plot(dt, rot[:, isv, irx], 'b') ax42.set_ylabel('ROT [TECu]') ax42.grid(axis='both') ax42.xaxis.set_major_formatter( mdates.DateFormatter('%H:%M')) if savefig: if not os.path.exists(figfolder): import subprocess if platform.system() == 'Linux': subprocess.call( 'mkdir -p {}'.format(figfolder), shell=True, timeout=5) else: subprocess.call( 'mkdir "{}"'.format(figfolder), shell=True, timeout=5) plt.savefig(figfolder + '{}.png'.format(svf), dpi=100) plt.close(fig) else: print("Not enoughd data from PRN:{}".format(isv + 1)) except Exception as e: print(e) except Exception as e: print(e) if irxforce is not None: break rxn = f['rx_name'][:] rxm = f['rx_model'][:] f.close() # Save to new hdf5 file if irxforce is None: if log: with open(logfn, 'a') as LOG: LOG.write('Saving data to : \n {}'.format(ofn)) LOG.close() else: print('Saving data to : \n {}'.format(ofn)) f = h5py.File(ofn, 'w') gr = f.create_group('data') gr.create_dataset('rx_name', data=rxn, dtype='S10') gr.create_dataset('rx_model', data=rxm, dtype='S25') gr.create_dataset('time', data=time, compression='gzip', compression_opts=9) gr.create_dataset('sigma_tec', data=sigma_tec, compression='gzip', compression_opts=9) gr.create_dataset('snr4', data=snr4, compression='gzip', compression_opts=9) gr.create_dataset('s4', data=s4, compression='gzip', compression_opts=9) gr.create_dataset('roti', data=roti, compression='gzip', compression_opts=9) gr.create_dataset('ipp', data=ipp, compression='gzip', compression_opts=9) gr.create_dataset('rxp', data=rxpall, compression='gzip', compression_opts=9) gr.create_dataset('scint_limits', data=scint_limits, compression='gzip', compression_opts=9) gr.create_dataset('rxstd', data=receiver_std, compression='gzip', compression_opts=9) gr.create_dataset('rxstdmedian', data=receiver_std_median, compression='gzip', compression_opts=9) gr.attrs[u'altitude_km'] = H gr.attrs[u'hpf_fc'] = fc gr.attrs[u'hpf_order'] = hpf_order f.close() if log: with open(logfn, 'a') as LOG: LOG.write('Successfully saved!') LOG.close() else: print('Successfully saved!')
mask0[:3] = False # Merge with the elevation mask idel = np.logical_and(D['idel'].values, mask0) # sb = satbias[sv] dt = D.time.values tsps = np.diff(dt.astype('datetime64[s]'))[0].astype(int) eps = 1 * np.sqrt(30 / tsps) elv = D.el.values mask = (np.nan_to_num(elv) >= el_mask) stec, intervals = tecPerLOS(D, maxjump=1, maxgap=10) F = np.nan * np.copy(stec) F[np.isfinite(elv)] = pyGnss.getMappingFunction( elv[np.isfinite(elv)], 350) stec *= F if np.sum(np.isfinite(stec)) < (15 * (60 / tsps)): # If shorter than 15 minutes, skip continue tecd, err_list = tecdPerLOS(stec, intervals, mask, polynom_list=polynom_list, eps=eps) # Print the shit tecd[~mask] = np.nan stec[~mask] = np.nan elv[np.isnan(tecd)] = np.nan if PLOT: plots(dt, stec, elv, tecd, polynom_list, err_list, odir=None)
L2 = D['L2'].values L2[~idel] = np.nan # obs = {'C1': C1, 'P2': C2, 'L1': L1, 'L2': L2} # tec, tecd, F = pyGnss.processTEC(obs, sv, frequency = 2, Ts=tsps, # H=H, elevation = el, sat_bias=sb) # Intervals ixin, intervals = pyGnss.getIntervals(L1,L2,C1,C2, maxgap=1) tec = np.nan * stec[0] for r in intervals: tec[r[0]:r[-1]] = pyGnss.slantTEC(C1[r[0]:r[-1]], C2[r[0]:r[-1]], L1[r[0]:r[-1]], L2[r[0]:r[-1]]) stec[i, :] = tec + sb F[i, :] = pyGnss.getMappingFunction(el, H) vtec = stec * F rxb = -5 eps = 1e6 #cost_column0 = np.nanmean(vtec[:,0]) #cost0 = np.nanmean(cost_column0) #cost = np.nan * np.ones(stec.shape[1]) #for it in range(stec.shape[1]): # idx = np.where(np.isfinite(vtec[:,it]))[0] # d = [] # for i in range(idx.shape[0]): # for j in np.arange(i+1, idx.shape[0]): # d.append(abs(vtec[idx[i], it] - vtec[idx[j], it])) # cost[it] = np.nanmean(d)