def __read_txyz(file_name, lGts, verbose=False):
"""
Reads a single txyz.dat
"""
NP_STR_TXYZ = np.genfromtxt(file_name, dtype=str)
NP_CODE = NP_STR_TXYZ[:, 2]
NP_DATE = np.array(NP_STR_TXYZ[:, 1], dtype=float)
NP_XYZ = np.array(NP_STR_TXYZ[:, 4:], dtype=float)
for i in np.arange(NP_CODE.shape[0]):
code = NP_CODE[i]
if code in list(lGts.keys()):
lGts[code].add_obs_xyz(NP_DATE[i],
NP_XYZ[i],
in_place=True,
check=False)
else:
new_gts = Gts(code=code)
new_gts.add_obs_xyz(NP_DATE[i],
NP_XYZ[i],
in_place=True,
check=False)
lGts[code] = new_gts
return (lGts)
def get_unr(self, lcode):
"""
:param lcode: list of GNSS site codes to be downloaded from UNR web site
:return: new Sgts instance
"""
# import
from pyacs.gts.Gts import Gts
from pyacs.gts.Sgts import Sgts
# initialize Sgts
ts = Sgts(read=False)
# loop on sites
for code in lcode:
print("-- downloading %s from UNR" % code)
try:
ts.append(Gts().get_unr(code))
except:
print("!!!ERROR downloading %s from UNR" % code)
# return
return ts
def obs_tensor2sgts(T_OBS_RAW, np_names_t_obs, np_obs_date_s, verbose=True):
"""
Converts a obs_tensor object into a Sgts
"""
# import
import numpy as np
from pyacs.gts.Gts import Gts
from pyacs.gts.Sgts import Sgts
import pyacs.lib.astrotime as at
# import pyeq.message.message as MESSAGE
# import pyeq.message.verbose_message as VERBOSE
# import pyeq.message.error as ERROR
# import pyeq.message.debug_message as DEBUG
# initialize Sgts
sgts = Sgts(read=False)
# loop on sites
for i in np.arange(np_names_t_obs.shape[0]):
code = np_names_t_obs[i]
#DEBUG('converting %s ' % code)
# get the index of valid dates
lindex = np.where(np.isfinite(T_OBS_RAW[:, i, 0]))[0]
data = np.zeros((lindex.shape[0], 10))
# obs_tensor is ENU and Gts are NEU
data[:, 2] = T_OBS_RAW[lindex, i, 0] * 1E-3
data[:, 1] = T_OBS_RAW[lindex, i, 1] * 1E-3
data[:, 3] = T_OBS_RAW[lindex, i, 2] * 1E-3
data[:, 5] = T_OBS_RAW[lindex, i, 3] * 1E-3
data[:, 4] = T_OBS_RAW[lindex, i, 4] * 1E-3
data[:, 6] = T_OBS_RAW[lindex, i, 5] * 1E-3
data[:, 0] = at.datetime2decyear(
at.seconds2datetime(np_obs_date_s[lindex]))
sgts.append(Gts(code=code, data=data))
#VERBOSE("converted %d time series " % ( len( sgts.lcode() )) )
return sgts
def __pos(ts_dir=ts_dir,
name_filter=name_filter,
add_key=add_key,
verbose=verbose,
xyz=True):
import glob
lGts = {}
lpos = sorted(glob.glob(ts_dir + '/????*.pos'))
import os
for pos_w_path in lpos:
if not os.path.isfile(pos_w_path): continue
_drive, path_and_file = os.path.splitdrive(pos_w_path)
_path, pos = os.path.split(path_and_file)
code = pos[:4]
if code not in sites and sites != []: continue
if verbose: print("-- Reading ", pos)
lGts[code + add_key] = Gts(code=code)
lGts[code + add_key].read_pos(tsfile=pos_w_path, xyz=xyz)
return (lGts)
def __mb_files(ts_dir=ts_dir,
name_filter=name_filter,
add_key=add_key,
verbose=verbose):
import glob
import os
lGts = {}
ldat = sorted(glob.glob(ts_dir + '/' + '*' + name_filter + '*.dat1'))
for fdat in ldat:
if not os.path.isfile(fdat): continue
mb_file = fdat[0:-1]
code = mb_file[-12:-8]
if code not in sites and sites != []: continue
if verbose:
print("-- Reading ", mb_file)
lGts[code + add_key] = Gts(code=code)
lGts[code + add_key].read_mb_file(ifile=mb_file, gmt=False)
return (lGts)
def __cats(ts_dir=ts_dir,
name_filter=name_filter,
add_key=add_key,
verbose=verbose):
import glob
import os
lGts = {}
lcats = sorted(glob.glob(ts_dir + '/cats*.dat'))
for cats in lcats:
if not os.path.isfile(cats): continue
code = cats.split('cats_')[-1][:4].upper()
if verbose: print("-- Reading ", cats)
lGts[code + add_key] = Gts(code=code)
try:
lGts[code + add_key].read_cats_file(ifile=cats, gmt=False)
except:
print("!!! Error. Could not read ", cats)
del lGts[code + add_key]
continue
return (lGts)
def __kenv(ts_dir=ts_dir,
name_filter=name_filter,
add_key=add_key,
verbose=verbose):
import glob
import os
lGts = {}
lkenv = sorted(glob.glob(ts_dir + '/*.kenv'))
for fkenv in lkenv:
if not os.path.isfile(fkenv): continue
code = fkenv[-20:-9]
if verbose: print("-- Reading ", fkenv)
lGts[code + add_key] = Gts(code=code)
try:
lGts[code + add_key].read_kenv(fkenv, date_type='jd')
except:
print("!!! Error. Could not read ", fkenv)
del lGts[code + add_key]
continue
return (lGts)
def __pride(ts_dir=ts_dir,
name_filter=name_filter,
add_key=add_key,
verbose=verbose):
import glob
import os
lGts = {}
lfile = glob.glob(ts_dir + '/' + 'kin_*' + name_filter)
lcode = []
for ifile in lfile:
if not os.path.isfile(ifile): continue
code = ifile[-4:].upper()
if code not in lcode:
lcode.append(code)
for code in lcode:
if verbose:
print("-- Reading pride files for: %s " % code)
lGts[code + add_key] = Gts(code=code)
lGts[code + add_key].read_pride(tsdir=ts_dir, verbose=verbose)
return (lGts)
def __track_NEU(ts_dir='.',
name_filter=name_filter,
add_key=add_key,
verbose=verbose):
import glob
lGts = {}
ltrack = sorted(glob.glob(ts_dir + '/*.NEU.*'))
import os
for track_w_path in ltrack:
if not os.path.isfile(track_w_path): continue
_drive, path_and_file = os.path.splitdrive(track_w_path)
_path, track = os.path.split(path_and_file)
code = track.split('.')[-2].upper()
if code not in sites and sites != []: continue
if verbose: print("-- Reading ", track)
lGts[code + add_key] = Gts(code=code)
try:
lGts[code + add_key].read_track_NEU(tsfile=track_w_path)
except:
print("!!! Error. Could not read ", track)
del lGts[code + add_key]
continue
return (lGts)
def get_unr(self, site, verbose=False):
"""
Get a time series from http://geodesy.unr.edu/gps_timeseries/txyz/IGS14/ in PYACS
:param site: 4-letters code
:param verbose: verbose mode
"""
# import
import urllib.request
from urllib.error import HTTPError, URLError
import socket
from pyacs.gts.Gts import Gts
import numpy as np
import os
import pyacs.lib.astrotime as at
import datetime
from datetime import timedelta
delta_12h = timedelta(hours=12)
# url
url = ("http://geodesy.unr.edu/gps_timeseries/txyz/IGS14/%s.txyz2" %
site.upper())
# get data
try:
urllib.request.urlretrieve(url=url, filename="test.dat")
except HTTPError as error:
print('Data not retrieved because %s\nURL: %s', error, url)
except URLError as error:
if isinstance(error.reason, socket.timeout):
print('socket timed out - URL %s', url)
else:
print('some other error happened')
# creates Gts
# get code
code = np.genfromtxt('test.dat', usecols=0, dtype=str)[0]
gts = Gts(code=code)
# get data
gts.data_xyz = np.genfromtxt('test.dat',
usecols=(2, 3, 4, 5, 6, 7, 8, 9, 10, 11))
# decimal year dates in UNR files only have 4 digits, making the day time very approximate
# we round the dates at 12:00 and prefer the date string
str_date = np.genfromtxt('test.dat', usecols=(1), dtype=str)
np_datetime = np.array(
[datetime.datetime.strptime(x, "%y%b%d")
for x in str_date]) + delta_12h
#unr_dates_decyear = gts.data_xyz[:,0]
#np_year = np.array(unr_dates_decyear, dtype=int)
#(np_doy,_np_ut) = at.decyear2dayno( unr_dates_decyear )
#gts.data_xyz[:,0] = at.dayno2decyear( np_doy , np_year )
gts.data_xyz[:, 0] = at.datetime2decyear(np_datetime)
# convert data
gts.xyz2neu(corr=True, ref_xyz=None, verbose=verbose)
# remove 'test.dat'
os.remove('test.dat')
# return
return gts
def common_mode(self,
lref=[],
detrend_method='detrend_median',
method='median',
center=True,
verbose=True):
###################################################################
"""
calculates a common mode
:param lref: liste of site codes used to calculate the common mode
:param detrend_method: 'detrend_median' or 'detrend', method used to detrend the reference sites time series
:param method: method to calculate the common mode 'median' or 'mean'
:return: a Sgts instance with filtered time series. This new instance has a _CMM time series for the common mode
:note: time series are assumed to be daily time series
"""
# import
import pyacs.gts.lib.tensor_ts.sgts2obs_tensor
import pyacs.gts.lib.tensor_ts.obs_tensor2sgts
import numpy as np
from pyacs.gts.Gts import Gts
import pyacs.lib.astrotime as at
import pyacs.lib.coordinates as coor
# detrend for the reference sites
if detrend_method is not None:
dts = self.gts(detrend_method)
else:
dts = self.copy()
# creates an obs_tensor instance from the detrend time series
T_OBS_RAW, np_names_t_obs, np_obs_date_s = pyacs.gts.lib.tensor_ts.sgts2obs_tensor.sgts2tensor(
dts, rounding='day', verbose=verbose)
# center time series
if center:
T_OBS_RAW[:, :,
0] = T_OBS_RAW[:, :, 0] - np.nanmedian(T_OBS_RAW[:, :, 0],
axis=0)
T_OBS_RAW[:, :,
1] = T_OBS_RAW[:, :, 1] - np.nanmedian(T_OBS_RAW[:, :, 1],
axis=0)
T_OBS_RAW[:, :,
2] = T_OBS_RAW[:, :, 2] - np.nanmedian(T_OBS_RAW[:, :, 2],
axis=0)
# get the index of sites used for the common mode
lidx_code = []
for i in np.arange(np_names_t_obs.shape[0]):
if np_names_t_obs[i] in lref:
lidx_code.append(i)
np_idx_code = np.array(sorted(lidx_code))
# compute common mode
T_OBS_RAW_REF = T_OBS_RAW[:, np_idx_code, :]
if method == 'median':
CMM = np.nanmedian(T_OBS_RAW_REF, axis=1)
if method == 'mean':
CMM = np.nanmean(T_OBS_RAW_REF, axis=1)
# creates an obs_tensor instance from the original time series
T_OBS_RAW, np_names_t_obs, np_obs_date_s = pyacs.gts.lib.tensor_ts.sgts2obs_tensor.sgts2tensor(
self, rounding='day', verbose=verbose)
# remove common mode
T_OBS_RAW[:, :, 0] = T_OBS_RAW[:, :, 0] - CMM[:, 0].reshape(-1, 1)
T_OBS_RAW[:, :, 1] = T_OBS_RAW[:, :, 1] - CMM[:, 1].reshape(-1, 1)
T_OBS_RAW[:, :, 2] = T_OBS_RAW[:, :, 2] - CMM[:, 2].reshape(-1, 1)
# dispersion of the common mode
# TBD
# converts obs_tensor object to Sgts
filtered_sgts = pyacs.gts.lib.tensor_ts.obs_tensor2sgts.obs_tensor2sgts(
T_OBS_RAW, np_names_t_obs, np_obs_date_s, verbose=verbose)
# adds the common mode time series
cmm_ts = Gts(code='_CMM')
# get index where values are Nan
lindex = np.where(np.isfinite(CMM[:, 0]))[0]
data = np.zeros((lindex.shape[0], 10))
# obs_tensor is ENU and Gts are NEU
data[:, 2] = CMM[lindex, 0] * 1E-3
data[:, 1] = CMM[lindex, 1] * 1E-3
data[:, 3] = CMM[lindex, 2] * 1E-3
# set std as 1 mm to avoid singularity
data[:, 4] = 1E-3
data[:, 5] = 1E-3
data[:, 6] = 1E-3
data[:,
0] = at.datetime2decyear(at.seconds2datetime(np_obs_date_s[lindex]))
# populate X0, Y0, Z0, lon, lat, he
X_cmm = 0
Y_cmm = 0
Z_cmm = 0
for code in filtered_sgts.lcode():
filtered_sgts.__dict__[code].X0 = self.__dict__[code].X0
filtered_sgts.__dict__[code].Y0 = self.__dict__[code].Y0
filtered_sgts.__dict__[code].Z0 = self.__dict__[code].Z0
filtered_sgts.__dict__[code].lon = self.__dict__[code].lon
filtered_sgts.__dict__[code].lat = self.__dict__[code].lat
filtered_sgts.__dict__[code].h = self.__dict__[code].h
X_cmm = X_cmm + self.__dict__[code].X0
Y_cmm = Y_cmm + self.__dict__[code].Y0
Z_cmm = Z_cmm + self.__dict__[code].Z0
cmm_ts.data = data
X_cmm = X_cmm / len(filtered_sgts.lcode())
Y_cmm = Y_cmm / len(filtered_sgts.lcode())
Z_cmm = Z_cmm / len(filtered_sgts.lcode())
cmm_ts.X0 = X_cmm / len(filtered_sgts.lcode())
cmm_ts.Y0 = Y_cmm / len(filtered_sgts.lcode())
cmm_ts.Z0 = Z_cmm / len(filtered_sgts.lcode())
(cmm_ts.lon, cmm_ts.lat, cmm_ts.h) = coor.xyz2geo(cmm_ts.X0,
cmm_ts.Y0,
cmm_ts.Z0,
unit='dec_deg')
filtered_sgts.append(cmm_ts)
return filtered_sgts
code, soln = SORTED_RESIDUALS[i, :2]
[Re, Rn, Ru] = list(map(float, SORTED_RESIDUALS[i, 2:5]))
Re = Re * 1.E-3
Rn = Rn * 1.E-3
Ru = Ru * 1.E-3
if l_res_ts.has_ts(code):
l_res_ts.__dict__[code].add_obs(
date, [Rn, Re, Ru, 1.E-3, 1.E-3, 1.E-3, 0., 0., 0.],
in_place=True,
check=False,
verbose=False)
else:
new_ts = Gts(code=code)
new_ts.add_obs(date, [Rn, Re, Ru, 1.E-3, 1.E-3, 1.E-3, 0., 0., 0.],
in_place=True,
check=False,
verbose=False)
l_res_ts.append(new_ts)
H_res_helmert[sinex_basename] = [date] + list(H_stat['wrms'])
###########################################################################
# WRITING HELMERT SUMMARY AND STORING TSG VALUE
###########################################################################
VERBOSE("writing Helmert summary")
if isinstance(T, np.ndarray):
nps = InSAR.shape[0]
#nps = 10
RESULTS = np.zeros((nps,14))
# fills longitudes and latitudes
RESULTS[:,0] = InSAR[:nps,0]
RESULTS[:,1] = InSAR[:nps,1]
for i in np.arange(nps):
print("-- # %05d / %05d %03d %%" % (i,nps,int(i/nps*100.)) )
# initialize a ts
ts=Gts()
ts.data = np.zeros( (np_dec_year.shape[0],10) )
ts.data[:,0] = np_dec_year
ts.data[:,1] = InSAR[i,2:20]
ts.data[:,2] = InSAR[i,2:20]
ts.data[:,3] = InSAR[i,2:20]
ts.data[:,4] = 1.
ts.data[:,5] = 1.
ts.data[:,6] = 1.
# Full ts L2 norm
dts = ts.detrend()
vel = dts.velocity
wrms = dts.wrms()
RESULTS[i,2] = vel[0]
def same_site(self, dc=10, in_place=True, verbose=False):
###################################################################
"""
Check that all gts in the current Sgts are actually the same site. If a given time series is
found to be of two separate sites, then a new gts is added to the return Sgts instance.
param dc: critical distance to decide to split the time series
param in_place: if True modify current Sgts, False retuen a new Sgts
param verbose: verbose mode
return: a new Sgts instance
"""
# import
import numpy as np
from pyacs.gts.Sgts import Sgts
from pyacs.gts.Gts import Gts
if not in_place:
new_Sgts = Sgts(read=False)
# start loop on sites
lcode = self.lcode()
for site in lcode:
if verbose:
print('-- Processing ', site)
my_ts = self.__dict__[site].copy()
if my_ts.data_xyz is not None:
data = my_ts.data_xyz[:, 1:4]
ddata = np.copy(my_ts.data_xyz[:, 1:4])
else:
# if no data_xyz go to next gts
print(
"!!! WARNING: data_xyz attribute required for method same_site and not found gts %s"
% (site))
# ensure median calculation
if np.mod(data.shape[0], 2) == 0:
# duplicates the last date
ddata = np.vstack((ddata, ddata[-1, :]))
median = np.median(ddata, axis=0)
dist_data = np.sqrt(np.sum((data - median)**2, axis=1))
lindex = np.where(dist_data > dc * 1.E3)
# case gts needs to be split
if len(lindex[0]) > 0:
# create a new code
new_code = my_ts.code[:3] + '_'
if new_code in self.lcode():
print(
"!!! ERROR: try to create a new gts with code %s and it already exists."
% (new_code))
new_code = my_ts.code[:2] + '__'
if verbose:
print(
"-- time series for site %s appears to include different sites because there are coordinates at %d dates %.1lf km from the median position"
% (site, len(lindex), np.max(ddata) * 1.E-3))
print(
"-- %s time series will be split into code %s and code %s"
% (site, site, new_code))
# create a new gts
new_gts = Gts(code=new_code,
data_xyz=np.copy(my_ts.data_xyz[lindex]))
new_gts.xyz2neu(corr=True)
# remove the line from my_ts
my_ts.data_xyz = np.delete(my_ts.data_xyz, lindex, axis=0)
my_ts.xyz2neu(corr=True)
# update the ouput
if in_place:
self.append(new_gts)
else:
new_Sgts.append(new_gts)
if in_place:
self.__dict__[site] = my_ts
else:
new_Sgts.append(my_ts)
if in_place:
return self
else:
return new_Sgts
import matplotlib.pyplot as plt
plt.plot(t_save, y)
plt.plot(t, y_model)
plt.show()
########################################################################################
# MAIN
########################################################################################
from pyacs.lib import astrotime as at
from pyacs.gts.Gts import Gts
raw_ts = Gts()
my_ts = raw_ts.read_pos(
tsfile=
'/Users/nocquet/projets/2018/soam_proc/test_pyacs_061_snx/pos/IBEC.pos')
t = at.decyear2mjd(my_ts.data[:, 0])
y = my_ts.data[:, 2] * 1.E3
sy = my_ts.data[:, 5] * 1.E3
offset_dates = [at.decyear2mjd(2012.524590164)]
#offset_dates=[]
eq_dates = [at.decyear2mjd(2016.292349727)]
#eq_dates=[]
H_constraints = {}
bounds = {}
