def print_residuals(NetworkCode, StationCode, residuals, lat, lon, components=('N', 'E', 'U')):
# check if sending NEU or XYZ
if components[0] == 'X':
cresiduals = ct2lg(residuals[0], residuals[1], residuals[2], lat, lon)
ccomponent = ('N', 'E', 'U')
else:
cresiduals = lg2ct(residuals[0], residuals[1], residuals[2], lat, lon)
ccomponent = ('X', 'Y', 'Z')
r = ''
for i, c in enumerate(components):
r = r + ' %s: ' % c + ' '.join(['%8.4f' % np.multiply(k, 1000) for k in residuals[i]]) + \
' %s: ' % ccomponent[i] + ' '.join(['%8.4f' % np.multiply(k, 1000) for k in cresiduals[i]]) + '\n'
tqdm.write(' -- %s.%s\n' % (NetworkCode, StationCode) + r)
def parse_summary(self):
self.summary = ''.join(self.out)
self.ppp_version = re.findall(r'.*Version\s+(\d.\d+)\/', self.summary)
if len(self.ppp_version) == 0:
self.ppp_version = re.findall(r'.*CSRS-PPP ver.\s+(\d.\d+)\/', self.summary)[0]
else:
self.ppp_version = self.ppp_version[0]
self.file_summary = self.get_text(self.summary, 'SECTION 1.', 'SECTION 2.')
self.proc_parameters = self.get_text(self.summary, 'SECTION 2. ', ' SECTION 3. ')
self.observation_session = self.get_text(self.summary,
'3.2 Observation Session', '3.3 Coordinate estimates')
self.coordinate_estimate = self.get_text(self.summary,
'3.3 Coordinate estimates', '3.4 Coordinate differences ITRF')
self.clock_estimates = self.get_text(self.summary,
'3.5 Receiver clock estimates', '3.6 Observation rejection table')
if self.strict and not self.check_phase_center(self.proc_parameters):
raise pyRunPPPException(
'Error while running PPP: could not find the antenna and radome in antex file. '
'Check RINEX header for formatting issues in the ANT # / TYPE field. RINEX header follows:\n' + ''.join(
self.rinex.get_header()))
if self.strict and not self.check_otl(self.proc_parameters):
raise pyRunPPPException(
'Error while running PPP: could not find the OTL coefficients. '
'Check RINEX header for formatting issues in the APPROX ANT POSITION field. If APR is too far from OTL '
'coordinates (declared in the HARPOS or BLQ format) NRCAN will reject the coefficients. '
'OTL coefficients record follows:\n' + self.otl_coeff)
if not self.check_eop(self.file_summary):
raise pyRunPPPExceptionEOPError('EOP returned NaN in Pole XYZ.')
# parse rejected and accepted observations
self.processed_obs, self.rejected_obs = self.get_pr_observations(self.observation_session, self.kinematic)
if self.processed_obs == 0:
raise pyRunPPPExceptionZeroProcEpochs('PPP returned zero processed epochs')
# if self.strict and (self.processed_obs == 0 or self.rejected_obs > 0.95 * self.processed_obs):
# raise pyRunPPPExceptionTooFewAcceptedObs('The processed observations (' + str(self.processed_obs) +
# ') is zero or more than 95% of the observations were rejected (' +
# str(self.rejected_obs) + ')')
# FRAME now comes from the startup process, where the function Utils.determine_frame is called
# self.frame = self.get_frame(self.coordinate_estimate)
self.x, self.y, self.z = self.get_xyz(self.coordinate_estimate)
self.lat, self.lon, self.h = ecef2lla([self.x, self.y, self.z])
self.sigmax, self.sigmay, self.sigmaz, \
self.sigmaxy, self.sigmaxz, self.sigmayz = self.get_sigmas(self.coordinate_estimate, self.kinematic)
self.clock_phase, self.clock_phase_sigma, \
self.phase_drift, self.phase_drift_sigma, \
self.clock_rms, self.clock_rms_number = self.get_clock(self.clock_estimates, self.kinematic)
# not implemented in PPP: apply NE offset if is NOT zero
if self.rinex.antOffsetN != 0.0 or self.rinex.antOffsetE != 0.0:
dx, dy, dz = lg2ct(numpy.array(self.rinex.antOffsetN), numpy.array(self.rinex.antOffsetE),
numpy.array([0]), self.lat, self.lon)
# reduce coordinates
self.x -= dx[0]
self.y -= dy[0]
self.z -= dz[0]
self.lat, self.lon, self.h = ecef2lla([self.x, self.y, self.z])
def remove_common_modes(self, target_periods=None, use_stations=None):
if target_periods is None:
tqdm.write(' >> Removing periodic common modes...')
# load all the periodic terms
etm_objects = self.cnn.query_float('SELECT etmsv2."NetworkCode", etmsv2."StationCode", stations.lat, '
'stations.lon, '
'frequencies as freq, params FROM etmsv2 '
'LEFT JOIN stations ON '
'etmsv2."NetworkCode" = stations."NetworkCode" AND '
'etmsv2."StationCode" = stations."StationCode" '
'WHERE "object" = \'periodic\' AND soln = \'gamit\' '
'AND frequencies <> \'{}\' '
'ORDER BY etmsv2."NetworkCode", etmsv2."StationCode"', as_dict=True)
else:
tqdm.write(' >> Inheriting periodic components...')
# load the periodic terms of the stations that will produce the inheritance
etm_objects = self.cnn.query_float('SELECT etmsv2."NetworkCode", etmsv2."StationCode", stations.lat, '
'stations.lon, '
'frequencies as freq, params FROM etmsv2 '
'LEFT JOIN stations ON '
'etmsv2."NetworkCode" = stations."NetworkCode" AND '
'etmsv2."StationCode" = stations."StationCode" '
'WHERE "object" = \'periodic\' AND soln = \'gamit\' '
'AND frequencies <> \'{}\' AND etmsv2."NetworkCode" || \'.\' || '
'etmsv2."StationCode" IN (\'%s\') '
'ORDER BY etmsv2."NetworkCode", etmsv2."StationCode"'
% '\', \''.join(use_stations), as_dict=True)
# load the frequencies to subtract
frequencies = self.cnn.query_float('SELECT frequencies FROM etmsv2 WHERE soln = \'gamit\' AND '
'object = \'periodic\' '
'AND frequencies <> \'{}\' GROUP BY frequencies', as_dict=True)
# get the unique list of frequencies
f_vector = []
for freq in frequencies:
f_vector += [f for f in freq['frequencies']]
f_vector = np.array(list(set(f_vector)))
# initialize the vectors
ox = np.zeros((len(f_vector), len(etm_objects), 2))
oy = np.zeros((len(f_vector), len(etm_objects), 2))
oz = np.zeros((len(f_vector), len(etm_objects), 2))
for s, p in enumerate(etm_objects):
tqdm.write(' -- Periodic parameters for %s.%s' % (p['NetworkCode'], p['StationCode']))
if target_periods:
n = []
e = []
u = []
# inheritance invoked! we want to remove the difference between current periodic terms and target
# terms from the parent frame
for i in (0, 1):
# i is a var to select the sin and cos terms from the target_periods structure
for f in p['freq']:
t = target_periods[p['StationCode']]['%.3f' % (1 / f)]
n.append(t['n'][i])
e.append(t['e'][i])
u.append(t['u'][i])
params = np.array(p['params']) - np.array([n, e, u]).flatten()
else:
# no inheritance: make a vector of current periodic terms to be removed as common modes
params = np.array(p['params'])
params = params.reshape((3, params.shape[0] / 3))
param_count = params.shape[1] / 2
# convert from NEU to XYZ
for j in range(params.shape[1]):
params[:, j] = np.array(lg2ct(params[0, j], params[1, j], params[2, j],
p['lat'], p['lon'])).flatten()
for i, f in enumerate(p['freq']):
ox[f_vector == f, s] = params[0, i:i + param_count + 1:param_count]
oy[f_vector == f, s] = params[1, i:i + param_count + 1:param_count]
oz[f_vector == f, s] = params[2, i:i + param_count + 1:param_count]
# build the design matrix using the stations involved in inheritance or all stations if no inheritance
sql_where = ','.join(["'" + stn['NetworkCode'] + '.' + stn['StationCode'] + "'" for stn in etm_objects])
x = self.cnn.query_float('SELECT 0, -auto_z*1e-9, auto_y*1e-9, 1, 0, 0 FROM stations WHERE '
'"NetworkCode" || \'.\' || "StationCode" '
'IN (%s) ORDER BY "NetworkCode", "StationCode"' % sql_where)
y = self.cnn.query_float('SELECT auto_z*1e-9, 0, -auto_x*1e-9, 0, 1, 0 FROM stations WHERE '
'"NetworkCode" || \'.\' || "StationCode" '
'IN (%s) ORDER BY "NetworkCode", "StationCode"' % sql_where)
z = self.cnn.query_float('SELECT -auto_y*1e-9, auto_x*1e-9, 0, 0, 0, 1 FROM stations WHERE '
'"NetworkCode" || \'.\' || "StationCode" '
'IN (%s) ORDER BY "NetworkCode", "StationCode"' % sql_where)
Ax = np.array(x)
Ay = np.array(y)
Az = np.array(z)
A = np.row_stack((Ax, Ay, Az))
# loop through the frequencies
for freq in f_vector:
for i, cs in enumerate((np.sin, np.cos)):
L = np.row_stack((ox[f_vector == freq, :, i].flatten(),
oy[f_vector == freq, :, i].flatten(),
oz[f_vector == freq, :, i].flatten())).flatten()
c = np.linalg.lstsq(A, L, rcond=-1)[0]
# loop through all the polyhedrons
for poly in tqdm(self, ncols=160, desc=' -- Applying transformation -> %s(2 * pi * 1/%.2f)' %
(cs.__name__, np.divide(1., freq))):
# subtract the inverted common modes
poly.vertices['x'] = poly.vertices['x'] - cs(2 * pi * freq * 365.25 * poly.date.fyear) * \
np.dot(poly.Ax, c)
poly.vertices['y'] = poly.vertices['y'] - cs(2 * pi * freq * 365.25 * poly.date.fyear) * \
np.dot(poly.Ay, c)
poly.vertices['z'] = poly.vertices['z'] - cs(2 * pi * freq * 365.25 * poly.date.fyear) * \
np.dot(poly.Az, c)
tqdm.write(' -- Done!')
def align_spaces(self, target_dict):
# get the list of stations to use during the alignment
use_stations = target_dict.keys()
# reference date used to align the stack
# epochs SHOULD all be the same. Get first item and then the epoch
ref_date = Date(fyear=target_dict.values()[0]['epoch'])
# convert the target dict to a list
target_list = []
stack_list = []
tqdm.write(' >> Aligning coordinate space...')
for stn in use_stations:
if not np.isnan(target_dict[stn]['x']):
target_list.append((stn, target_dict[stn]['x'], target_dict[stn]['y'], target_dict[stn]['z'],
ref_date.year, ref_date.doy, ref_date.fyear))
# get the ETM coordinate for this station
net = stn.split('.')[0]
ssn = stn.split('.')[1]
ts = pyETM.GamitSoln(self.cnn, self.get_station(net, ssn), net, ssn, self.name)
etm = pyETM.GamitETM(self.cnn, net, ssn, gamit_soln=ts)
stack_list += etm.get_etm_soln_list()
c_array = np.array(stack_list, dtype=[('stn', 'S8'), ('x', 'float64'), ('y', 'float64'),
('z', 'float64'), ('yr', 'i4'), ('dd', 'i4'),
('fy', 'float64')])
comb = Polyhedron(c_array, 'etm', ref_date)
# build a target polyhedron from the target_list
vertices = np.array(target_list, dtype=[('stn', 'S8'), ('x', 'float64'), ('y', 'float64'),
('z', 'float64'), ('yr', 'i4'), ('dd', 'i4'),
('fy', 'float64')])
target = Polyhedron(vertices, 'target_frame', ref_date)
# start aligning the coordinates
tqdm.write(' -- Aligning polyhedron at %.3f (%s)' % (ref_date.fyear, ref_date.yyyyddd()))
scale = False
# align the polyhedron to the target
r_before, r_after, a_stn = comb.align(target, scale=scale, verbose=True)
# extract the Helmert parameters to apply to the rest of the polyhedrons
# remove the scale factor
helmert = comb.helmert
tqdm.write(' -- Reporting coordinate space residuals (in mm) before and after frame alignment\n'
' Before After | Before After ')
# format r_before and r_after to satisfy the required print_residuals format
r_before = r_before.reshape(3, r_before.shape[0] / 3).transpose()
r_after = r_after.reshape(3, r_after.shape[0] / 3).transpose()
residuals = np.stack((r_before, r_after), axis=2)
stn_lla = []
for i, stn in enumerate(a_stn):
n = stn.split('.')[0]
s = stn.split('.')[1]
# get the lat lon of the station to report back in the json
lla = self.cnn.query_float('SELECT lat, lon FROM stations WHERE "NetworkCode" = \'%s\' '
'AND "StationCode" = \'%s\'' % (n, s))[0]
stn_lla.append([lla[0], lla[1]])
# print residuals to screen
print_residuals(n, s, residuals[i], lla[0], lla[1], ['X', 'Y', 'Z'])
# save the position space residuals
self.position_space = {'stations': {'codes': a_stn.tolist(), 'latlon': stn_lla},
'residuals_before_alignment': r_before.tolist(),
'residuals_after_alignment': r_after.tolist(),
'reference_date': ref_date,
'helmert_transformation': comb.helmert.tolist(),
'comments': 'No scale factor estimated.'}
for poly in tqdm(self, ncols=160, desc=' -- Applying coordinate space transformation'):
if poly.date != ref_date:
poly.align(helmert=helmert, scale=scale)
tqdm.write(' >> Aligning velocity space...')
# choose the stations that have a velocity
use_stn = []
for stn in use_stations:
if not np.isnan(target_dict[stn]['vx']):
use_stn.append(stn)
# load the polynomial terms of the stations
etm_objects = self.cnn.query_float('SELECT etms."NetworkCode", etms."StationCode", stations.lat, '
'stations.lon, params FROM etms '
'LEFT JOIN stations ON '
'etms."NetworkCode" = stations."NetworkCode" AND '
'etms."StationCode" = stations."StationCode" '
'WHERE "object" = \'polynomial\' AND soln = \'gamit\' AND stack = \'%s\' '
'AND etms."NetworkCode" || \'.\' || etms."StationCode" IN (\'%s\') '
'ORDER BY etms."NetworkCode", etms."StationCode"'
% (self.name, '\', \''.join(use_stn)), as_dict=True)
# first, align the velocity space by finding a Helmert transformation that takes vx, vy, and vz of the stack at
# each station and makes it equal to vx, vy, and vz of the ITRF structure
dvx = np.zeros(len(etm_objects))
dvy = np.zeros(len(etm_objects))
dvz = np.zeros(len(etm_objects))
for s, p in enumerate(etm_objects):
stn_ts = self.get_station(p['NetworkCode'], p['StationCode'])
self.cnn.query('DELETE FROM etms WHERE "soln" = \'gamit\' AND "NetworkCode" = \'%s\' AND '
'"StationCode" = \'%s\' AND stack = \'%s\' '
% (p['NetworkCode'], p['StationCode'], self.name))
# save the time series
ts = pyETM.GamitSoln(self.cnn, stn_ts, p['NetworkCode'], p['StationCode'], self.name)
# create the ETM object
pyETM.GamitETM(self.cnn, p['NetworkCode'], p['StationCode'], False, False, ts)
q = self.cnn.query_float('SELECT params FROM etms '
'WHERE "object" = \'polynomial\' AND soln = \'gamit\' '
'AND "NetworkCode" = \'%s\' AND "StationCode" = \'%s\' AND stack = \'%s\' '
% (p['NetworkCode'], p['StationCode'], self.name), as_dict=True)[0]
params = np.array(q['params'])
params = params.reshape((3, params.shape[0] / 3))
# first item, i.e. params[:][0] in array is position
# second item is velocity, which is what we are interested in
v = np.array(lg2ct(params[0, 1], params[1, 1], params[2, 1], p['lat'], p['lon'])).flatten()
# put the residuals in an array
td = target_dict['%s.%s' % (p['NetworkCode'], p['StationCode'])]
dvx[s] = v[0] - np.array(td['vx'])
dvy[s] = v[1] - np.array(td['vy'])
dvz[s] = v[2] - np.array(td['vz'])
scale = False
A = self.build_design(etm_objects, scale=scale)
# loop through the frequencies
L = np.row_stack((dvx.flatten(), dvy.flatten(), dvz.flatten())).flatten()
c, _, _, _, wrms, _, it = adjust_lsq(A, L)
tqdm.write(' -- Velocity space transformation: ' + ' '.join(['%7.4f' % cc for cc in c]) +
' wrms: %.3f it: %i' % (wrms * 1000, it))
# loop through all the polyhedrons
for poly in tqdm(self, ncols=160, desc=' -- Applying velocity space transformation'):
t = np.repeat(poly.date.fyear - ref_date.fyear, poly.Ax.shape[0])
poly.vertices['x'] = poly.vertices['x'] - t * np.dot(poly.ax(scale=scale), c)
poly.vertices['y'] = poly.vertices['y'] - t * np.dot(poly.ay(scale=scale), c)
poly.vertices['z'] = poly.vertices['z'] - t * np.dot(poly.az(scale=scale), c)
tqdm.write(' -- Reporting velocity space residuals (in mm/yr) before and after frame alignment\n'
' Before After | Before After ')
dvxa = np.zeros(len(etm_objects))
dvya = np.zeros(len(etm_objects))
dvza = np.zeros(len(etm_objects))
for s, p in enumerate(etm_objects):
# redo the etm for this station
stn_ts = self.get_station(p['NetworkCode'], p['StationCode'])
self.cnn.query('DELETE FROM etms WHERE "soln" = \'gamit\' AND "NetworkCode" = \'%s\' AND '
'"StationCode" = \'%s\' AND stack = \'%s\''
% (p['NetworkCode'], p['StationCode'], self.name))
# save the time series
ts = pyETM.GamitSoln(self.cnn, stn_ts, p['NetworkCode'], p['StationCode'], self.name)
# create the ETM object
pyETM.GamitETM(self.cnn, p['NetworkCode'], p['StationCode'], False, False, ts)
q = self.cnn.query_float('SELECT params FROM etms '
'WHERE "object" = \'polynomial\' AND soln = \'gamit\' '
'AND "NetworkCode" = \'%s\' AND "StationCode" = \'%s\' AND stack = \'%s\''
% (p['NetworkCode'], p['StationCode'], self.name), as_dict=True)[0]
params = np.array(q['params'])
params = params.reshape((3, params.shape[0] / 3))
# first item, i.e. params[:][0] in array is position
# second item is velocity, which is what we are interested in
v = np.array(lg2ct(params[0, 1], params[1, 1], params[2, 1], p['lat'], p['lon'])).flatten()
# put the residuals in an array
td = target_dict['%s.%s' % (p['NetworkCode'], p['StationCode'])]
dvxa[s] = v[0] - np.array(td['vx'])
dvya[s] = v[1] - np.array(td['vy'])
dvza[s] = v[2] - np.array(td['vz'])
lla = self.cnn.query_float('SELECT lat, lon FROM stations WHERE "NetworkCode" = \'%s\' '
'AND "StationCode" = \'%s\'' % (p['NetworkCode'], p['StationCode']))[0]
print_residuals(p['NetworkCode'], p['StationCode'],
np.array([[dvx[s], dvxa[s]], [dvy[s], dvya[s]], [dvz[s], dvza[s]]]), lla[0], lla[1],
['X', 'Y', 'Z'])
# save the position space residuals
self.velocity_space = {'stations': {'codes': [p['NetworkCode'] + '.' + p['StationCode'] for p in etm_objects],
'latlon': [[p['lat'], p['lon']] for p in etm_objects]},
'residuals_before_alignment':
np.column_stack((dvx.flatten(), dvy.flatten(), dvz.flatten())).tolist(),
'residuals_after_alignment':
np.column_stack((dvxa.flatten(), dvya.flatten(), dvza.flatten())).tolist(),
'reference_date': ref_date,
'helmert_transformation': c.tolist(),
'comments': 'Velocity space transformation.'}
tqdm.write(' -- Done!')
def remove_common_modes(self, target_periods=None):
if target_periods is None:
tqdm.write(' >> Removing periodic common modes...')
# load all the periodic terms
etm_objects = self.cnn.query_float('SELECT etms."NetworkCode", etms."StationCode", stations.lat, '
'stations.lon, '
'frequencies as freq, params FROM etms '
'LEFT JOIN stations ON '
'etms."NetworkCode" = stations."NetworkCode" AND '
'etms."StationCode" = stations."StationCode" '
'WHERE "object" = \'periodic\' AND soln = \'gamit\' AND stack = \'%s\' '
'AND frequencies <> \'{}\' '
'ORDER BY etms."NetworkCode", etms."StationCode"'
% self.name, as_dict=True)
else:
use_stations = []
for s in target_periods.keys():
# check that the stations have not one or both periods with NaNs
if not np.isnan(target_periods[s]['365.250']['n'][0]) and \
not np.isnan(target_periods[s]['182.625']['n'][0]):
use_stations.append(s)
tqdm.write(' >> Inheriting periodic components...')
# load the periodic terms of the stations that will produce the inheritance
etm_objects = self.cnn.query_float('SELECT etms."NetworkCode", etms."StationCode", stations.lat, '
'stations.lon, '
'frequencies as freq, params FROM etms '
'LEFT JOIN stations ON '
'etms."NetworkCode" = stations."NetworkCode" AND '
'etms."StationCode" = stations."StationCode" '
'WHERE "object" = \'periodic\' AND soln = \'gamit\' AND stack = \'%s\' '
'AND frequencies <> \'{}\' AND etms."NetworkCode" || \'.\' || '
'etms."StationCode" IN (\'%s\') '
'ORDER BY etms."NetworkCode", etms."StationCode"'
% (self.name, '\', \''.join(use_stations)), as_dict=True)
# load the frequencies to subtract
frequencies = self.cnn.query_float('SELECT frequencies FROM etms WHERE soln = \'gamit\' AND '
'object = \'periodic\' AND frequencies <> \'{}\' AND stack = \'%s\' '
'GROUP BY frequencies' % self.name, as_dict=True)
# get the unique list of frequencies
f_vector = []
for freq in frequencies:
f_vector += [f for f in freq['frequencies']]
f_vector = np.array(list(set(f_vector)))
# initialize the vectors
ox = np.zeros((len(f_vector), len(etm_objects), 2))
oy = np.zeros((len(f_vector), len(etm_objects), 2))
oz = np.zeros((len(f_vector), len(etm_objects), 2))
# vector for residuals after alignment
rx = np.zeros((len(f_vector), len(etm_objects), 2))
ry = np.zeros((len(f_vector), len(etm_objects), 2))
rz = np.zeros((len(f_vector), len(etm_objects), 2))
tqdm.write(' -- Reporting periodic residuals (in mm) before %s'
% ('inheritance' if target_periods else 'common mode removal'))
for s, p in enumerate(etm_objects):
# DDG: should only activate this portion of the code if for any reason ETMs stored in database made with
# a stack different to what is being used here
# stn_ts = self.get_station(p['NetworkCode'], p['StationCode'])
# self.cnn.query('DELETE FROM etms WHERE "soln" = \'gamit\' AND "NetworkCode" = \'%s\' AND '
# '"StationCode" = \'%s\'' % (p['NetworkCode'], p['StationCode']))
# save the time series
# ts = pyETM.GamitSoln(self.cnn, stn_ts, p['NetworkCode'], p['StationCode'], self.project)
# create the ETM object
# pyETM.GamitETM(self.cnn, p['NetworkCode'], p['StationCode'], False, False, ts)
# REDUNDANT CALL, but leave anyways
q = self.cnn.query_float('SELECT frequencies as freq, * FROM etms '
'WHERE "object" = \'periodic\' AND soln = \'gamit\' '
'AND "NetworkCode" = \'%s\' AND '
'"StationCode" = \'%s\' AND stack = \'%s\''
% (p['NetworkCode'], p['StationCode'], self.name), as_dict=True)[0]
if target_periods:
n = []
e = []
u = []
# inheritance invoked! we want to remove the difference between current periodic terms and target
# terms from the parent frame
for k in range(2):
for f in q['freq']:
t = target_periods['%s.%s' % (p['NetworkCode'], p['StationCode'])]['%.3f' % (1 / f)]
n += [t['n'][k]]
e += [t['e'][k]]
u += [t['u'][k]]
params = np.array(q['params']) - np.array([n, e, u]).flatten()
else:
# no inheritance: make a vector of current periodic terms to be removed as common modes
params = np.array(q['params'])
params = params.reshape((3, params.shape[0] / 3))
param_count = params.shape[1] / 2
print_residuals(p['NetworkCode'], p['StationCode'], params, p['lat'], p['lon'])
# convert from NEU to XYZ
for j in range(params.shape[1]):
params[:, j] = np.array(lg2ct(params[0, j], params[1, j], params[2, j],
p['lat'], p['lon'])).flatten()
for i, f in enumerate(p['freq']):
ox[f_vector == f, s] = params[0, i:i + param_count + 1:param_count]
oy[f_vector == f, s] = params[1, i:i + param_count + 1:param_count]
oz[f_vector == f, s] = params[2, i:i + param_count + 1:param_count]
# build the design matrix using the stations involved in inheritance or all stations if no inheritance
sql_where = ','.join(["'" + stn['NetworkCode'] + '.' + stn['StationCode'] + "'" for stn in etm_objects])
x = self.cnn.query_float('SELECT 0, -auto_z*1e-9, auto_y*1e-9, 1, 0, 0, auto_x*1e-9 FROM stations WHERE '
'"NetworkCode" || \'.\' || "StationCode" '
'IN (%s) ORDER BY "NetworkCode", "StationCode"' % sql_where)
y = self.cnn.query_float('SELECT auto_z*1e-9, 0, -auto_x*1e-9, 0, 1, 0, auto_y*1e-9 FROM stations WHERE '
'"NetworkCode" || \'.\' || "StationCode" '
'IN (%s) ORDER BY "NetworkCode", "StationCode"' % sql_where)
z = self.cnn.query_float('SELECT -auto_y*1e-9, auto_x*1e-9, 0, 0, 0, 1, auto_z*1e-9 FROM stations WHERE '
'"NetworkCode" || \'.\' || "StationCode" '
'IN (%s) ORDER BY "NetworkCode", "StationCode"' % sql_where)
Ax = np.array(x)
Ay = np.array(y)
Az = np.array(z)
A = np.row_stack((Ax, Ay, Az))
solution_vector = []
# vector to display down-weighted stations
st = dict()
st['stn'] = [s['NetworkCode'] + '.' + s['StationCode'] for s in etm_objects]
xyzstn = ['X-%s' % ss for ss in st['stn']] + ['Y-%s' % ss for ss in st['stn']] + \
['Z-%s' % ss for ss in st['stn']]
# loop through the frequencies
for freq in f_vector:
for i, cs in enumerate((np.sin, np.cos)):
L = np.row_stack((ox[f_vector == freq, :, i].flatten(),
oy[f_vector == freq, :, i].flatten(),
oz[f_vector == freq, :, i].flatten())).flatten()
c, _, index, _, wrms, _, it = adjust_lsq(A, L)
# c = np.linalg.lstsq(A, L, rcond=-1)[0]
tqdm.write(' -- Transformation for %s(2 * pi * 1/%.2f) : %s'
% (cs.__name__, np.divide(1., freq), ' '.join(['%7.4f' % cc for cc in c])) +
' wrms: %.3f it: %i\n' % (wrms * 1000, it) +
' Down-weighted station components: %s'
% ' '.join(['%s' % ss for ss in np.array(xyzstn)[np.logical_not(index)]]))
# save the transformation parameters to output to json file
solution_vector.append(['%s(2 * pi * 1/%.2f)' % (cs.__name__, np.divide(1., freq)), c.tolist()])
# loop through all the polyhedrons
for poly in tqdm(self, ncols=160, desc=' -- Applying transformation -> %s(2 * pi * 1/%.2f)' %
(cs.__name__, np.divide(1., freq))):
# subtract the inverted common modes
poly.vertices['x'] = poly.vertices['x'] - cs(2 * pi * freq * 365.25 * poly.date.fyear) * \
np.dot(poly.ax(scale=True), c)
poly.vertices['y'] = poly.vertices['y'] - cs(2 * pi * freq * 365.25 * poly.date.fyear) * \
np.dot(poly.ay(scale=True), c)
poly.vertices['z'] = poly.vertices['z'] - cs(2 * pi * freq * 365.25 * poly.date.fyear) * \
np.dot(poly.az(scale=True), c)
tqdm.write(' -- Reporting periodic residuals (in mm) after %s\n'
' 365.25 182.62 365.25 182.62 \n'
' sin sin cos cos '
% ('inheritance' if target_periods else 'common mode removal'))
for s, p in enumerate(etm_objects):
# redo the etm for this station
# DDG: etms need to be redone because we changed the stack!
stn_ts = self.get_station(p['NetworkCode'], p['StationCode'])
self.cnn.query('DELETE FROM etms WHERE "soln" = \'gamit\' AND "NetworkCode" = \'%s\' AND '
'"StationCode" = \'%s\' AND stack = \'%s\''
% (p['NetworkCode'], p['StationCode'], self.name))
# save the time series
ts = pyETM.GamitSoln(self.cnn, stn_ts, p['NetworkCode'], p['StationCode'], self.name)
# create the ETM object
pyETM.GamitETM(self.cnn, p['NetworkCode'], p['StationCode'], False, False, ts)
# obtain the updated parameters
# they should exist for sure!
q = self.cnn.query_float('SELECT frequencies as freq, * FROM etms '
'WHERE "object" = \'periodic\' AND soln = \'gamit\' '
'AND "NetworkCode" = \'%s\' AND '
'"StationCode" = \'%s\' AND stack = \'%s\''
% (p['NetworkCode'], p['StationCode'], self.name), as_dict=True)[0]
if target_periods:
n = []
e = []
u = []
# inheritance invoked! we want to remove the difference between current periodic terms and target
# terms from the parent frame
for k in range(2):
for f in q['freq']:
t = target_periods['%s.%s' % (p['NetworkCode'], p['StationCode'])]['%.3f' % (1 / f)]
n += [t['n'][k]]
e += [t['e'][k]]
u += [t['u'][k]]
residuals = (np.array(q['params']) - np.array([n, e, u]).flatten())
else:
# residuals are the minimized frequencies
residuals = np.array(q['params'])
# reshape the array to NEU
residuals = residuals.reshape((3, residuals.shape[0] / 3))
param_count = residuals.shape[1] / 2
print_residuals(p['NetworkCode'], p['StationCode'], residuals, p['lat'], p['lon'])
# convert from NEU to XYZ
for j in range(residuals.shape[1]):
residuals[:, j] = np.array(lg2ct(residuals[0, j], residuals[1, j], residuals[2, j],
p['lat'], p['lon'])).flatten()
for i, f in enumerate(p['freq']):
rx[f_vector == f, s] = residuals[0, i:i + param_count + 1:param_count]
ry[f_vector == f, s] = residuals[1, i:i + param_count + 1:param_count]
rz[f_vector == f, s] = residuals[2, i:i + param_count + 1:param_count]
# save the position space residuals
self.periodic_space = {'stations': {'codes': [p['NetworkCode'] + '.' + p['StationCode'] for p in etm_objects],
'latlon': [[p['lat'], p['lon']] for p in etm_objects]},
'frequencies': f_vector.tolist(),
'components': ['sin', 'cos'],
'residuals_before_alignment': np.array([ox, oy, oz]).tolist(),
'residuals_after_alignment': np.array([rx, ry, rz]).tolist(),
'helmert_transformations': solution_vector,
'comments': 'Periodic space transformation. Each residual component (X, Y, Z) '
'stored as X[freq, station, component]. Frequencies, stations, and '
'components ordered as in respective elements.'}
tqdm.write(' -- Done!')
def remove_common_modes(self, cnn):
tqdm.write(' >> Removing periodic common modes...')
# load all the periodic terms
etm_objects = cnn.query_float(
'SELECT etmsv2."NetworkCode", etmsv2."StationCode", stations.lat, stations.lon, '
'frequencies as freq, params FROM etmsv2 '
'LEFT JOIN stations ON '
'etmsv2."NetworkCode" = stations."NetworkCode" AND '
'etmsv2."StationCode" = stations."StationCode" '
'WHERE "object" = \'periodic\' AND soln = \'gamit\' '
'AND frequencies <> \'{}\' '
'ORDER BY etmsv2."NetworkCode", etmsv2."StationCode"',
as_dict=True)
# load the frequencies to subtract
frequencies = cnn.query_float(
'SELECT frequencies FROM etmsv2 WHERE soln = \'gamit\' AND object = \'periodic\' '
'AND frequencies <> \'{}\' GROUP BY frequencies',
as_dict=True)
# get the unique list of frequencies
f_vector = []
for freq in frequencies:
f_vector += [f for f in freq['frequencies']]
f_vector = numpy.array(list(set(f_vector)))
ox = numpy.zeros((len(f_vector), len(etm_objects), 2))
oy = numpy.zeros((len(f_vector), len(etm_objects), 2))
oz = numpy.zeros((len(f_vector), len(etm_objects), 2))
for s, p in enumerate(etm_objects):
params = numpy.array(p['params'])
params = params.reshape((3, params.shape[0] / 3))
param_count = params.shape[1] / 2
# convert from NEU to XYZ
for j in range(params.shape[1]):
params[:, j] = numpy.array(
lg2ct(params[0, j], params[1, j], params[2, j], p['lat'],
p['lon'])).flatten()
for i, f in enumerate(p['freq']):
ox[f_vector == f,
s] = params[0, i:i + param_count + 1:param_count]
oy[f_vector == f,
s] = params[1, i:i + param_count + 1:param_count]
oz[f_vector == f,
s] = params[2, i:i + param_count + 1:param_count]
# build the design matrix
sql_where = ','.join([
"'" + stn['NetworkCode'] + '.' + stn['StationCode'] + "'"
for stn in etm_objects
])
x = cnn.query_float(
'SELECT 0, -auto_z, auto_y, 1, 0, 0 FROM stations WHERE "NetworkCode" || \'.\' || '
'"StationCode" IN (%s) ORDER BY "NetworkCode", "StationCode"' %
sql_where)
y = cnn.query_float(
'SELECT auto_z, 0, -auto_x, 0, 1, 0 FROM stations WHERE "NetworkCode" || \'.\' || '
'"StationCode" IN (%s) ORDER BY "NetworkCode", "StationCode"' %
sql_where)
z = cnn.query_float(
'SELECT -auto_y, auto_x, 0, 0, 0, 1 FROM stations WHERE "NetworkCode" || \'.\' || '
'"StationCode" IN (%s) ORDER BY "NetworkCode", "StationCode"' %
sql_where)
Ax = numpy.array(x)
Ay = numpy.array(y)
Az = numpy.array(z)
A = numpy.row_stack((Ax, Ay, Az))
# select everybody (not just the stations with ETMs)
x = cnn.query_float(
'SELECT 0, -"Z", "Y", 1, 0, 0 FROM stacks WHERE "Project" = \'%s\' '
'ORDER BY "NetworkCode", "StationCode", "FYear"' % self.name)
y = cnn.query_float(
'SELECT "Z", 0, -"X", 0, 1, 0 FROM stacks WHERE "Project" = \'%s\' '
'ORDER BY "NetworkCode", "StationCode", "FYear"' % self.name)
z = cnn.query_float(
'SELECT -"Y", "X", 0, 0, 0, 1 FROM stacks WHERE "Project" = \'%s\' '
'ORDER BY "NetworkCode", "StationCode", "FYear"' % self.name)
t = cnn.query_float(
'SELECT "FYear", "X", "Y", "Z" FROM stacks WHERE "Project" = \'%s\' '
'ORDER BY "NetworkCode", "StationCode", "FYear"' % self.name)
metadata = cnn.query(
'SELECT "NetworkCode", "StationCode", "Year", "DOY" FROM stacks '
'WHERE "Project" = \'%s\' ORDER BY "NetworkCode", "StationCode", "FYear"'
% self.name)
metadata = metadata.dictresult()
AX = numpy.array(x)
AY = numpy.array(y)
AZ = numpy.array(z)
t = numpy.array(t)
for freq in f_vector:
for i, cs in enumerate((numpy.sin, numpy.cos)):
L = numpy.row_stack(
(ox[f_vector == freq, :,
i].flatten(), oy[f_vector == freq, :, i].flatten(),
oz[f_vector == freq, :, i].flatten())).flatten()
c = numpy.linalg.lstsq(A, L, rcond=-1)[0]
# subtract the inverted common modes
t[:, 1] = t[:, 1] - cs(
2 * pi * freq * 365.25 * t[:, 0]) * numpy.dot(AX, c)
t[:, 2] = t[:, 2] - cs(
2 * pi * freq * 365.25 * t[:, 0]) * numpy.dot(AY, c)
t[:, 3] = t[:, 3] - cs(
2 * pi * freq * 365.25 * t[:, 0]) * numpy.dot(AZ, c)
polyhedron = []
for i, stn in enumerate(metadata):
polyhedron += [{
'NetworkCode': stn['NetworkCode'],
'StationCode': stn['StationCode'],
'X': t[i][1],
'Y': t[i][2],
'Z': t[i][3],
'Year': stn['Year'],
'DOY': stn['DOY'],
'FYear': t[i][0]
}]
return polyhedron