def to_json(self, json_file):
# print(repr(self.transformations))
file_write(
json_file,
json.dumps({'transformations': self.transformations},
indent=4,
sort_keys=False,
default=json_converter))
def create_files():
cnn = dbConnection.Cnn("gnss_data.cfg")
rs = cnn.query('SELECT * FROM stations WHERE "NetworkCode" NOT LIKE \'?%%\' AND "Harpos_coeff_otl" LIKE \'%%HARPOS%%\' ORDER BY "NetworkCode", "StationCode"')
# rs = cnn.query(
# 'SELECT * FROM stations WHERE "NetworkCode" NOT LIKE \'?%%\' ORDER BY "NetworkCode", "StationCode"')
stations = rs.dictresult()
print(' >> Cantidad de estaciones a procesar en Chalmers: %d' % len(stations))
stnlist = []
index = 0
for stn in stations:
stnlist += ['%-24s %16.3f%16.3f%16.3f' % (stn['NetworkCode'] + '.' + stn['StationCode'],
float(stn['auto_x']),
float(stn['auto_y']),
float(stn['auto_z']))]
if len(stnlist) == 99:
body = '\n'.join(stnlist)
file_write('otl_%i.list' % index,
body)
index += 1
# msg = MIMEText(body)
# msg['Subject'] = 'Subject: Ocean Loading Tides'
# msg['From'] = '*****@*****.**'
# msg['To'] = '*****@*****.**'
#
# s = smtplib.SMTP_SSL('64.233.190.108', 465)
# s.ehlo()
# s.login('*****@*****.**', 'demostenes0624')
# s.sendmail('*****@*****.**', '*****@*****.**', msg.as_string())
# s.close()
stnlist = []
if len(stnlist) > 0:
body = '\n'.join(stnlist)
file_write('otl_%i.list' % index,
body)
def plot_etm(cnn, stack, station, directory):
try:
ts = stack.get_station(station['NetworkCode'], station['StationCode'])
ts = pyETM.GamitSoln(cnn, ts, station['NetworkCode'],
station['StationCode'], stack.project)
etm = pyETM.GamitETM(cnn,
station['NetworkCode'],
station['StationCode'],
gamit_soln=ts)
pngfile = os.path.join(directory, stationID(etm) + '_gamit.png')
jsonfile = os.path.join(directory, stationID(etm) + '_gamit.json')
etm.plot(pngfile, plot_missing=False)
file_write(
os.path.join(jsonfile),
json.dumps(etm.todictionary(False), indent=4, sort_keys=False))
except pyETM.pyETMException as e:
tqdm.write(str(e))
def __init__(self, remote_pwd, params, solution_pwd):
self.pwd = remote_pwd
self.solution_pwd = solution_pwd
self.pwd_igs = os.path.join(remote_pwd, 'igs')
self.pwd_brdc = os.path.join(remote_pwd, 'brdc')
self.pwd_rinex = os.path.join(remote_pwd, 'rinex')
self.pwd_tables = os.path.join(remote_pwd, 'tables')
self.params = params
self.options = params['options']
self.orbits = params['orbits']
self.gamitopts = params['gamitopts']
self.date = params['date']
self.success = False
self.stdout = ''
self.stderr = ''
self.p = None
file_write(os.path.join(self.solution_pwd, 'monitor.log'),
now_str() +
' -> GamitTask initialized for %s: %s\n' % (self.params['DirName'],
self.date.yyyyddd()))
def pull_rinex(cnn, date, Config, JobServer):
# before starting the sync, determine if there were any station code changes that will require file deletions in AWS
# Join aws_sync with stations. If an entry in aws_sync has has no record in stations, station was renamed and needs
# to be deleted. It will be resent in this run.
rs = cnn.query(
'SELECT a."NetworkCode", a."StationCode", a."StationAlias" FROM aws_sync as a '
'LEFT JOIN stations as s on '
'a."NetworkCode" = s."NetworkCode" and '
'a."StationCode" = s."StationCode" '
'WHERE "Year" = %i AND "DOY" = %i AND s."StationCode" IS NULL' %
(date.year, date.doy))
deletes = rs.dictresult()
for stn in deletes:
# produce a single file with the deletions that need to occur in the AWS
file_append(
'file_ops.log',
'rm %s/%s* # %s.%s not found in stations table with net.stn code declared in aws_sync\n'
% (date.yyyyddd().replace(' ', '/'), stn['StationAlias'],
stn['NetworkCode'], stn['StationCode']))
# delete the records from aws_sync
cnn.query(
'DELETE FROM aws_sync WHERE "Year" = %i AND "DOY" = %i AND "NetworkCode" = \'%s\' AND '
'"StationCode" = \'%s\'' %
(date.year, date.doy, stn['NetworkCode'], stn['StationCode']))
# Join aws_sync with stationalias (stationalias is FK-ed to stations).
# If an entry in aws_sync that has StationCode <> StationAlias has no record in stationalias OR
# the stationalias declared is different than the station alias in the aws_sync, delete from AWS.
# will be resent in this batch
rs = cnn.query(
'SELECT a."NetworkCode", a."StationCode", a."StationAlias" FROM aws_sync as a '
'LEFT JOIN stationalias as sa on '
'a."NetworkCode" = sa."NetworkCode" and '
'a."StationCode" = sa."StationCode" '
'WHERE "Year" = %i AND "DOY" = %i AND '
'a."StationAlias" <> sa."StationAlias" OR '
'(sa."StationAlias" IS NULL AND a."StationCode" <> a."StationAlias")' %
(date.year, date.doy))
deletes = rs.dictresult()
for stn in deletes:
# produce a single file with the deletions that need to occur in the AWS
file_append(
'file_ops.log',
'rm %s/%s* # alias declared in aws_sync for %s.%s does not match alias in stationalias table\n'
% (date.yyyyddd().replace(' ', '/'), stn['StationAlias'],
stn['NetworkCode'], stn['StationCode']))
# delete the records from aws_sync
cnn.query(
'DELETE FROM aws_sync WHERE "Year" = %i AND "DOY" = %i AND "NetworkCode" = \'%s\' AND '
'"StationCode" = \'%s\'' %
(date.year, date.doy, stn['NetworkCode'], stn['StationCode']))
# check the individual files for this day. All files reported as uploaded should have a match in the rinex_proc
# table, otherwise this could be a station split or deletion. If that's the case, order their deletion from the AWS
rs = cnn.query(
'SELECT a."NetworkCode", a."StationCode", a."StationAlias" FROM aws_sync as a '
'LEFT JOIN rinex_proc as rx on '
'a."NetworkCode" = rx."NetworkCode" and '
'a."StationCode" = rx."StationCode" and '
'a."Year" = rx."ObservationYear" and '
'a."DOY" = rx."ObservationDOY" '
'WHERE "Year" = %i AND "DOY" = %i AND '
'rx."StationCode" IS NULL ' % (date.year, date.doy))
deletes = rs.dictresult()
for stn in deletes:
# produce a single file with the deletions that need to occur in the AWS
file_append(
'file_ops.log',
'rm %s/%s* # rinex file for %s.%s could not be found in the rinex_proc table\n'
% (date.yyyyddd().replace(' ', '/'), stn['StationAlias'],
stn['NetworkCode'], stn['StationCode']))
# delete the records from aws_sync
cnn.query(
'DELETE FROM aws_sync WHERE "Year" = %i AND "DOY" = %i AND "NetworkCode" = \'%s\' AND '
'"StationCode" = \'%s\'' %
(date.year, date.doy, stn['NetworkCode'], stn['StationCode']))
####################################################################################################################
# continue with sync of files
####################################################################################################################
# behavior requested by Abel: ALWAYS output the metadata but don't output a RINEX if already synced.
rs = cnn.query(
'SELECT rinex_proc.* FROM rinex_proc '
'WHERE "ObservationYear" = %i AND "ObservationDOY" = %i AND "Completion" >= 0.3'
% (date.year, date.doy))
rinex = rs.dictresult()
pbar = tqdm(total=len(rinex), ncols=80)
metafile = date.yyyy() + '/' + date.ddd() + '/' + date.yyyyddd().replace(
' ', '-')
date_subpath = date.yyyy() + '/' + date.ddd()
date_path = './' + date_path
# following Abel's request, make a subdir for the files
lele_path = '/media/leleiona/aws-files/' + date_subpath
for p in (date_path, lele_path):
if not os.path.isdir(p):
os.makedirs(p)
# write the header to the .info file
file_write(
'./' + metafile + '.info',
'*SITE Station Name Session Start Session Stop Ant Ht HtCod Ant N Ant E '
'Receiver Type Vers SwVer Receiver SN Antenna Type Dome '
'Antenna SN \n')
modules = ('dbConnection', 'pyETM', 'pyDate', 'pyRinex', 'pyStationInfo',
'pyOptions', 'pyArchiveStruct', 'os', 'numpy', 'traceback',
'platform', 'Utils', 'shutil')
depfuncs = (window_rinex, sigmas_neu2xyz)
JobServer.create_cluster(rinex_task,
depfuncs,
output_handle,
pbar,
modules=modules)
for rnx in rinex:
JobServer.submit(rnx['NetworkCode'], rnx['StationCode'], date,
rnx['ObservationFYear'], metafile)
JobServer.wait()
pbar.close()
JobServer.close_cluster()
print('Done, chau!')
def main():
parser = argparse.ArgumentParser(
description='Plot ETM for stations in the database')
parser.add_argument(
'stnlist',
type=str,
nargs='+',
help=
"List of networks/stations to plot given in [net].[stnm] format or just [stnm] "
"(separated by spaces; if [stnm] is not unique in the database, all stations with that "
"name will be plotted). Use keyword 'all' to plot all stations in all networks. "
"If [net].all is given, all stations from network [net] will be plotted"
)
parser.add_argument('-nop',
'--no_plots',
action='store_true',
help="Do not produce plots",
default=False)
parser.add_argument('-nom',
'--no_missing_data',
action='store_true',
help="Do not show missing days",
default=False)
parser.add_argument('-nm',
'--no_model',
action='store_true',
help="Plot time series without fitting a model")
parser.add_argument('-r',
'--residuals',
action='store_true',
help="Plot time series residuals",
default=False)
parser.add_argument(
'-dir',
'--directory',
type=str,
help=
"Directory to save the resulting PNG files. If not specified, assumed to be the "
"production directory")
parser.add_argument(
'-json',
'--json',
type=int,
help="Export ETM adjustment to JSON. Append '0' to just output "
"the ETM parameters, '1' to export time series without "
"model and '2' to export both time series and model.")
parser.add_argument(
'-gui',
'--interactive',
action='store_true',
help="Interactive mode: allows to zoom and view the plot interactively"
)
parser.add_argument(
'-win',
'--time_window',
nargs='+',
metavar='interval',
help=
'Date range to window data. Can be specified in yyyy/mm/dd, yyyy.doy or as a single '
'integer value (N) which shall be interpreted as last epoch-N')
parser.add_argument(
'-q',
'--query',
nargs=2,
metavar='{type} {date}',
type=str,
help=
'Dates to query the ETM. Specify "model" or "solution" to get the ETM value or the value '
'of the daily solution (if exists). Output is in XYZ.')
parser.add_argument(
'-gamit',
'--gamit',
type=str,
nargs=1,
metavar='{stack}',
help="Plot the GAMIT time series specifying which stack name to plot.")
parser.add_argument(
'-lang',
'--language',
type=str,
help="Change the language of the plots. Default is English. "
"Use ESP to select Spanish. To add more languages, "
"include the ISO 639-1 code in pyETM.py",
default='ENG')
parser.add_argument('-hist',
'--histogram',
action='store_true',
help="Plot histogram of residuals")
parser.add_argument('-file',
'--filename',
type=str,
help="Obtain data from an external source (filename). "
"Format should be specified with -format.")
parser.add_argument(
'-format',
'--format',
nargs='+',
type=str,
help=
"To be used together with --filename. Specify order of the fields as found in the input "
"file. Format strings are gpsWeek, gpsWeekDay, year, doy, fyear, month, day, mjd, "
"x, y, z, na. Use 'na' to specify a field that should be ignored. If fields to be ignored "
"are at the end of the line, then there is no need to specify those.")
parser.add_argument('-outliers',
'--plot_outliers',
action='store_true',
help="Plot an additional panel with the outliers")
parser.add_argument('-vel',
'--velocity',
action='store_true',
help="During query, output the velocity in XYZ.")
parser.add_argument('-seasonal',
'--seasonal_terms',
action='store_true',
help="During query, output the seasonal terms in NEU.")
parser.add_argument('-quiet',
'--suppress_messages',
action='store_true',
help="Quiet mode: suppress information messages")
args = parser.parse_args()
cnn = dbConnection.Cnn('gnss_data.cfg')
stnlist = process_stnlist(cnn, args.stnlist)
# define the language
pyETM.LANG = args.language.lower()
# set the logging level
if not args.suppress_messages:
pyETM.logger.setLevel(pyETM.INFO)
#####################################
# date filter
dates = None
if args.time_window is not None:
if len(args.time_window) == 1:
try:
dates = process_date(args.time_window,
missing_input=None,
allow_days=False)
dates = (dates[0].fyear, )
except ValueError:
# an integer value
dates = float(args.time_window[0])
else:
dates = process_date(args.time_window)
dates = (dates[0].fyear, dates[1].fyear)
if stnlist:
# do the thing
if args.directory:
if not os.path.exists(args.directory):
os.mkdir(args.directory)
else:
if not os.path.exists('production'):
os.mkdir('production')
args.directory = 'production'
for stn in stnlist:
try:
if args.gamit is None and args.filename is None:
etm = pyETM.PPPETM(cnn, stn['NetworkCode'],
stn['StationCode'], False,
args.no_model)
elif args.filename is not None:
etm = from_file(args, cnn, stn)
else:
polyhedrons = cnn.query_float(
'SELECT "X", "Y", "Z", "Year", "DOY" FROM stacks '
'WHERE "name" = \'%s\' AND "NetworkCode" = \'%s\' AND '
'"StationCode" = \'%s\' '
'ORDER BY "Year", "DOY", "NetworkCode", "StationCode"'
% (args.gamit[0], stn['NetworkCode'],
stn['StationCode']))
soln = pyETM.GamitSoln(cnn, polyhedrons,
stn['NetworkCode'],
stn['StationCode'], args.gamit[0])
etm = pyETM.GamitETM(cnn,
stn['NetworkCode'],
stn['StationCode'],
False,
args.no_model,
gamit_soln=soln)
# print ' > %5.2f %5.2f %5.2f %i %i' % \
# (etm.factor[0]*1000, etm.factor[1]*1000, etm.factor[2]*1000, etm.soln.t.shape[0],
# etm.soln.t.shape[0] -
# np.sum(np.logical_and(np.logical_and(etm.F[0], etm.F[1]), etm.F[2])))
# print two largest outliers
if etm.A is not None:
lres = np.sqrt(np.sum(np.square(etm.R), axis=0))
slres = lres[np.argsort(-lres)]
print(' >> Two largest residuals:')
for i in [0, 1]:
print(' %s %6.3f %6.3f %6.3f' %
(pyDate.Date(mjd=etm.soln.mjd[
lres == slres[i]]).yyyyddd(),
etm.R[0, lres == slres[i]],
etm.R[1, lres == slres[i]],
etm.R[2, lres == slres[i]]))
if args.interactive:
xfile = None
else:
postfix = "gamit"
if args.gamit is None:
postfix = "ppp" if args.filename is None else "file"
xfile = os.path.join(
args.directory, '%s.%s_%s' %
(etm.NetworkCode, etm.StationCode, postfix))
# leave pngfile empty to enter interactive mode (GUI)
if not args.no_plots:
etm.plot(xfile + '.png',
t_win=dates,
residuals=args.residuals,
plot_missing=not args.no_missing_data,
plot_outliers=args.plot_outliers)
if args.histogram:
etm.plot_hist(xfile + '_hist.png')
if args.json is not None:
if args.json == 1:
obj = etm.todictionary(time_series=True)
elif args.json == 2:
obj = etm.todictionary(time_series=True, model=True)
else:
obj = etm.todictionary(False)
file_write(xfile + '.json',
json.dumps(obj, indent=4, sort_keys=False))
if args.query is not None:
model = (args.query[0] == 'model')
q_date = pyDate.Date(fyear=float(args.query[1]))
xyz, _, _, txt = etm.get_xyz_s(q_date.year,
q_date.doy,
force_model=model)
strp = ''
# if user requests velocity too, output it
if args.velocity:
if etm.A is not None:
vxyz = etm.rotate_2xyz(etm.Linear.p.params[:, 1])
strp = '%8.5f %8.5f %8.5f ' \
% (vxyz[0, 0], vxyz[1, 0], vxyz[2, 0])
# also output seasonal terms, if requested
if args.seasonal_terms and etm.Periodic.frequency_count > 0:
strp += ' '.join([
'%8.5f' % (x * 1000)
for x in etm.Periodic.p.params.flatten().tolist()
])
print(' %s.%s %14.5f %14.5f %14.5f %8.3f %s -> %s' \
% (etm.NetworkCode, etm.StationCode, xyz[0], xyz[1], xyz[2], q_date.fyear, strp, txt))
print('Successfully plotted ' + stn['NetworkCode'] + '.' +
stn['StationCode'])
except pyETM.pyETMException as e:
print(str(e))
except Exception:
print('Error during processing of ' + stn['NetworkCode'] +
'.' + stn['StationCode'])
print(traceback.format_exc())
def config_session(self):
options = self.options
# create the def file
file_write(os.path.join(self.rootdir, 'gpsppp.def'),
"'LNG' 'ENGLISH'\n"
"'TRF' 'gpsppp.trf'\n"
"'SVB' 'gpsppp.svb_gnss_yrly'\n"
"'PCV' '%s'\n"
"'FLT' 'gpsppp.flt'\n"
"'OLC' '%s.olc'\n"
"'MET' 'gpsppp.met'\n"
"'ERP' '%s'\n"
"'GSD' '%s'\n"
"'GSD' '%s'\n"
% (os.path.basename(self.atx),
self.rinex.StationCode,
self.eop_file,
options['institution'],
options['info']))
file_write(os.path.join(self.rootdir, 'commands.cmd'),
"' UT DAYS OBSERVED (1-45)' 1\n"
"' USER DYNAMICS (1=STATIC,2=KINEMATIC)' %s\n"
"' OBSERVATION TO PROCESS (1=COD,2=C&P)' 2\n"
"' FREQUENCY TO PROCESS (1=L1,2=L2,3=L3)' 3\n"
"' SATELLITE EPHEMERIS INPUT (1=BRD ,2=SP3)' 2\n"
"' SATELLITE PRODUCT (1=NO,2=Prc,3=RTCA,4=RTCM)' 2\n"
"' SATELLITE CLOCK INTERPOLATION (1=NO,2=YES)' %s\n"
"' IONOSPHERIC GRID INPUT (1=NO,2=YES)' 1\n"
"' SOLVE STATION COORDINATES (1=NO,2=YES)' 2\n"
"' SOLVE TROP. (1=NO,2-5=RW MM/HR) (+100=grad) ' 105\n"
"' BACKWARD SUBSTITUTION (1=NO,2=YES)' 1\n"
"' REFERENCE SYSTEM (1=NAD83,2=ITRF)' 2\n"
"' COORDINATE SYSTEM(1=ELLIPSOIDAL,2=CARTESIAN)' 2\n"
"' A-PRIORI PSEUDORANGE SIGMA (m)' 2.000\n"
"' A-PRIORI CARRIER PHASE SIGMA (m)' 0.015\n"
"' LATITUDE (ddmmss.sss,+N) or ECEF X (m)' 0.0000\n"
"' LONGITUDE (ddmmss.sss,+E) or ECEF Y (m)' 0.0000\n"
"' HEIGHT (m) or ECEF Z (m)' 0.0000\n"
"' ANTENNA HEIGHT (m)' %6.4f\n"
"' CUTOFF ELEVATION (deg)' 10.000\n"
"' GDOP CUTOFF ' 20.000\n"
% ('1' if not self.kinematic else '2', '1'
if not self.clock_interpolation else '2', self.antH))
file_write(os.path.join(self.rootdir, 'input.inp'),
"%s\n"
"commands.cmd\n"
"0 0\n"
"0 0\n"
"orbits/%s\n"
"orbits/%s\n"
"orbits/%s\n"
"orbits/%s\n"
% (self.rinex.rinex,
self.orbits1.sp3_filename,
self.clocks1.clk_filename,
self.orbits2.sp3_filename,
self.clocks2.clk_filename))
def write_otl(self):
file_write(os.path.join(self.rootdir, self.rinex.StationCode + '.olc'),
self.otl_coeff)