def process_crinex_file(crinez, filename, data_rejected, data_retry):
# create a uuid temporary folder in case we cannot read the year and doy from the file (and gets rejected)
reject_folder = os.path.join(data_rejected, str(uuid.uuid4()))
try:
cnn = dbConnection.Cnn("gnss_data.cfg")
Config = pyOptions.ReadOptions("gnss_data.cfg")
archive = pyArchiveStruct.RinexStruct(cnn)
# apply local configuration (path to repo) in the executing node
crinez = os.path.join(Config.repository_data_in, crinez)
except Exception:
return traceback.format_exc() + ' while opening the database to process file ' + \
crinez + ' node ' + platform.node(), None
# assume a default networkcode
NetworkCode = 'rnx'
# get the station code year and doy from the filename
fileparts = archive.parse_crinex_filename(filename)
if fileparts:
StationCode = fileparts[0].lower()
doy = int(fileparts[1])
year = int(Utils.get_norm_year_str(fileparts[3]))
else:
event = pyEvents.Event(
Description='Could not read the station code, year or doy for file '
+ crinez,
EventType='error')
error_handle(cnn,
event,
crinez,
reject_folder,
filename,
no_db_log=True)
return event['Description'], None
# we can now make better reject and retry folders
reject_folder = os.path.join(
data_rejected, '%reason%/' + Utils.get_norm_year_str(year) + '/' +
Utils.get_norm_doy_str(doy))
retry_folder = os.path.join(
data_retry, '%reason%/' + Utils.get_norm_year_str(year) + '/' +
Utils.get_norm_doy_str(doy))
try:
# main try except block
with pyRinex.ReadRinex(NetworkCode, StationCode,
crinez) as rinexinfo: # type: pyRinex.ReadRinex
# STOP! see if rinexinfo is a multiday rinex file
if not verify_rinex_multiday(cnn, rinexinfo, Config):
# was a multiday rinex. verify_rinex_date_multiday took care of it
return None, None
# DDG: we don't use otl coefficients because we need an approximated coordinate
# we therefore just calculate the first coordinate without otl
# NOTICE that we have to trust the information coming in the RINEX header (receiver type, antenna type, etc)
# we don't have station info data! Still, good enough
# the final PPP coordinate will be calculated by pyScanArchive on a different process
# make sure that the file has the appropriate coordinates in the header for PPP.
# put the correct APR coordinates in the header.
# ppp didn't work, try using sh_rx2apr
brdc = pyBrdc.GetBrdcOrbits(Config.brdc_path, rinexinfo.date,
rinexinfo.rootdir)
# inflate the chi**2 limit to make sure it will pass (even if we get a crappy coordinate)
try:
rinexinfo.auto_coord(brdc, chi_limit=1000)
# normalize header to add the APR coordinate
# empty dict since nothing extra to change (other than the APR coordinate)
rinexinfo.normalize_header(dict())
except pyRinex.pyRinexExceptionNoAutoCoord:
# could not determine an autonomous coordinate, try PPP anyways. 50% chance it will work
pass
with pyPPP.RunPPP(
rinexinfo,
'',
Config.options,
Config.sp3types,
Config.sp3altrn,
rinexinfo.antOffset,
strict=False,
apply_met=False,
clock_interpolation=True) as ppp: # type: pyPPP.RunPPP
try:
ppp.exec_ppp()
except pyPPP.pyRunPPPException as ePPP:
# inflate the chi**2 limit to make sure it will pass (even if we get a crappy coordinate)
# if coordinate is TOO bad it will get kicked off by the unreasonable geodetic height
try:
auto_coords_xyz, auto_coords_lla = rinexinfo.auto_coord(
brdc, chi_limit=1000)
except pyRinex.pyRinexExceptionNoAutoCoord as e:
# catch pyRinexExceptionNoAutoCoord and convert it into a pyRunPPPException
raise pyPPP.pyRunPPPException(
'Both PPP and sh_rx2apr failed to obtain a coordinate for %s.\n'
'The file has been moved into the rejection folder. '
'Summary PPP file and error (if exists) follows:\n%s\n\n'
'ERROR section:\n%s\npyRinex.auto_coord error follows:\n%s'
% (crinez.replace(Config.repository_data_in, ''),
ppp.summary, str(ePPP).strip(), str(e).strip()))
# DDG: this is correct - auto_coord returns a numpy array (calculated in ecef2lla),
# so ppp.lat = auto_coords_lla is consistent.
ppp.lat = auto_coords_lla[0]
ppp.lon = auto_coords_lla[1]
ppp.h = auto_coords_lla[2]
ppp.x = auto_coords_xyz[0]
ppp.y = auto_coords_xyz[1]
ppp.z = auto_coords_xyz[2]
# check for unreasonable heights
if ppp.h[0] > 9000 or ppp.h[0] < -400:
raise pyRinex.pyRinexException(
os.path.relpath(crinez, Config.repository_data_in) +
' : unreasonable geodetic height (%.3f). '
'RINEX file will not enter the archive.' % (ppp.h[0]))
Result, match, _ = ppp.verify_spatial_coherence(
cnn, StationCode)
if Result:
# insert: there is only 1 match with the same StationCode.
rinexinfo.rename(NetworkCode=match[0]['NetworkCode'])
insert_data(cnn, archive, rinexinfo)
else:
if len(match) == 1:
error = "%s matches the coordinate of %s.%s (distance = %8.3f m) but the filename " \
"indicates it is %s. Please verify that this file belongs to %s.%s, rename it and " \
"try again. The file was moved to the retry folder. " \
"Rename script and pSQL sentence follows:\n" \
"BASH# mv %s %s\n" \
"PSQL# INSERT INTO stations (\"NetworkCode\", \"StationCode\", \"auto_x\", " \
"\"auto_y\", \"auto_z\", \"lat\", \"lon\", \"height\") VALUES " \
"('???','%s', %12.3f, %12.3f, %12.3f, " \
"%10.6f, %10.6f, %8.3f)\n" \
% (os.path.relpath(crinez, Config.repository_data_in), match[0]['NetworkCode'],
match[0]['StationCode'], float(match[0]['distance']), StationCode,
match[0]['NetworkCode'], match[0]['StationCode'],
os.path.join(retry_folder, filename),
os.path.join(retry_folder, filename.replace(StationCode, match[0]['StationCode'])),
StationCode, ppp.x, ppp.y, ppp.z, ppp.lat[0], ppp.lon[0], ppp.h[0])
raise pyPPP.pyRunPPPExceptionCoordConflict(error)
elif len(match) > 1:
# a number of things could have happened:
# 1) wrong station code, and more than one matching stations
# (that do not match the station code, of course)
# see rms.lhcl 2007 113 -> matches rms.igm0: 34.293 m, rms.igm1: 40.604 m, rms.byns: 4.819 m
# 2) no entry in the database for this solution -> add a lock and populate the exit args
# no match, but we have some candidates
error = "Solution for RINEX in repository (%s %s) did not match a unique station location " \
"(and station code) within 5 km. Possible cantidate(s): %s. This file has been moved " \
"to data_in_retry. pSQL sentence follows:\n" \
"PSQL# INSERT INTO stations (\"NetworkCode\", \"StationCode\", \"auto_x\", " \
"\"auto_y\", \"auto_z\", \"lat\", \"lon\", \"height\") VALUES " \
"('???','%s', %12.3f, %12.3f, %12.3f, %10.6f, %10.6f, %8.3f)\n" \
% (os.path.relpath(crinez, Config.repository_data_in), rinexinfo.date.yyyyddd(),
', '.join(['%s.%s: %.3f m' %
(m['NetworkCode'], m['StationCode'], m['distance']) for m in match]),
StationCode, ppp.x, ppp.y, ppp.z, ppp.lat[0], ppp.lon[0], ppp.h[0])
raise pyPPP.pyRunPPPExceptionCoordConflict(error)
else:
# only found a station removing the distance limit (could be thousands of km away!)
# The user will have to add the metadata to the database before the file can be added,
# but in principle no problem was detected by the process. This file will stay in this folder
# so that it gets analyzed again but a "lock" will be added to the file that will have to be
# removed before the service analyzes again.
# if the user inserted the station by then, it will get moved to the appropriate place.
# we return all the relevant metadata to ease the insert of the station in the database
otl = pyOTL.OceanLoading(StationCode,
Config.options['grdtab'],
Config.options['otlgrid'])
# use the ppp coordinates to calculate the otl
coeff = otl.calculate_otl_coeff(x=ppp.x,
y=ppp.y,
z=ppp.z)
# add the file to the locks table so that it doesn't get processed over and over
# this will be removed by user so that the file gets reprocessed once all the metadata is ready
cnn.insert('locks',
filename=os.path.relpath(
crinez, Config.repository_data_in))
return None, [
StationCode, (ppp.x, ppp.y, ppp.z), coeff,
(ppp.lat[0], ppp.lon[0], ppp.h[0]), crinez
]
except (pyRinex.pyRinexExceptionBadFile, pyRinex.pyRinexExceptionSingleEpoch, pyRinex.pyRinexExceptionNoAutoCoord) \
as e:
reject_folder = reject_folder.replace('%reason%', 'bad_rinex')
# add more verbose output
e.event['Description'] = e.event['Description'] + '\n' + os.path.relpath(crinez, Config.repository_data_in) + \
': (file moved to ' + reject_folder + ')'
e.event['StationCode'] = StationCode
e.event['NetworkCode'] = '???'
e.event['Year'] = year
e.event['DOY'] = doy
# error, move the file to rejected folder
error_handle(cnn, e.event, crinez, reject_folder, filename)
return None, None
except pyRinex.pyRinexException as e:
retry_folder = retry_folder.replace('%reason%', 'rinex_issues')
# add more verbose output
e.event['Description'] = e.event['Description'] + '\n' + os.path.relpath(crinez, Config.repository_data_in) + \
': (file moved to ' + retry_folder + ')'
e.event['StationCode'] = StationCode
e.event['NetworkCode'] = '???'
e.event['Year'] = year
e.event['DOY'] = doy
# error, move the file to rejected folder
error_handle(cnn, e.event, crinez, retry_folder, filename)
return None, None
except pyPPP.pyRunPPPExceptionCoordConflict as e:
retry_folder = retry_folder.replace('%reason%', 'coord_conflicts')
e.event['Description'] = e.event['Description'].replace(
'%reason%', 'coord_conflicts')
e.event['StationCode'] = StationCode
e.event['NetworkCode'] = '???'
e.event['Year'] = year
e.event['DOY'] = doy
error_handle(cnn, e.event, crinez, retry_folder, filename)
return None, None
except pyPPP.pyRunPPPException as e:
reject_folder = reject_folder.replace('%reason%', 'no_ppp_solution')
e.event['StationCode'] = StationCode
e.event['NetworkCode'] = '???'
e.event['Year'] = year
e.event['DOY'] = doy
error_handle(cnn, e.event, crinez, reject_folder, filename)
return None, None
except pyStationInfo.pyStationInfoException as e:
retry_folder = retry_folder.replace('%reason%',
'station_info_exception')
e.event['Description'] = e.event['Description'] + '. The file will stay in the repository and will be ' \
'processed during the next cycle of pyArchiveService.'
e.event['StationCode'] = StationCode
e.event['NetworkCode'] = '???'
e.event['Year'] = year
e.event['DOY'] = doy
error_handle(cnn, e.event, crinez, retry_folder, filename)
return None, None
except pyOTL.pyOTLException as e:
retry_folder = retry_folder.replace('%reason%', 'otl_exception')
e.event['Description'] = e.event['Description'] + ' while calculating OTL for %s. ' \
'The file has been moved into the retry folder.' \
% os.path.relpath(crinez, Config.repository_data_in)
e.event['StationCode'] = StationCode
e.event['NetworkCode'] = '???'
e.event['Year'] = year
e.event['DOY'] = doy
error_handle(cnn, e.event, crinez, retry_folder, filename)
return None, None
except pyProducts.pyProductsExceptionUnreasonableDate as e:
# a bad RINEX file requested an orbit for a date < 0 or > now()
reject_folder = reject_folder.replace('%reason%', 'bad_rinex')
e.event['Description'] = e.event['Description'] + ' during %s. The file has been moved to the rejected ' \
'folder. Most likely bad RINEX header/data.' \
% os.path.relpath(crinez, Config.repository_data_in)
e.event['StationCode'] = StationCode
e.event['NetworkCode'] = '???'
e.event['Year'] = year
e.event['DOY'] = doy
error_handle(cnn, e.event, crinez, reject_folder, filename)
return None, None
except pyProducts.pyProductsException as e:
# if PPP fails and ArchiveService tries to run sh_rnx2apr and it doesn't find the orbits, send to retry
retry_folder = retry_folder.replace('%reason%', 'sp3_exception')
e.event['Description'] = e.event['Description'] + ': %s. Check the brdc/sp3/clk files and also check that ' \
'the RINEX data is not corrupt.' \
% os.path.relpath(crinez, Config.repository_data_in)
e.event['StationCode'] = StationCode
e.event['NetworkCode'] = '???'
e.event['Year'] = year
e.event['DOY'] = doy
error_handle(cnn, e.event, crinez, retry_folder, filename)
return None, None
except dbConnection.dbErrInsert as e:
reject_folder = reject_folder.replace('%reason%', 'duplicate_insert')
# insert duplicate values: two parallel processes tried to insert different filenames
# (or the same) of the same station to the db: move it to the rejected folder.
# The user might want to retry later. Log it in events
# this case should be very rare
event = pyEvents.Event(
Description='Duplicate rinex insertion attempted while processing '
+ os.path.relpath(crinez, Config.repository_data_in) +
' : (file moved to rejected folder)\n' + str(e),
EventType='warn',
StationCode=StationCode,
NetworkCode='???',
Year=year,
DOY=doy)
error_handle(cnn, event, crinez, reject_folder, filename)
return None, None
except Exception:
retry_folder = retry_folder.replace('%reason%', 'general_exception')
event = pyEvents.Event(
Description=traceback.format_exc() + ' processing: ' +
os.path.relpath(crinez, Config.repository_data_in) + ' in node ' +
platform.node() + ' (file moved to retry folder)',
EventType='error')
error_handle(cnn,
event,
crinez,
retry_folder,
filename,
no_db_log=True)
return event['Description'], None
return None, None
def execute_ppp(rinexinfo,
args,
stnm,
options,
sp3types,
sp3altrn,
brdc_path,
erase,
apply_met=True,
decimate=True):
# put the correct APR coordinates in the header.
# stninfo = pyStationInfo.StationInfo(None, allow_empty=True)
stninfo = dict()
brdc = pyBrdc.GetBrdcOrbits(brdc_path, rinexinfo.date, rinexinfo.rootdir)
try:
# inflate the chi**2 limit
rinexinfo.purge_comments()
rinexinfo.auto_coord(brdc=brdc, chi_limit=1000)
rinexinfo.normalize_header(
stninfo) # empty dict: only applies the coordinate change
except pyRinex.pyRinexException as e:
print str(e)
if args.load_rinex:
rinexinfo.compress_local_copyto('./')
print 'RINEX created in current directory.'
return
otl_coeff = ''
try:
if args.ocean_loading or args.insert_sql:
# get a first ppp coordinate
ppp = pyPPP.RunPPP(rinexinfo,
'',
options,
sp3types,
sp3altrn,
0,
strict=False,
apply_met=False,
kinematic=False,
clock_interpolation=True)
ppp.exec_ppp()
# use it to get the OTL (when the auto_coord is very bad, PPP doesn't like the resulting OTL).
otl = pyOTL.OceanLoading(stnm, options['grdtab'],
options['otlgrid'], ppp.x, ppp.y, ppp.z)
otl_coeff = otl.calculate_otl_coeff()
# run again, with OTL
ppp = pyPPP.RunPPP(rinexinfo,
otl_coeff,
options,
sp3types,
sp3altrn,
0,
strict=False,
apply_met=apply_met,
kinematic=False,
clock_interpolation=True,
erase=erase,
decimate=decimate)
else:
ppp = pyPPP.RunPPP(rinexinfo,
'',
options,
sp3types,
sp3altrn,
0,
strict=False,
apply_met=apply_met,
kinematic=False,
clock_interpolation=True,
erase=erase,
decimate=decimate)
ppp.exec_ppp()
if not ppp.check_phase_center(ppp.proc_parameters):
print 'WARNING: phase center parameters not found for declared antenna!'
if not args.insert_sql:
print '%s %10.5f %13.4f %13.4f %13.4f %14.9f %14.9f %8.3f' % (
stnm, rinexinfo.date.fyear, ppp.x, ppp.y, ppp.z, ppp.lat[0],
ppp.lon[0], ppp.h[0])
else:
print 'INSERT INTO stations ("NetworkCode", "StationCode", "auto_x", "auto_y", "auto_z", ' \
'"Harpos_coeff_otl", lat, lon, height) VALUES ' \
'(\'???\', \'%s\', %.4f, %.4f, %.4f, \'%s\', %.8f, %.8f, %.3f)' \
% (stnm, ppp.x, ppp.y, ppp.z, otl_coeff, ppp.lat[0], ppp.lon[0], ppp.h[0])
if args.find:
cnn = dbConnection.Cnn('gnss_data.cfg')
Result, match, closest_stn = ppp.verify_spatial_coherence(
cnn, stnm)
if Result:
print 'Found matching station: %s.%s' % (
match[0]['NetworkCode'], match[0]['StationCode'])
elif not Result and len(match) == 1:
print '%s matches the coordinate of %s.%s (distance = %8.3f m) but the filename indicates it is %s' \
% (rinexinfo.rinex, match[0]['NetworkCode'], match[0]['StationCode'],
float(match[0]['distance']), stnm)
elif not Result and len(match) > 0:
print 'Solution for RINEX (%s %s) did not match a unique station location (and station code) ' \
'within 10 km. Possible cantidate(s): %s' \
% (rinexinfo.rinex, rinexinfo.date.yyyyddd(), ', '.join(['%s.%s: %.3f m' %
(m['NetworkCode'],
m['StationCode'],
m['distance']) for m in match]))
elif not Result and len(match) == 0 and len(closest_stn) > 0:
print 'No matches found. Closest station: %s.%s. (distance = %8.3f m)' \
% (closest_stn[0]['NetworkCode'], closest_stn[0]['StationCode'], closest_stn[0]['distance'])
except pyPPP.pyRunPPPException as e:
print 'Exception in PPP: ' + str(e)
except pyRinex.pyRinexException as e:
print 'Exception in pyRinex: ' + str(e)
def execute_ppp(rinexinfo,
args,
stnm,
options,
sp3types,
sp3altrn,
brdc_path,
erase,
apply_met=True,
decimate=True,
fix_coordinate=None,
solve_troposphere=105,
copy_results=None,
backward_substitution=False,
elevation_mask=5):
# put the correct APR coordinates in the header.
# stninfo = pyStationInfo.StationInfo(None, allow_empty=True)
brdc = pyBrdc.GetBrdcOrbits(brdc_path, rinexinfo.date, rinexinfo.rootdir)
try:
# inflate the chi**2 limit
rinexinfo.purge_comments()
rinexinfo.auto_coord(brdc=brdc, chi_limit=1000)
stninfo = {}
rinexinfo.normalize_header(
stninfo) # empty dict: only applies the coordinate change
except pyRinex.pyRinexException as e:
print(str(e))
if args.load_rinex:
rinexinfo.compress_local_copyto('./')
print('RINEX created in current directory.')
return
try:
otl_coeff = ''
if args.ocean_loading or args.insert_sql:
# get a first ppp coordinate
ppp = pyPPP.RunPPP(rinexinfo,
'',
options,
sp3types,
sp3altrn,
0,
strict=False,
apply_met=False,
kinematic=False,
clock_interpolation=True)
ppp.exec_ppp()
# use it to get the OTL (when the auto_coord is very bad, PPP doesn't like the resulting OTL).
otl = pyOTL.OceanLoading(stnm, options['grdtab'],
options['otlgrid'], ppp.x, ppp.y, ppp.z)
otl_coeff = otl.calculate_otl_coeff()
# run again, now with OTL coeff:
# determine if need to solve for coordinates or not
x = y = z = 0
if fix_coordinate is not None:
if len(fix_coordinate) > 1:
x = float(fix_coordinate[0])
y = float(fix_coordinate[1])
z = float(fix_coordinate[2])
else:
# read from file
cstr = file_readlines(fix_coordinate[0])
xyz = re.findall(
r'%s (-?\d+\.\d+)\s+(-?\d+\.\d+)\s+(-?\d+\.\d+)' %
rinexinfo.StationCode, ''.join(cstr), re.IGNORECASE)
if len(xyz):
x = float(xyz[0][0])
y = float(xyz[0][1])
z = float(xyz[0][2])
else:
print(
'WARNING: coordinate fixing invoked but could not find %s in list of coordinates -> '
'unfixing station coordinate in PPP' %
rinexinfo.StationCode)
fix_coordinate = False
print('%14.4f %14.4f %14.4f' % (x, y, z))
ppp = pyPPP.RunPPP(
rinexinfo,
otl_coeff,
options,
sp3types,
sp3altrn,
0,
strict=False,
apply_met=apply_met,
kinematic=False,
clock_interpolation=True,
erase=erase,
decimate=decimate,
solve_coordinates=True if not fix_coordinate else False,
solve_troposphere=solve_troposphere,
back_substitution=backward_substitution,
elev_mask=elevation_mask,
x=x,
y=y,
z=z)
ppp.exec_ppp()
if not ppp.check_phase_center(ppp.proc_parameters):
print(
'WARNING: phase center parameters not found for declared antenna!'
)
if not args.insert_sql:
print(
'%s %10.5f %13.4f %13.4f %13.4f %14.9f %14.9f %8.3f %8.3f %8.3f %8.3f %8.3f %8.3f'
%
(stnm, rinexinfo.date.fyear, ppp.x, ppp.y, ppp.z, ppp.lat[0],
ppp.lon[0], ppp.h[0], ppp.clock_phase, ppp.clock_phase_sigma,
ppp.phase_drift, ppp.phase_drift_sigma, ppp.clock_rms))
else:
print('INSERT INTO stations ("NetworkCode", "StationCode", "auto_x", "auto_y", "auto_z", ' \
'"Harpos_coeff_otl", lat, lon, height) VALUES ' \
'(\'???\', \'%s\', %.4f, %.4f, %.4f, \'%s\', %.8f, %.8f, %.3f)' \
% (stnm, ppp.x, ppp.y, ppp.z, otl_coeff, ppp.lat[0], ppp.lon[0], ppp.h[0]))
if args.find:
cnn = dbConnection.Cnn('gnss_data.cfg')
Result, match, closest_stn = ppp.verify_spatial_coherence(
cnn, stnm)
if Result:
print('Found matching station: %s.%s' %
(match[0]['NetworkCode'], match[0]['StationCode']))
elif len(match) == 1:
print('%s matches the coordinate of %s.%s (distance = %8.3f m) but the filename indicates it is %s' \
% (rinexinfo.rinex,
match[0]['NetworkCode'],
match[0]['StationCode'],
float(match[0]['distance']),
stnm))
elif len(match) > 0:
print('Solution for RINEX (%s %s) did not match a unique station location (and station code) ' \
'within 10 km. Possible cantidate(s): %s' \
% (rinexinfo.rinex,
rinexinfo.date.yyyyddd(),
', '.join(['%s.%s: %.3f m' %
(m['NetworkCode'],
m['StationCode'],
m['distance']) for m in match])))
elif len(match) == 0 and len(closest_stn) > 0:
print('No matches found. Closest station: %s.%s. (distance = %8.3f m)' \
% (closest_stn[0]['NetworkCode'],
closest_stn[0]['StationCode'],
closest_stn[0]['distance']))
if copy_results:
copy_results = copy_results[0]
try:
fpath = os.path.join(copy_results, rinexinfo.StationCode)
if not os.path.exists(fpath):
os.makedirs(fpath)
shutil.copyfile(
ppp.path_res_file,
os.path.join(fpath, os.path.basename(ppp.path_res_file)))
shutil.copyfile(
ppp.path_pos_file,
os.path.join(fpath, os.path.basename(ppp.path_pos_file)))
shutil.copyfile(
ppp.path_ses_file,
os.path.join(fpath, os.path.basename(ppp.path_ses_file)))
shutil.copyfile(
ppp.path_sum_file,
os.path.join(fpath, os.path.basename(ppp.path_sum_file)))
shutil.copyfile(
os.path.join(ppp.rootdir, 'commands.cmd'),
os.path.join(fpath,
os.path.basename(ppp.path_sum_file) + '.cmd'))
except Exception as e:
print(
'WARNING: There was a problem copying results to %s: %s' %
(copy_results, str(e)))
except pyPPP.pyRunPPPException as e:
print('Exception in PPP: ' + str(e))
except pyRinex.pyRinexException as e:
print('Exception in pyRinex: ' + str(e))