def create_replace_links(self):
replace_ln_file_path = os.path.join(self.pwd, 'replace_links.sh')
try:
replace_ln_file = file_open(replace_ln_file_path, 'w')
except (OSError, IOError):
raise Exception('could not open file ' + replace_ln_file_path)
replace_ln_file.write("""#!/bin/bash
set -e
for link; do
test -h "$link" || continue
dir=$(dirname "$link")
reltarget=$(readlink "$link")
case $reltarget in
/*) abstarget=$reltarget;;
*) abstarget=$dir/$reltarget;;
esac
rm -fv "$link"
cp -afv "$abstarget" "$link" || {
# on failure, restore the symlink
rm -rfv "$link"
ln -sfv "$reltarget" "$link"
}
done
""")
replace_ln_file.close()
chmod_exec(replace_ln_file_path)
def link_tables(self):
script_path = 'link_tables.sh'
try:
link_tables = file_open(script_path, 'w')
except (OSError, IOError):
raise GamitSessionException('Could not create script file link_tables.sh')
# link the apr file as the lfile.
contents = \
"""#!/bin/bash
# set up links
cd %s;
sh_links.tables -frame J2000 -year %s -eop %s -topt none &> sh_links.out;
# kill the earthquake rename file
rm eq_rename
# create an empty rename file
echo "" > eq_rename
cd ..;
""" % (self.pwd_tables, self.date.yyyy(), self.GamitOpts['eop_type'])
link_tables.write(contents)
link_tables.close()
chmod_exec(script_path)
os.system('./'+script_path)
def load_session_config(self, session_config_file, check_config):
try:
# parse session config file
config = configparser.ConfigParser()
with file_open(session_config_file) as f:
config.read_file(f)
# check that all required items are there and files exist
if check_config:
self.__check_config(config)
# get gamit config items from session config file
self.gamitopt.update(dict(config.items('gamit')))
self.NetworkConfig = pyBunch.Bunch().fromDict(
dict(config.items('network')))
if 'type' not in self.NetworkConfig.keys():
raise ValueError(
'Network "type" must be specified in config file: use "regional" or "global"'
)
self.gamitopt['gnss_data'] = config.get('Archive', 'gnss_data')
self.gamitopt['max_cores'] = int(self.gamitopt['max_cores'])
# TO-DO: check that all the required parameters are present
if len(self.gamitopt['expt']) != 4:
raise ValueError(
'The experiment name parameter must be 4 characters long.')
except configparser.NoOptionError:
raise
def __init__(self, sp3archive, date, sp3types, copyto, no_cleanup=False):
# try both compressed and non-compressed sp3 files
# loop through the types of sp3 files to try
self.sp3_path = None
self.RF = None
self.no_cleanup = no_cleanup
for sp3type in sp3types:
self.sp3_filename = sp3type + date.wwwwd() + '.sp3'
try:
pyProducts.OrbitalProduct.__init__(self, sp3archive, date, self.sp3_filename, copyto)
self.sp3_path = self.file_path
self.type = sp3type
break
except pyProducts.pyProductsExceptionUnreasonableDate:
raise
except pyProducts.pyProductsException:
# if the file was not found, go to next
continue
# if we get here and self.sp3_path is still none, then no type of sp3 file was found
if self.sp3_path is None:
raise pySp3Exception('Could not find a valid orbit file (types: ' +
', '.join(sp3types) + ') for '
'week ' + str(date.gpsWeek) +
' day ' + str(date.gpsWeekDay) +
' using any of the provided sp3 types')
# parse the RF of the orbit file
with file_open(self.sp3_path) as fileio:
line = fileio.readline()
self.RF = line[46:51].strip()
def create_otl_list(self):
otl_path = os.path.join(self.pwd_tables, 'otl.list')
if os.path.isfile(otl_path):
os.remove(otl_path)
with file_open(otl_path, 'w') as otl_list:
otl_list.write('%s 8-character GAMIT ID read by grdtab (M -> CM)\n' % (self.Config.options['otlmodel']))
otl_list.write("""$$ Ocean loading displacement
$$
$$ Calculated on holt using olfg/olmpp of H.-G. Scherneck
$$
$$ COLUMN ORDER: M2 S2 N2 K2 K1 O1 P1 Q1 MF MM SSA
$$
$$ ROW ORDER:
$$ AMPLITUDES (m)
$$ RADIAL
$$ TANGENTL EW
$$ TANGENTL NS
$$ PHASES (degrees)
$$ RADIAL
$$ TANGENTL EW
$$ TANGENTL NS
$$
$$ Displacement is defined positive in upwards, South and West direction.
$$ The phase lag is relative to Greenwich and lags positive. The
$$ Gutenberg-Bullen Greens function is used. In the ocean tide model the
$$ deficit of tidal water mass has been corrected by subtracting a uniform
$$ layer of water with a certain phase lag globally.
$$
$$ Complete <model name> : No interpolation of ocean model was necessary
$$ <model name>_PP : Ocean model has been interpolated near the station
$$ (PP = Post-Processing)
$$
$$ CMC: YES (corr.tide centre of mass)
$$
$$ Ocean tide model: %s
$$
$$ END HEADER
$$\n""" % (self.Config.options['otlmodel']))
for stn in self.StationInstances:
otl = stn.otl_H.split('\n')
# remove BLQ header
otl = otl[29:]
# need to change the station record for GAMIT to take it
otl[0] = ' %s' % stn.StationAlias.upper()
if stn.lon < 0:
lon = 360+stn.lon
else:
lon = stn.lon
otl[3] = '$$ %s RADI TANG lon/lat:%10.4f%10.4f' \
% (stn.StationAlias.upper(), lon, stn.lat)
otl_list.write('\n'.join(otl))
# write BLQ format termination
otl_list.write("$$ END TABLE\n")
def copy_sestbl_procdef_atx(self):
self.frame, atx = determine_frame(self.Config.options['frames'], self.date)
# copy process.defaults and sestbl.
copyfile(self.GamitOpts['process_defaults'],
os.path.join(self.pwd_tables, 'process.defaults'))
# copyfile(self.GamitOpts['atx'], os.path.join(self.pwd_tables, 'antmod.dat'))
copyfile(atx, os.path.join(self.pwd_tables, 'antmod.dat'))
# change the scratch directory in the sestbl. file
with file_open(os.path.join(self.pwd_tables, 'sestbl.'), 'w') as sestbl:
with file_open(self.GamitOpts['sestbl']) as orig_sestbl:
for line in orig_sestbl:
if 'Scratch directory' in line:
# empty means local directory! LA RE PU...
sestbl.write('Scratch directory = \n')
else:
sestbl.write(line)
def create_apr_sittbl_file(self):
lfile_path = os.path.join(self.pwd_tables, 'lfile.')
sittbl_path = os.path.join(self.pwd_tables, 'sittbl.')
log_path = os.path.join(self.pwd_tables, 'debug.log')
for f in (lfile_path, sittbl_path):
if os.path.isfile(f):
os.remove(f)
with file_open(lfile_path, 'w') as lfile:
with file_open(sittbl_path, 'w') as sittbl:
with file_open(log_path, 'w') as debug:
sittbl.write('SITE FIX --COORD.CONSTR.-- \n')
sittbl.write(' << default for regional sites >>\n')
sittbl.write('ALL NNN 100. 100. 100. \n')
for stn in self.StationInstances:
lfile .write(stn.GetApr() + '\n')
sittbl.write(stn.GetSittbl() + '\n')
debug .write(stn.DebugCoord() + '\n')
def replace_in_sinex(sinex, observations, unknowns, new_val):
new_unknowns = \
""" NUMBER OF UNKNOWNS%22i
NUMBER OF DEGREES OF FREEDOM%12i
PHASE MEASUREMENTS SIGMA 0.0025
SAMPLING INTERVAL (SECONDS) 30
""" % (new_val, observations - new_val)
sinex_path = os.path.basename(os.path.splitext(sinex)[0]) + '_MOD.snx'
with file_open(sinex_path, 'w') as nsnx:
with file_open(sinex, 'r') as osnx:
for line in osnx:
if ' NUMBER OF UNKNOWNS%22i' % unknowns in line:
# empty means local directory! LA RE PU...
nsnx.write(new_unknowns)
else:
nsnx.write(line)
# rename file
os.remove(sinex)
os.renames(sinex_path, sinex)
def create_sitedef(self):
sitedefFile = os.path.join(self.pwd_tables, 'sites.defaults')
try:
with file_open(sitedefFile, 'w') as sitedef:
sitedef.write(' all_sites %s xstinfo\n' % (self.GamitOpts['expt']))
for StationInstance in self.StationInstances:
sitedef.write(" %s_GPS %s localrx\n" % (StationInstance.StationAlias.upper(),
self.GamitOpts['expt']))
except Exception as e:
raise GamitSessionException(e)
def run(self):
retry = 0
while True:
cmd_stdin = None
try:
if self.cat_file:
cmd_stdin = file_open(
os.path.join(self.cwd or '', self.cat_file))
self.p = subprocess.Popen(
self.cmd.split(),
shell=False,
stdin=cmd_stdin,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
cwd=self.cwd,
close_fds=True,
bufsize=-1,
# text mode:
universal_newlines=True,
encoding='utf-8',
errors='ignore')
# Block until finalization
self.stdout, self.stderr = self.p.communicate()
break
except OSError as e:
if str(e) == '[Errno 35] Resource temporarily unavailable':
if retry <= 2:
retry += 1
# wait a moment
time.sleep(0.5)
continue
else:
print(self.cmd)
raise OSError(
str(e) + ' after 3 retries on node: ' +
platform.node())
else:
print(self.cmd)
raise
except Exception:
print(self.cmd)
raise
finally:
if cmd_stdin:
cmd_stdin.close()
def create_station_info(self):
# delete any current station.info files
station_path = os.path.join(self.pwd_tables, 'station.info')
if os.path.isfile(station_path):
os.remove(station_path)
with file_open(station_path, 'w') as stninfo_file:
stninfo_file.write('*SITE Station Name Session Start Session Stop Ant Ht HtCod '
'Ant N Ant E Receiver Type Vers SwVer '
'Receiver SN Antenna Type Dome Antenna SN \n')
for stn in self.StationInstances:
# stninfo_file.write(stn.StationInfo.return_stninfo() + '\n')
stninfo_file.write(stn.GetStationInformation() + '\n')
def linktables(self, year, eop_type):
script_path = os.path.join(self.pwd_comb, 'link_tables.sh')
try:
link_tables = file_open(script_path, 'w')
except:
raise GlobkException('could not open file link_tables.sh')
# link the apr file as the lfile.
with link_tables:
contents = \
"""#!/bin/bash
# set up links
sh_links.tables -frame J2000 -year %s -eop %s -topt none &> sh_links.out;
# link the bulletin A
ln -s ~/gg/tables/pmu.usno .
""" % (year, eop_type)
link_tables.write(contents)
chmod_exec(script_path)
def create_finish_script(self):
year = self.date.yyyy()
doy = self.date.ddd()
# extract the gps week and convert to string
gpsWeek_str = str(self.date.gpsWeek)
# normalize gps week string
if self.date.gpsWeek < 1000: gpsWeek_str = '0' + gpsWeek_str
# extract the gps week and day of week
gps_week = self.date.gpsWeek
gps_week_day = self.date.gpsWeekDay
finish_file_path = os.path.join(self.pwd, 'finish.sh')
try:
finish_file = file_open(finish_file_path,'w')
except (OSError, IOError):
raise Exception('could not open file '+finish_file_path)
contents = """#!/bin/bash
export INSTITUTE=%s
echo "finish.sh (`date +"%%Y-%%m-%%d %%T"`): Finish script started" >> monitor.log
# set the name of the outfile
FILE=%s%s%s
DOY=%s
YEAR=%s
# move to the solution path
if [ ! -d ./glbf ]; then
# something went wrong! no glbf dir
mkdir glbf
fi
cd glbf
# make sure an h file exists, if not exit
if [ ! -f ../$DOY/h*.${YEAR}${DOY} ]; then
echo "FATAL in finish.sh (`date +"%%Y-%%m-%%d %%T"`): h-files not found in $DOY folder. Exit" >> ../monitor.log
exit;
fi
# get the WL and NL ambiguity resolution and the nrms double diff statistics
echo "finish.sh (`date +"%%Y-%%m-%%d %%T"`): NRMS and WL-NL ambiguity summary follows:" >> ../monitor.log
grep 'nrms' ../$DOY/sh_gamit_${DOY}.summary >> ../monitor.log
grep 'WL fixed' ../$DOY/sh_gamit_${DOY}.summary >> ../monitor.log
# link the svnav.dat file
ln -s ../tables/svnav.dat .
# create the binary h-file
htoglb . tmp.svs -a ../$DOY/h*.${YEAR}${DOY} >> ../${FILE}.out
# grep any missing stations to report them to monitor.log
grep 'No data for site ' ../${FILE}.out | sort | uniq >> ../monitor.log
# convert the binary h-file to sinex file
glbtosnx . "" h*.glx ${FILE}.snx >> ../${FILE}.out
# clean up
rm HTOGLB.* tmp.svs l* svnav.dat
# move back to home
cd ..;
""" % (self.gamitopts['org'], self.gamitopts['org'], gpsWeek_str, str(gps_week_day), doy, year[2:4])
# dump contents to the script file
finish_file.write(contents)
# this section is to calculate the orbits
if self.gamitopts['expt_type'] == 'relax':
# create an sp3 file from the g-file
contents = """
# move to the solutions directory
cd ./solutions/*
# make temporary directory
mkdir tmp
# copy the gfile to temp dir
cp gfile* tmp/
# move to the temp dir
cd tmp;
# do the damn thing
mksp3.sh %s %s %s
# copy the sp3 file to solution dir if exists
[ -e *.sp3 ] && mv *.sp3 ..;
# move out of temporary directory
cd ..;
# clean up
rm -rf tmp gfile*;
# back to home directory
cd ../..
""" % (year,doy,self.options['org'])
finish_file.write(contents)
return
contents = """
# move to the solutions directory
cd $DOY
# rename o file to znd file
if [ -f o*a.[0-9][0-9][0-9]* ]; then
mv -f o*a.[0-9][0-9][0-9]* ../glbf/%s%s%s.znd;
fi
# remove a priori o file
if [ -f o*p.[0-9][0-9][0-9]* ]; then
rm -f o*p.[0-9][0-9][0-9]*;
fi
# restore home dir
cd ..
""" % (self.gamitopts['org'], gpsWeek_str, str(gps_week_day))
finish_file.write(contents)
contents = """
# move to the solutions directory
cd $DOY
# clean up
# remove the grid files, rinex files, etc
rm -rf gfile* *.grid ????????.??o
# compress remaining files
for file in $(ls);do gzip --force $file; done
# return to home directory
cd ..
cd rinex
rm -rf *
cd ..
echo "finish.sh (`date +"%Y-%m-%d %T"`): Done processing h-files and generating SINEX." >> monitor.log
"""
finish_file.write(contents)
# make sure to close the file
finish_file.close()
# add executable permissions
chmod_exec(finish_file_path)
def create_combination_script(self, org, gpsweek, gpsweekday, sites):
# set the path and name for the run script
run_file_path = os.path.join(self.pwd_comb, 'globk.sh')
try:
run_file = file_open(run_file_path, 'w')
except:
raise GlobkException('could not open file '+run_file_path)
sites = split_string(sites, 80)
site_list_string = []
for s in sites:
site_list_string.append('echo " use_site %s" >> globk.cmd' % s)
site_string = '\n'.join(site_list_string)
contents = \
"""#!/bin/bash
export INSTITUTE=%s
export GPSWEEK=%s
export GPSWEEKDAY=%i
# data product file names
OUT_FILE=${INSTITUTE}${GPSWEEK}${GPSWEEKDAY};
# create global directory listing for globk
for file in $(find . -name "*.GLX" -print | sort);do echo "$file";done | grep "\/n0\/" > globk.gdl
for file in $(find . -name "*.GLX" -print | sort);do echo "$file";done | grep -v "\/n0\/" >> globk.gdl
# create the globk cmd file
echo " eq_file eq_rename.txt" > globk.cmd
echo " use_site clear" >> globk.cmd
%s
echo " prt_opt GDLF MIDP CMDS PLST " >> globk.cmd
echo " out_glb $OUT_FILE.GLX" >> globk.cmd
echo " in_pmu /opt/gamit_globk/tables/pmu.usno" >> globk.cmd
echo " descript Weekly combined solution at $INSTITUTE" >> globk.cmd
echo " max_chii 3. 0.6" >> globk.cmd
echo " apr_site all 1 1 1 0 0 0" >> globk.cmd
# DO NOT ACTIVATE ATM COMBINATION BECAUSE IT WILL NOT WORK!
echo "#apr_atm all 1 1 1" >> globk.cmd
# create the sinex header file
echo "+FILE/REFERENCE " > head.snx
echo " DESCRIPTION Instituto Geografico Nacional " >> head.snx
echo " OUTPUT Solucion GPS combinada " >> head.snx
echo " CONTACT [email protected] " >> head.snx
echo " SOFTWARE glbtosnx Version " >> head.snx
echo " HARDWARE . " >> head.snx
echo " INPUT Archivos binarios Globk " >> head.snx
echo "-FILE/REFERENCE " >> head.snx
# run globk
globk 0 file.prt globk.log globk.gdl globk.cmd 2>&1 > globk.out
# convert the GLX file into sinex
glbtosnx . head.snx $OUT_FILE.GLX ${OUT_FILE}.snx 2>&1 > glbtosnx.out
# figure out where the parameters start in the prt file
LINE=`grep -n "PARAMETER ESTIMATES" file.prt | cut -d ":" -f1`
# reduce line by one to make a little cleaner
let LINE--;
# print prt header
sed -n 1,${LINE}p file.prt > ${OUT_FILE}.out
# append the log file
cat globk.log >> ${OUT_FILE}.out
# create the fsnx file which contains only the solution estimate
lineNumber=`grep --binary-file=text -m 1 -n "\-SOLUTION/ESTIMATE" ${OUT_FILE}.snx | cut -d : -f 1`
# extract the solution estimate
head -$lineNumber ${OUT_FILE}.snx > ${OUT_FILE}.fsnx;
""" % (org, gpsweek, gpsweekday, site_string)
run_file.write(contents)
# all done
run_file.close()
# add executable permissions
chmod_exec(run_file_path)
def start(self, dirname, year, doy, dry_run=False):
monitor_open = False
try:
# copy the folder created by GamitSession in the solution_pwd to the remote_pwd (pwd)
try:
if not os.path.exists(os.path.dirname(self.pwd)):
os.makedirs(os.path.dirname(self.pwd))
except OSError:
# racing condition having several processes trying to create the same folder
# if OSError occurs, ignore and continue
pass
# if the local folder exists (due to previous incomplete processing, erase it).
if os.path.exists(self.pwd):
shutil.rmtree(self.pwd)
# ready to copy the shared solution_dir to pwd
shutil.copytree(self.solution_pwd, self.pwd, symlinks=True)
with file_open(os.path.join(self.pwd, 'monitor.log'), 'a') as monitor:
monitor_open = True
def log(s):
monitor.write(now_str() + ' -> ' + s + '\n')
log('%s %i %i executing on %s' % (dirname, year, doy, platform.node()))
log('fetching orbits')
try:
Sp3 = pySp3.GetSp3Orbits(self.orbits['sp3_path'], self.date, self.orbits['sp3types'],
self.pwd_igs, True) # type: pySp3.GetSp3Orbits
except pySp3.pySp3Exception:
log('could not find principal orbits, fetching alternative')
# try alternative orbits
if self.options['sp3altrn']:
Sp3 = pySp3.GetSp3Orbits(self.orbits['sp3_path'], self.date, self.orbits['sp3altrn'],
self.pwd_igs, True) # type: pySp3.GetSp3Orbits
else:
raise
if Sp3.type != 'igs':
# rename file
shutil.copyfile(Sp3.file_path, Sp3.file_path.replace(Sp3.type, 'igs'))
log('fetching broadcast orbits')
pyBrdc.GetBrdcOrbits(self.orbits['brdc_path'], self.date, self.pwd_brdc,
no_cleanup=True) # type: pyBrdc.GetBrdcOrbits
for rinex in self.params['rinex']:
log('fetching rinex for %s %s %s %s'
% (stationID(rinex), rinex['StationAlias'],
'{:10.6f} {:11.6f}'.format(rinex['lat'], rinex['lon']), 'tie' if rinex['is_tie'] else ''))
try:
with pyRinex.ReadRinex(rinex['NetworkCode'],
rinex['StationCode'],
rinex['source'], False) as Rinex: # type: pyRinex.ReadRinex
# WARNING! some multiday RINEX were generating conflicts because the RINEX has a name, say,
# tuc12302.10o and the program wants to rename it as tuc12030.10o but because it's a
# multiday file, during __init__ it's already split and renamed as tuc12300.10o and
# additional folders are generated with the information for each file. Therefore, find
# the rinex that corresponds to the date being processed and use that one instead of the
# original file. These files are not allowed by pyArchiveService, but the "start point" of
# the database (i.e. the files already in the folders read by pyScanArchive) has such
# problems.
# figure out if this station has been affected by an earthquake
# if so, window the data
if rinex['jump'] is not None:
monitor.write(
' -> RINEX file has been windowed: ETM detected jump on ' +
rinex['jump'].datetime().strftime('%Y-%m-%d %H:%M:%S') + '\n')
if Rinex.multiday:
# find the rinex that corresponds to the session being processed
for Rnx in Rinex.multiday_rnx_list:
if Rnx.date == self.date:
Rnx.rename(rinex['destiny'])
if rinex['jump'] is not None:
self.window_rinex(Rnx, rinex['jump'])
# before creating local copy, decimate file
Rnx.decimate(30)
Rnx.purge_comments()
Rnx.compress_local_copyto(self.pwd_rinex)
break
else:
Rinex.rename(rinex['destiny'])
if rinex['jump'] is not None:
self.window_rinex(Rinex, rinex['jump'])
# before creating local copy, decimate file
Rinex.decimate(30)
Rinex.purge_comments()
Rinex.compress_local_copyto(self.pwd_rinex)
except (OSError, IOError):
log('An error occurred while trying to copy ' +
rinex['source'] + ' to ' + rinex['destiny'] + ': File skipped.')
except (pyRinex.pyRinexException, Exception) as e:
log('An error occurred while trying to copy ' +
rinex['source'] + ': ' + str(e))
log('executing GAMIT')
# create the run script
self.create_replace_links()
self.create_run_script()
self.create_finish_script()
# run the script to replace the links of the tables directory
self.p = subprocess.Popen('find ./tables ! -name "otl.grid" -type l -exec ./replace_links.sh {} +',
shell=True, stdout=subprocess.PIPE,stderr=subprocess.PIPE, cwd=self.pwd)
_, _ = self.p.communicate()
# now execute the run script
if not dry_run:
self.p = subprocess.Popen('./run.sh', shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
cwd=self.pwd)
self.stdout, self.stderr = self.p.communicate()
self.p = subprocess.Popen('./finish.sh', shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
cwd=self.pwd)
self.stdout, self.stderr = self.p.communicate()
# check for any fatals
self.p = subprocess.Popen('grep -q \'FATAL\' monitor.log', shell=True, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, cwd=self.pwd)
_, _ = self.p.communicate()
self.success = (self.p.returncode != 0)
# output statistics to the parent to display
result = self.parse_monitor(self.success)
file_append(os.path.join(self.pwd, 'monitor.log'),
now_str() + ' -> return to Parallel.GAMIT\n')
# no matter the result of the processing, move folder to final destination
if not dry_run:
self.finish()
return result
except:
msg = traceback.format_exc() + '\nProcessing %s date %s on node %s' \
% (self.params['NetName'], self.date.yyyyddd(), platform.node())
# DDG: do not attempt to write to monitor.log or do any file operations (maybe permission problem)
# problem might occur during copytree or rmtree or some other operation before opening monitor.log
if monitor_open:
file_append(os.path.join(self.pwd, 'monitor.log'),
now_str() +
' -> ERROR in pyGamitTask.start()\n%s' % msg)
# the solution folder exists because it was created by GamitSession to start the processing.
# erase it to upload the result
if os.path.exists(self.solution_pwd):
shutil.rmtree(self.solution_pwd)
# execute final error step: copy to self.solution_pwd
shutil.copytree(self.pwd, self.solution_pwd, symlinks=True)
# remove the remote pwd
shutil.rmtree(self.pwd)
# output statistics to the parent to display
result = self.parse_monitor(False)
else:
result = {'session' : '%s %s' % (self.date.yyyyddd(), self.params['DirName']),
'Project' : self.params['NetName'],
'subnet' : self.params['subnet'],
'Year' : self.date.year,
'DOY' : self.date.doy,
'FYear' : self.date.fyear,
'wl' : 0,
'nl' : 0,
'nrms' : 0,
'relaxed_constrains' : '',
'max_overconstrained' : '',
'node' : platform.node(),
'execution_time' : 0,
'execution_date' : 0,
'missing' : '',
'success' : False,
'fatals' : []
}
result['error'] = msg
# return useful information to the main node
return result
def create_run_script(self):
year = self.date.yyyy()
doy = self.date.ddd()
# extract the gps week and convert to string
gpsWeek_str = str(self.date.gpsWeek)
# normalize gps week string
if self.date.gpsWeek < 1000: gpsWeek_str = '0'+gpsWeek_str
# set the path and name for the run script
run_file_path = os.path.join(self.pwd,'run.sh')
try:
run_file = file_open(run_file_path, 'w')
except (OSError, IOError):
raise Exception('could not open file '+run_file_path)
contents = """#!/bin/bash
# just in case, create a temporary dir for fortran
if [ ! -d ./tmp ]; then
mkdir ./tmp
fi
export TMPDIR=`pwd`/tmp
export INSTITUTE=%s
# set max depth for recursion
MAX_LEVEL=3;
# parse input
level=$1; [ $# -eq 0 ] && level=1;
# check that level less than max depth
if [[ $level -gt $MAX_LEVEL ]];then
# if so then exit
echo "run.sh (`date +"%%Y-%%m-%%d %%T"`): MAX ITERATION DEPTH REACHED ... MUST EXIT" >> monitor.log
exit 0;
fi
echo "run.sh (`date +"%%Y-%%m-%%d %%T"`): Iteration depth: $level" >> monitor.log
# set the params
EXPT=%s;
YEAR=%s;
DOY=%s;
MIN_SPAN=%s;
EOP=%s;
NOFPT=%s;
# set the name of the outfile
OUT_FILE=%s%s%s.out
# execution flag for sh_gamit
EXE=1;
COUNTER=0;
while [ $EXE -eq 1 ]; do
if [ $COUNTER -gt 9 ]; then
echo "run.sh (`date +"%%Y-%%m-%%d %%T"`): Maximum number of retries (10) reached. Abnormal exit in run.sh. Check processing log." >> monitor.log
exit 1
fi
# set exe to 0 so that we exit exe loop if no problems found
EXE=0;
# save a copy of the lfile. before running sh_gamit
iter_ext=`printf "l%%02d_i%%02d" $level $COUNTER`
cp ./tables/lfile. ./tables/lfile.${iter_ext}
# do the damn thing
if [ "$NOFTP" = "no" ]; then
sh_gamit -update_l N -topt none -c -copt null -dopt c x -expt $EXPT -d $YEAR $DOY -minspan $MIN_SPAN -remakex Y -eop $EOP &> $OUT_FILE;
else
sh_gamit -update_l N -topt none -c -copt null -noftp -dopt c x -expt $EXPT -d $YEAR $DOY -minspan $MIN_SPAN -remakex Y -eop $EOP &> $OUT_FILE;
fi
""" \
% (self.gamitopts['org'], self.gamitopts['expt'], year, doy, '12', self.gamitopts['eop_type'],
self.gamitopts['noftp'], self.gamitopts['org'], gpsWeek_str, str(self.date.gpsWeekDay))
# if we're in debug mode do not pipe output to file
# if not session.options['debug']: contents += """ &> $OUT_FILE; """;
contents += """
grep -q "Geodetic height unreasonable" $OUT_FILE;
if [ $? -eq 0 ]; then
sstn=`grep "MODEL/open: Site" $OUT_FILE | tail -1 | cut -d ":" -f 5 | cut -d " " -f 3 |tr '[:upper:]' '[:lower:]'`;
echo "run.sh (`date +"%Y-%m-%d %T"`): deleting station ${sstn}: unreasonable geodetic height" >> monitor.log
rm rinex/${sstn}* ;
rm $DOY/${sstn}* ;
grep "MODEL/open: Site" $OUT_FILE | tail -1
echo "run.sh (`date +"%Y-%m-%d %T"`): will try sh_gamit again ..." >> monitor.log
EXE=1;
fi
grep "FATAL.*MAKEX/lib/rstnfo: No match for" $OUT_FILE
if [ $? -eq 0 ];then
sstn=`grep "FATAL.*MAKEX/lib/rstnfo: No match for" $OUT_FILE | tail -1 | cut -d ":" -f5 | awk '{print $4}' | tr '[:upper:]' '[:lower:]'`
echo "run.sh (`date +"%Y-%m-%d %T"`): deleting station ${sstn}: no station info" >> monitor.log
rm rinex/${sstn}* ;
rm $DOY/${sstn}* ;
echo "run.sh (`date +"%Y-%m-%d %T"`): will try sh_gamit again ..." >> monitor.log
EXE=1;
fi
grep -q "Error extracting velocities for" $OUT_FILE;
if [ $? -eq 0 ]; then
sstn=`grep "Error extracting velocities for" $OUT_FILE | head -1 | cut -d ":" -f 5 | cut -d " " -f 6 |tr '[:upper:]' '[:lower:]'`;
echo "run.sh (`date +"%Y-%m-%d %T"`): deleting station ${sstn}: Error extracting velocities for" >> monitor.log
rm rinex/${sstn}* ;
rm $DOY/${sstn}* ;
grep "Error extracting velocities for" $OUT_FILE | tail -1
echo "run.sh (`date +"%Y-%m-%d %T"`): will try sh_gamit again ..." >> monitor.log
EXE=1;
fi
grep -q "Bad WAVELENGTH FACT" $OUT_FILE;
if [ $? -eq 0 ]; then
sstn=`grep "Bad WAVELENGTH FACT" $OUT_FILE | tail -1 | cut -d ":" -f 5 | cut -d " " -f 6 | cut -c 3-6`
echo "run.sh (`date +"%Y-%m-%d %T"`): deleting station ${sstn}: Bad WAVELENGTH FACT in rinex header" >> monitor.log
rm rinex/${sstn}*;
rm $DOY/${sstn}* ;
echo "run.sh (`date +"%Y-%m-%d %T"`): will try sh_gamit again ..." >> monitor.log
EXE=1;
fi
grep -q "Error decoding swver" $OUT_FILE;
if [ $? -eq 0 ]; then
grep "Error decoding swver" $OUT_FILE;
sstn=`grep "Error decoding swver" $OUT_FILE | tail -1 | awk '{print $8}' | tr '[:upper:]' '[:lower:]'`
echo "run.sh (`date +"%Y-%m-%d %T"`): deleting station ${sstn}: Error decoding swver" >> monitor.log
rm rinex/${sstn}*;
rm $DOY/${sstn}* ;
echo "run.sh (`date +"%Y-%m-%d %T"`): will try sh_gamit again ..." >> monitor.log
EXE=1;
fi
grep -q "FATAL.*MAKEX/lib/hisub: Antenna code.*not in hi.dat" $OUT_FILE;
if [ $? -eq 0 ]; then
grep "FATAL.*MAKEX/lib/hisub: Antenna code.*not in hi.dat" $OUT_FILE;
sstn=`grep "FATAL.*MAKEX/lib/hisub: Antenna code.*not in hi.dat" $OUT_FILE | tail -1 | awk '{print $9}' | cut -c2-5 | tr '[:upper:]' '[:lower:]'`
echo "run.sh (`date +"%Y-%m-%d %T"`): deleting station ${sstn}: Antenna code not in hi.dat" >> monitor.log
rm rinex/${sstn}*;
rm $DOY/${sstn}* ;
echo "run.sh (`date +"%Y-%m-%d %T"`): will try sh_gamit again ..." >> monitor.log
EXE=1;
fi
grep -q "FATAL.*FIXDRV/dcheck: Only one or no existing X-files" $OUT_FILE;
if [ $? -eq 0 ]; then
grep "FATAL.*FIXDRV/dcheck: Only one or no existing X-files" $OUT_FILE;
echo "run.sh (`date +"%Y-%m-%d %T"`): FIXDRV/dcheck: Only one or no existing X-files" >> monitor.log
echo "run.sh (`date +"%Y-%m-%d %T"`): will try sh_gamit again ..." >> monitor.log
EXE=1;
fi
grep -q "FATAL.*MAKEXP/makexp: No RINEX or X-files found" $OUT_FILE;
if [ $? -eq 0 ]; then
grep "FATAL.*MAKEXP/makexp: No RINEX or X-files found" $OUT_FILE;
echo "run.sh (`date +"%Y-%m-%d %T"`): MAKEXP/makexp: No RINEX or X-files found" >> monitor.log
echo "run.sh (`date +"%Y-%m-%d %T"`): will try sh_gamit again ..." >> monitor.log
EXE=1;
fi
grep -q "FATAL.*MAKEX/get_rxfiles: Cannot find selected RINEX file" $OUT_FILE;
if [ $? -eq 0 ]; then
grep "FATAL.*MAKEX/get_rxfiles: Cannot find selected RINEX file" $OUT_FILE;
echo "run.sh (`date +"%Y-%m-%d %T"`): MAKEX/get_rxfiles: Cannot find selected RINEX file" >> monitor.log
echo "run.sh (`date +"%Y-%m-%d %T"`): will try sh_gamit again ..." >> monitor.log
EXE=1;
fi
grep -q "FATAL.*MAKEX/openf: Error opening file:.*" $OUT_FILE;
if [ $? -eq 0 ]; then
grep "FATAL.*MAKEX/openf: Error opening file:.*" $OUT_FILE;
echo "run.sh (`date +"%Y-%m-%d %T"`): MAKEX/openf: Error opening file" >> monitor.log
echo "run.sh (`date +"%Y-%m-%d %T"`): will try sh_gamit again ..." >> monitor.log
EXE=1;
fi
grep -q "SOLVE/get_widelane: Error reading first record" $OUT_FILE;
if [ $? -eq 0 ]; then
grep "SOLVE/get_widelane: Error reading first record" $OUT_FILE;
echo "run.sh (`date +"%Y-%m-%d %T"`): SOLVE/get_widelane: Error reading first record of temp file" >> monitor.log
echo "run.sh (`date +"%Y-%m-%d %T"`): will try sh_gamit again ..." >> monitor.log
EXE=1;
fi
grep -q "Failure in sh_preproc. STATUS 1 -- sh_gamit terminated" $OUT_FILE;
if [ $? -eq 0 ]; then
grep "Failure in sh_preproc. STATUS 1 -- sh_gamit terminated" $OUT_FILE;
echo "run.sh (`date +"%Y-%m-%d %T"`): Failure in sh_preproc. STATUS 1 -- sh_gamit terminated" >> monitor.log
echo "run.sh (`date +"%Y-%m-%d %T"`): will try sh_gamit again ..." >> monitor.log
EXE=1;
fi
# problems related to ill conditioned bias matrix
grep -q "FATAL.*SOLVE/lcloos: Inversion error in" $OUT_FILE;
if [ $? -eq 0 ]; then
# remove the FATAL from the message to avoid confusing finish.sh that there was an error during execution
err="SOLVE/lcloos: Inversion error in LCNORM(2)"
# determine which autocln.sum exists and has information in it
if [ -s $DOY/autcln.post.sum ]; then
autocln=autcln.post.sum
# error occurred after the prefit, read the autcln file and remove the station with low obs
echo "run.sh (`date +"%Y-%m-%d %T"`): $err (after prefit) Will remove the station with the lowest obs count in $autocln" >> monitor.log
sstn=`sed -n -e '/Number of data by site/,/^$/ p' $DOY/$autocln | tail -n +3 | sed '$d' | awk '{print $3, $4}' | awk -v min=999999 '{if($2<min){min=$2; stn=$1}}END{print stn}' | tr '[:upper:]' '[:lower:]'`
nobs=`sed -n -e '/Number of data by site/,/^$/ p' $DOY/$autocln | tail -n +3 | sed '$d' | awk '{print $3, $4}' | awk -v min=999999 '{if($2<min){min=$2; stn=$1}}END{print min}'`
echo "run.sh (`date +"%Y-%m-%d %T"`): deleting station ${sstn} -> observation count: $nobs" >> monitor.log
rm rinex/${sstn}*;
rm $DOY/${sstn}* ;
echo "run.sh (`date +"%Y-%m-%d %T"`): will try sh_gamit again ..." >> monitor.log
EXE=1;
else
# the error occurred during the prefit, autocln may or may not have gotten the problem. Use the observation count in the $OUTFILE
echo "run.sh (`date +"%Y-%m-%d %T"`): $err. (during prefit) Will analyze the MAKEX output and remove the file with more rejected observations" >> monitor.log
max_rejected=`grep "observations rejected" $OUT_FILE | awk -F ':' '{print $5}' | awk '{print $6}' | awk -v max=0 '{if($1>max){max=$1}}END{print max}'`
sstn=(`sed -n -e '/'$max_rejected' observations rejected/,/End processing/ p' $OUT_FILE | grep 'End' | awk -F ':' '{print $6}' | awk '{print $1'} | uniq | tr '[:upper:]' '[:lower:]'`)
if [ -z "$sstn" ]; then
echo "run.sh (`date +"%Y-%m-%d %T"`): could not determine the station with low observation count. Check $OUT_FILE" >> monitor.log
else
for stn in ${sstn[*]}
do
echo "run.sh (`date +"%Y-%m-%d %T"`): deleting station ${stn} -> rejected observation count: $max_rejected" >> monitor.log
rm rinex/${stn}*;
rm $DOY/${stn}* ;
done
echo "run.sh (`date +"%Y-%m-%d %T"`): will try sh_gamit again ..." >> monitor.log
EXE=1;
fi
fi
# different search methods, deprecated
#sstn=(`sed -n -e '/ .... valid observations/,/stop in MODEL/ p' $OUT_FILE | grep 'Site' | awk -F ':' '{print $5}' | awk '{print $2'} | uniq | tr '[:upper:]' '[:lower:]'`)
fi
# this case after SOLVE/lcloos because it also triggers GAMIT sh_chksolve
grep -q "FATAL GAMIT sh_chksolve: Solve failed to complete normally" $OUT_FILE;
if [ $? -eq 0 ] && [ $EXE -eq 0 ]; then
echo "run.sh (`date +"%Y-%m-%d %T"`): GAMIT sh_chksolve: Solve failed to complete normally" >> monitor.log
echo "run.sh (`date +"%Y-%m-%d %T"`): will try sh_gamit again ..." >> monitor.log
EXE=1;
fi
# grep over constrained sites
grep -q "over constrained" ./$DOY/sh_gamit_${DOY}.summary;
if [ $? -eq 0 ]; then
# get the number of lines
lines=`cat ./$DOY/sh_gamit_${DOY}.summary | sed -n 's/WARNING: \([0-9]*\) SITES.*$/\\1/p'`
grep -A $lines "over constrained" ./$DOY/sh_gamit_${DOY}.summary >> monitor.log
# DDG: new behavior -> remove the station with the largest over constrained coordinate
# grep the sites and get the unique list separeted by | (to do regex grep)
# stns=`grep "GCR APTOL" monitor.log | awk '{print $4"_GPS"}' | uniq | tr '<line break>' '|'`
# copy the sittbl. (just in case)
# cp tables/sittbl. tables/sittbl.${iter_ext}
# remove those from the sittbl list: this will relax station to 100 m
# grep -v -E "${stns:0:-1}" tables/sittbl.${iter_ext} > tables/sittbl.
stns=`grep "GCR APTOL" ./$DOY/sh_gamit_${DOY}.summary | awk '{print sqrt($(NF) * $(NF)), $4}' | sort -r | head -n1 | awk '{print $2}' | tr '[:upper:]' '[:lower:]'`
echo "run.sh (`date +"%Y-%m-%d %T"`): deleting over constrained station ${stns}" >> monitor.log
rm rinex/${stns}*;
rm $DOY/${stns}* ;
# echo "run.sh (`date +"%Y-%m-%d %T"`): relaxing over constrained stations ${stns:0:-1}" >> monitor.log
echo "run.sh (`date +"%Y-%m-%d %T"`): replacing lfile. from this run with lfile.${iter_ext}" >> monitor.log
rm ./tables/lfile.
cp ./tables/lfile.${iter_ext} ./tables/lfile.
echo "run.sh (`date +"%Y-%m-%d %T"`): will try sh_gamit again ..." >> monitor.log
EXE=1;
fi
if [ $EXE -eq 1 ]; then
# if it will retry, save the previous output using extension .l00_i00, .l00_i01, ... etc
# where lxx is the level of iteration and iyy is the interation in this level
mv $OUT_FILE $OUT_FILE.${iter_ext}
COUNTER=$((COUNTER+1));
fi
# grep updated coordinates
grep Updated ./tables/lfile.;
if [ $? -eq 0 ]; then
grep Updated ./tables/lfile. >> monitor.log
fi
done
# clean up
rm -rf ionex met teqc*
# blab about failures
grep "FATAL" *.out >> monitor.log
"""
run_file.write(contents)
contents = \
"""
# remove extraneous solution files
# rm ./$DOY/l*[ab].*;
# make sure to rename the gfilea to the correct gfile[0-9].doy
[ -e ./igs/gfilea.* ] && mv -f ./igs/gfilea* ./*/gfile[0-9]*
# see if any of the coordinates were updated, exit if not
# DDG: Actually, sometimes the final updated coordinate only differs by < .3 m when solve is invoked more than
# once from within sh_gamit. Therefore, also check that the updated coordinate is > .3 m from the original APR
# this happens because the first Updated coordinate (lxxxxa.ddd) triggers an iteration in solve (lxxxxb.ddd)
# with a solution that is again close to the original APR. Without this check, PG iterates 3 times unnecessarily
grep Updated ./tables/lfile.;
if [ $? -ne 0 ]; then
echo "run.sh (`date +"%Y-%m-%d %T"`): Normal exit from run.sh" >> monitor.log
# uncompress everything. Will be compressed later on
gunzip ./*/*;
exit
else
updated=(`grep Updated ./tables/lfile. | awk '{print $1}'`)
RERUN=0
for stn in ${updated[*]}
do
coords=`grep $stn ./tables/lfile. | awk '{print $2,$3,$4}'`
# use the copy of the lfile to grep the APR coordinates
aprs=`grep $stn ./tables/lfile.${iter_ext} | awk '{print $2,$3,$4}'`
# get the distance between Updated and APR
dist=`echo $coords $aprs | awk '{print sqrt(($1 - $4)^2 + ($2 - $5)^2 + ($3 - $6)^2)}'`
if (( $(echo "$dist > 0.3" | bc -l) )); then
RERUN=1;
fi
done
# if RERUN = 0, Updated coordinate was < 0.3 m
if [ $RERUN -eq 0 ]; then
echo "run.sh (`date +"%Y-%m-%d %T"`): Updated coordinate detected but final solution within 0.3 m of APR" >> monitor.log
echo "run.sh (`date +"%Y-%m-%d %T"`): Normal exit from run.sh" >> monitor.log
# uncompress everything. Will be compressed later on
gunzip ./*/*;
exit
fi
fi
# iteration detected!
echo "run.sh (`date +"%Y-%m-%d %T"`): Updated coordinate detected in lfile. Iterating..." >> monitor.log
# save this level's out file for debugging
mv $OUT_FILE $OUT_FILE.${iter_ext}
# apr file for updated coordinates
aprfile=${EXPT}.temp
# recreate the apr file with updated coordinates minus the comments
sed -e 's/Updated from l.....\.[0-9][0-9][0-9]//g' ./tables/lfile. > ./tables/${aprfile};
# the copy of the lfile was saved BEFORE running GAMIT. Replace with the updated version
cp ./tables/${aprfile} ./tables/lfile.
# copy over an updated gfile if it exists
# cp ./*/gfile* ./tables/
# update level
level=$((level+1));
# remove the 'old' solution
[ $level -le $MAX_LEVEL ] && rm -rf ./$DOY;
# decompress the remaining solution files
gunzip ./*/*;
# do another iteration
./run.sh $level;
"""
run_file.write(contents)
run_file.close()
chmod_exec(run_file_path)
def create_combination_script(self, date, org):
# extract the gps week and convert to string
gpsWeek_str = date.wwww()
# set the path and name for the run script
run_file_path = os.path.join(self.pwd_comb, 'globk.sh')
try:
run_file = file_open(run_file_path, 'w')
except:
raise GlobkException('could not open file '+run_file_path)
with run_file:
contents = \
"""#!/bin/bash
export INSTITUTE=%s
# data product file names
OUT_FILE=%s%s%s;
# mk solutions directory for prt files etc
[ ! -d tables ] && mkdir tables
cd tables
../link_tables.sh
# create global directory listing for globk
for file in $(find .. -name "*.glx" -print);do echo $file;done | grep "\/n0\/" > globk.gdl
for file in $(find .. -name "*.glx" -print);do echo $file;done | grep -v "\/n0\/" >> globk.gdl
# create the globk cmd file
echo " app_ptid all" > globk.cmd
echo " prt_opt GDLF MIDP CMDS" >> globk.cmd
echo " out_glb ../file.GLX" >> globk.cmd
echo " in_pmu pmu.usno" >> globk.cmd
echo " descript Daily combination of global and regional solutions" >> globk.cmd
""" % (org, org, gpsWeek_str, str(date.gpsWeekDay))
if self.net_type == 'global':
contents += \
"""
echo "# activated for global network merge" >> globk.cmd
echo " apr_wob 10 10 10 10 " >> globk.cmd
echo " apr_ut1 10 10 " >> globk.cmd
echo " apr_svs all 0.05 0.05 0.05 0.005 0.005 0.005 0.01 0.01 0.00 0.01 FR" >> globk.cmd
"""
contents += \
"""
echo " max_chii 1. 0.6" >> globk.cmd
echo " apr_site all 1 1 1 0 0 0" >> globk.cmd
echo " apr_atm all 1 1 1" >> globk.cmd
# create the sinex header file
echo "+FILE/REFERENCE " > head.snx
echo " DESCRIPTION Instituto Geografico Nacional " >> head.snx
echo " OUTPUT Solucion GPS combinada " >> head.snx
echo " CONTACT [email protected] " >> head.snx
echo " SOFTWARE glbtosnx Version " >> head.snx
echo " HARDWARE . " >> head.snx
echo " INPUT Archivos binarios Globk " >> head.snx
echo "-FILE/REFERENCE " >> head.snx
# run globk
globk 0 ../file.prt ../globk.log globk.gdl globk.cmd 2>&1 > ../globk.out
# convert the GLX file into sinex
glbtosnx . ./head.snx ../file.GLX ../${OUT_FILE}.snx 2>&1 > ../glbtosnx.out
# restore original directory
cd ..;
# figure out where the parameters start in the prt file
LINE=`grep -n "PARAMETER ESTIMATES" file.prt | cut -d ":" -f1`
# reduce line by one to make a little cleaner
let LINE--;
# print prt header
sed -n 1,${LINE}p file.prt > ${OUT_FILE}.out
# append the log file
cat globk.log >> ${OUT_FILE}.out
# create the fsnx file which contains only the solution estimate
lineNumber=`grep --binary-file=text -m 1 -n "\-SOLUTION/ESTIMATE" ${OUT_FILE}.snx | cut -d : -f 1`
# extract the solution estimate
head -$lineNumber ${OUT_FILE}.snx > ${OUT_FILE}.fsnx;
# move the H file to a meaningful name
mv file.GLX ${OUT_FILE}.GLX
# clear out log files
rm -rf tables
rm -f file*
rm -f globk*
rm -f glb*
rm -f *.sh
# compress sinex file
# gzip --force *.snx
gzip --force *.fsnx
gzip --force *.out
gzip --force *.glx
gzip --force *.GLX
"""
run_file.write(contents)
# all done
# add executable permissions
chmod_exec(run_file_path)
def parse(self):
foundSolutionEstimate = False
inSolutionEstimateSection = False
inSolutionMatrixEstimateSection = False
isHeaderLine = False
foundSiteId = False
inSiteIdSection = False
dictID = {}
stn_ID = {}
# if there's a file to parse
if self.snxFilePath != None:
# flag to rezip at end
wasZipped = False
wasCompressed = False
# check for gzip
if self.snxFilePath[-2:] == "gz":
#file_ops.gunzip(self.snxFilePath)
self.snxFilePath = self.snxFilePath[0:-3]
wasZipped = True
# check for unix compression
elif self.snxFilePath[-1:] == "Z":
#file_ops.uncompress(self.snxFilePath)
self.snxFilePath = self.snxFilePath[0:-2]
wasCompressed = True
# open the file
try:
snxFileHandle = file_open(self.snxFilePath)
except:
print("snxFileParser ERROR: Could not open file " + self.snxFilePath + " !!!")
raise
with snxFileHandle:
#make pattern to match to snx organization ...
self.snxFileName = os.path.basename(self.snxFilePath)
# @todo optimize by pre-compiling regexes
orgPattern = re.compile('^([a-zA-Z]+).*\.f?snx$')
orgMatch = orgPattern.findall(self.snxFileName)
self.orgName = orgMatch[0].upper()
# make pattern to look for SiteId start tag
siteIdStartPattern = re.compile('^\+SITE\/ID$')
# make pattern to look for end of siteId section
siteIdEndPattern = re.compile('^\-SITE\/ID$')
# make pattern to parse the siteId lines
# Example:
#
# TROM A 82397M001 P , USA 18 56 18.0 69 39 45.9 135.4
#
siteIdPattern = re.compile('^\s+(\w+)\s+\w\s+(\w+).*$')
# variance factor patther
# Example:
#
# VARIANCE FACTOR 0.048618461936712
#
#
varianceFactorPattern = re.compile('^ VARIANCE FACTOR\s+([\d+]?\.\d+)$')
observationsPattern = re.compile('^ NUMBER OF OBSERVATIONS\s+(\d+)$')
unknownsPattern = re.compile('^ NUMBER OF UNKNOWNS\s+(\d+)$')
# Make pattern to look for solution estimate start tag
startSolutionEstimatePattern = re.compile('^\+SOLUTION\/ESTIMATE.*')
# make pattern to look for solution estimate end tag
endSolutionEstimatePattern = re.compile('^\-SOLUTION\/ESTIMATE.*')
# make pattern to look for the L COVA start tag (+SOLUTION/MATRIX_ESTIMATE L COVA)
startSolutionMatrixEstimate = re.compile('^\+SOLUTION\/MATRIX_ESTIMATE.*')
# make pattern to look for the L COVA end tag (-SOLUTION/MATRIX_ESTIMATE L COVA)
endSolutionMatrixEstimate = re.compile('^\-SOLUTION\/MATRIX_ESTIMATE.*')
# make pattern to look for station coordinates
# Example:
#
# 1 STAX ALGO A ---- 05:180:43200 m 2 .91812936331043008E+6 .2511266E-2
#
stationCoordinatePattern = re.compile('^\s+(\d+)+\s+STA(\w)\s+(\w+)\s+(\w).*\d+\s+(-?[\d+]?\.\d+[Ee][+-]?\d+)\s+(-?[\d+]?\.\d+[Ee][+-]?\d+)$')
# make pattern to look for station velocities
# Example:
#
# 916 VELX YAKA A 1 00:001:00000 m/y 2 -.219615010076079E-01 0.13728E-03
#
stationVelocityPattern = re.compile('^\s+\d+\s+VEL(\w)\s+(\w+)\s+\w\s+....\s+(\d\d:\d\d\d).*\d+\s+(-?[\d+]?\.\d+[Ee][+-]?\d+)\s+(-?[\d+]?\.\d+[Ee][+-]?\d+)$')
for line in snxFileHandle:
if siteIdStartPattern.findall(line):
inSiteIdSection = True
continue
elif siteIdEndPattern.findall(line):
inSiteIdSection = False
continue
# check for solution estimate section
elif startSolutionEstimatePattern.findall(line):
inSolutionEstimateSection = True
continue
elif endSolutionEstimatePattern.findall(line):
inSolutionEstimateSection = False
continue
elif startSolutionMatrixEstimate.findall(line):
inSolutionMatrixEstimateSection = True
continue
elif endSolutionMatrixEstimate.findall(line):
inSolutionMatrixEstimateSection = False
break
m = varianceFactorPattern.findall(line)
if m:
self.varianceFactor = float(m[0])
m = unknownsPattern.findall(line)
if m:
self.unknowns = float(m[0])
m = observationsPattern.findall(line)
if m:
self.observations = float(m[0])
if inSiteIdSection:
# parse the siteID line
siteIdMatch = siteIdPattern.findall(line)
# blab about it
#print siteIdMatch
# if the line does not contain a match then move along
if not siteIdMatch:
continue
# extract the parsed info
(stationName, domesNumber) = siteIdMatch[0]
# make sure the name is upper case
stationName = stationName.upper()
# initialize station data if not seen this station before
if not stationName in self.stationDict:
self.stationDict[stationName] = StationData()
self.stationDict[stationName].domesNumber = domesNumber
#print "set domes number "+ domesNumber +" for station "+stationName
# if in the solution estimate section
if inSolutionEstimateSection:
# check for station coordinate match
stationCoordinateMatch = stationCoordinatePattern.findall(line)
#print line
#print stationCoordinateMatch
# if match then store result
if stationCoordinateMatch:
(ID,
coordID,
stationName,
pointCode,
coord,
sigCoord
) = stationCoordinateMatch[0]
if pointCode != 'A':
os.sys.stderr.write('ignoring solution/estimate STA'+coordID \
+' for station: '+stationName \
+', point code = '+pointCode \
+', file = '+ self.snxFileName \
+'\n' \
)
continue
# make sure station name is upper case
stationName = stationName.upper()
# save the correspondance between the ID and the coordID and stationName
dictID[ID] = coordID
stn_ID[ID] = stationName
if not stationName in self.stationDict:
self.stationDict[stationName] = StationData()
if coordID == 'X':
self.stationDict[stationName].X = float(coord)
self.stationDict[stationName].sigX = float(sigCoord)
elif coordID == 'Y':
self.stationDict[stationName].Y = float(coord)
self.stationDict[stationName].sigY = float(sigCoord)
else:
self.stationDict[stationName].Z = float(coord)
self.stationDict[stationName].sigZ = float(sigCoord)
# check for station velocity match
stationVelocityMatch = stationVelocityPattern.findall(line)
if stationVelocityMatch:
( \
coordID, \
stationName, \
refEpoch, \
vel, \
sigVel \
) = stationVelocityMatch[0]
stationName = stationName.upper()
# parse refEpoch String
(year,doy) = refEpoch.split(':')
# convert from string
doy = float(doy)
# normalize the year and convert to float
year = float(get_norm_year_str(year))
#compute fractional year to match matlab round off
fractionalYear = year+((doy-1)/366.0)+ 0.001413
# init if not already in dict
if not stationName in self.stationDict:
self.stationDict[stationName] = StationData()
# set the reference epoch for the velocity
self.stationDict[stationName].refEpoch = fractionalYear
if coordID == 'X':
self.stationDict[stationName].velX = float(vel)
self.stationDict[stationName].sigVelX = float(sigVel)
elif coordID == 'Y':
self.stationDict[stationName].velY = float(vel)
self.stationDict[stationName].sigVelY = float(sigVel)
else:
self.stationDict[stationName].velZ = float(vel)
self.stationDict[stationName].sigVelZ = float(sigVel)
# regzip the file is was .gz
#if wasZipped:
#file_ops.gzip(self.snxFilePath)
# recompress the file if was .Z
#if wasCompressed:
#file_ops.compress(self.snxFilePath)
if inSolutionMatrixEstimateSection:
matrixLine = line.split()
ID1 = matrixLine[0]
# check that the key is actually a station variance-covariance item
if ID1 in list(stn_ID.keys()):
for i, ID2 in enumerate(range(len(matrixLine) - 2)):
ID2 = str(int(matrixLine[1]) + i)
if stn_ID[ID1] == stn_ID[ID2] and dictID[ID1] != dictID[ID2]:
# we already have the variance, we want the covariance
if (dictID[ID1] == 'X' and dictID[ID2] == 'Y') or \
(dictID[ID1] == 'Y' and dictID[ID2] == 'X'):
self.stationDict[stn_ID[ID1]].sigXY = float(matrixLine[i+2])
elif (dictID[ID1] == 'X' and dictID[ID2] == 'Z') or \
(dictID[ID1] == 'Z' and dictID[ID2] == 'X'):
self.stationDict[stn_ID[ID1]].sigXZ = float(matrixLine[i + 2])
elif (dictID[ID1] == 'Y' and dictID[ID2] == 'Z') or \
(dictID[ID1] == 'Z' and dictID[ID2] == 'Y'):
self.stationDict[stn_ID[ID1]].sigYZ = float(matrixLine[i + 2])
return self
def __init__(self, configfile):
self.options = {
'path': None,
'repository': None,
'parallel': False,
'cups': None,
'node_list': None,
'ip_address': None,
'brdc': None,
'sp3_type_1': None,
'sp3_type_2': None,
'sp3_type_3': None,
'sp3_altr_1': None,
'sp3_altr_2': None,
'sp3_altr_3': None,
'grdtab': None,
'otlgrid': None,
'otlmodel': 'FES2014b',
'ppp_path': None,
'institution': None,
'info': None,
'sp3': None,
'frames': None,
'atx': None,
'height_codes': None,
'ppp_exe': None,
'ppp_remote_local': ()
}
config = configparser.ConfigParser()
with file_open(configfile) as f:
config.read_file(f)
# get the archive config
self.options.update(dict(config.items('archive')))
# get the otl config
self.options.update(dict(config.items('otl')))
# get the ppp config
for iconfig, val in dict(config.items('ppp')).items():
self.options[iconfig] = os.path.expandvars(val).replace('//', '/')
# frames and dates
frames = [item.strip() for item in self.options['frames'].split(',')]
atx = [item.strip() for item in self.options['atx'].split(',')]
self.Frames = []
for frame, atx in zip(frames, atx):
date = process_date(self.options[frame.lower()].split(','))
self.Frames += [{
'name':
frame,
'atx':
atx,
'dates': (Date(year=date[0].year,
doy=date[0].doy,
hour=0,
minute=0,
second=0),
Date(year=date[1].year,
doy=date[1].doy,
hour=23,
minute=59,
second=59))
}]
self.options['frames'] = self.Frames
self.archive_path = self.options['path']
self.sp3_path = self.options['sp3']
self.brdc_path = self.options['brdc']
self.repository = self.options['repository']
self.repository_data_in = os.path.join(self.repository, 'data_in')
self.repository_data_in_retry = os.path.join(self.repository,
'data_in_retry')
self.repository_data_reject = os.path.join(self.repository,
'data_rejected')
self.sp3types = [
self.options[k] for k in ('sp3_type_1', 'sp3_type_2', 'sp3_type_3')
if self.options[k] is not None
]
# alternative sp3 types
self.sp3altrn = [
self.options[k] for k in ('sp3_altr_1', 'sp3_altr_2', 'sp3_altr_3')
if self.options[k] is not None
]
self.run_parallel = (self.options['parallel'] == 'True')
def main():
parser = argparse.ArgumentParser(description='Program to perform weekly loosely-constrained solutions. Combination '
'is performed using GLOBK. Result is output in SINEX format.')
parser.add_argument('stnlist', type=str, nargs='+', metavar='all|net.stnm',
help="List of networks/stations to include in the solution.")
parser.add_argument('-s', '--session_config', type=str, nargs=1, metavar='session.cfg',
help="Filename with the session configuration to run Parallel.GAMIT")
parser.add_argument('-w', '--gpsweek', nargs=1,
help="GPS week to combine.")
parser.add_argument('-e', '--exclude', type=str, nargs='+', metavar='station',
help="List of stations to exclude (e.g. -e igm1 lpgs vbca)")
args = parser.parse_args()
cnn = dbConnection.Cnn("gnss_data.cfg")
# get the working dates
date_s = pyDate.Date(gpsWeek=int(args.gpsweek[0]), gpsWeekDay=0)
date_e = pyDate.Date(gpsWeek=int(args.gpsweek[0]), gpsWeekDay=6)
print(' >> Working with GPS week ' + args.gpsweek[0] + ' (%s to %s)' % (date_s.yyyyddd(), date_e.yyyyddd()))
exclude = args.exclude
if exclude is not None:
print(' >> User selected list of stations to exclude:')
Utils.print_columns(exclude)
args.stnlist += ['-' + exc for exc in exclude]
# get the station list
stnlist = Utils.process_stnlist(cnn, args.stnlist)
# check that the selected stations have all different station codes
# otherwise, exit with error
for i in range(len(stnlist) - 1):
for j in range(i + 1, len(stnlist)):
if stnlist[i]['StationCode'] == stnlist[j]['StationCode']:
print('During station selection, two identical station codes were found. '
'Please remove one and try again.')
exit()
GamitConfig = pyGamitConfig.GamitConfiguration(args.session_config[0]) # type: pyGamitConfig.GamitConfiguration
project = GamitConfig.NetworkConfig.network_id.lower()
org = GamitConfig.gamitopt['org']
print(' >> REMINDER: To automatically remove outliers during the weekly combination, '
'first run DRA.py to analyze the daily repetitivities')
soln_pwd = GamitConfig.gamitopt['solutions_dir']
# create a globk directory in production
if not os.path.exists('production/globk'):
os.makedirs('production/globk')
# check if week folder exists
globk_pwd = 'production/globk/' + args.gpsweek[0]
if os.path.exists(globk_pwd):
rmtree(globk_pwd)
# create the directory
os.makedirs(globk_pwd)
glx_list = []
# make a list of the h files that need to be combined
for day in range(0, 7):
date = pyDate.Date(gpsWeek = int(args.gpsweek[0]),
gpsWeekDay = day)
soln_dir = os.path.join(soln_pwd, "%s/%s/%s/glbf" % (date.yyyy(), date.ddd(), project))
if os.path.exists(soln_dir):
glx = glob.glob(os.path.join(soln_dir, '*.GLX.*'))
if not glx:
glx = glob.glob(os.path.join(soln_dir, '*.glx'))
glx_list.append({'file': glx[0], 'gpsweek': date.wwwwd()})
# create the earthquakes.txt file to remove outliers
with file_open(globk_pwd + '/eq_rename.txt', 'w') as fd:
rename = []
remove = []
use_site = []
fd.write('# LIST OF OUTLIERS DETECTED BY DRA\n')
for stn in stnlist:
# obtain the filtered solutions
rm = cnn.query_float('SELECT * FROM gamit_soln_excl WHERE "Project" = \'%s\' AND "NetworkCode" = \'%s\''
' AND "StationCode" = \'%s\' AND ("Year", "DOY") BETWEEN (%i, %i) AND (%i, %i) '
'ORDER BY residual' %
(project, stn['NetworkCode'], stn['StationCode'], date_s.year, date_s.doy,
date_e.year, date_e.doy), as_dict=True)
# obtain the total number of solutions
sl = cnn.query_float('SELECT * FROM gamit_soln WHERE "Project" = \'%s\' AND "NetworkCode" = \'%s\''
' AND "StationCode" = \'%s\' AND ("Year", "DOY") BETWEEN (%i, %i) AND (%i, %i) ' %
(project, stn['NetworkCode'], stn['StationCode'], date_s.year, date_s.doy,
date_e.year, date_e.doy), as_dict=True)
for i, r in enumerate(rm):
date = pyDate.Date(year=r['Year'], doy=r['DOY'])
# if the number of rejected solutions is equal to the number of total solutions,
# leave out the first one (i == 0) which is the one with the lowest residual (see ORDER BY in rm)
if len(rm) < len(sl) or (len(rm) == len(sl) and i != 0):
fd.write(' rename %s_gps %s_xcl %-20s %s %02i %02i 0 0 %s %02i %02i 24 0\n' %
(stn['StationCode'], stn['StationCode'], org + date.wwwwd() + '.GLX', date.yyyy()[2:],
date.month, date.day, date.yyyy()[2:], date.month, date.day))
# check for renames that might not agree between days
mv = cnn.query_float('SELECT * FROM gamit_subnets WHERE "Project" = \'%s\' AND ("Year", "DOY") '
'BETWEEN (%i, %i) AND (%i, %i) AND \'%s.%s\' = ANY(stations)' %
(project, date_s.year, date_s.doy, date_e.year, date_e.doy,
stn['NetworkCode'], stn['StationCode']), as_dict=True)
for m in mv:
date = pyDate.Date(year=m['Year'], doy=m['DOY'])
# check on each day to see if alias agrees with station code
for i, s in enumerate(m['stations']):
if s.split('.')[1] != m['alias'][i] and s == stationID(stn):
print(' -- %s alias for %s = %s: renaming' \
% (date.yyyyddd(), stationID(stn), m['alias'][i]))
# change the name of the station to the original name
rename.append(' rename %s_gps %s_dup %-20s %s %02i %02i 0 0 %s %02i %02i 24 0\n' %
(m['alias'][i], stn['StationCode'], org + date.wwwwd() + '.GLX', date.yyyy()[2:],
date.month, date.day, date.yyyy()[2:], date.month, date.day))
use_site.append('%s_dup' % stn['StationCode'])
elif s not in [stationID(st) for st in stnlist]:
# print ' -- Removing %s: not selected' % s
# just in case, remove any other occurrences of this station code
remove.append(' rename %s_gps %s_xcl %-20s %s %02i %02i 0 0 %s %02i %02i 24 0\n' %
(m['alias'][i], m['alias'][i], org + date.wwwwd() + '.GLX', date.yyyy()[2:],
date.month, date.day, date.yyyy()[2:], date.month, date.day))
else:
use_site.append('%s_gps' % stn['StationCode'])
fd.write('# LIST OF STATIONS TO BE REMOVED\n')
fd.write(''.join(remove))
fd.write('# LIST OF STATIONS TO BE RENAMED\n')
fd.write(''.join(rename))
print(' >> Converting to SINEX the daily solutions')
for day, glx in enumerate(glx_list):
date = pyDate.Date(gpsWeek = int(args.gpsweek[0]),
gpsWeekDay = day)
print(' -- Working on %s' % date.wwwwd())
# delete the existing GLX files
for ff in glob.glob(globk_pwd + '/*.GLX'):
os.remove(ff)
Globk(globk_pwd, org, [glx], date.wwww(), date.gpsWeekDay + 8, ' '.join(set(use_site)))
# convert the file to a valid gpsweek day
move(globk_pwd + '/' + org + date.wwww() + '%i.snx' % (date.gpsWeekDay + 8),
globk_pwd + '/' + org + date.wwww() + '%i.snx' % date.gpsWeekDay)
process_sinex(cnn, project, [date, date], globk_pwd + '/' + org + date.wwww() + '%i.snx' % date.gpsWeekDay)
# delete the existing GLX files: get ready for weekly combination
for ff in glob.glob(globk_pwd + '/*.GLX'):
os.remove(ff)
# ready to pass list to globk object
Globk(globk_pwd, org, glx_list, date_s.wwww(), 7, ' '.join(set(use_site)))
print(' >> Formatting the SINEX file')
process_sinex(cnn, project, [date_s, date_e], globk_pwd + '/' + org + date_s.wwww() + '7.snx')
def load_constrains(constrains_file):
"""
Load the frame parameters
:param constrains_file:
:return: dictionary with the parameters for the given frame
"""
params = dict()
with file_open(constrains_file) as f:
lines = f.read()
stn = re.findall(
r'^\s(\w+.\w+)\s*(-?\d*\.\d+|NaN)\s*(-?\d*\.\d+|NaN)\s*(-?\d*\.\d+|NaN)\s*(-?\d*\.\d+|NaN)'
r'\s*(-?\d*\.\d+|NaN)\s*(-?\d*\.\d+|NaN)\s*(-?\d*\.\d+|NaN)\s*(-?\d*\.\d+|NaN)'
r'\s*(-?\d*\.\d+|NaN)\s*(-?\d*\.\d+|NaN)\s*(-?\d*\.\d+|NaN)\s*(-?\d*\.\d+|NaN)'
r'\s*(-?\d*\.\d+|NaN)\s*(-?\d*\.\d+|NaN)\s*(-?\d*\.\d+|NaN)\s*(-?\d*\.\d+|NaN)'
r'\s*(-?\d*\.\d+|NaN)\s*(-?\d*\.\d+|NaN)\s*(-?\d*\.\d+|NaN)',
lines, re.MULTILINE)
for s in stn:
params[s[0]] = {
'x': float(s[1]),
'y': float(s[2]),
'z': float(s[3]),
'epoch': float(s[4]),
'vx': float(s[5]),
'vy': float(s[6]),
'vz': float(s[7]),
# sin and cos to arranged as [n:sin, n:cos] ... it as we have it in the database
'365.250': {
'n': [
np.divide(float(s[8]), 1000.),
np.divide(float(s[10]), 1000.)
],
'e': [
np.divide(float(s[12]), 1000.),
np.divide(float(s[14]), 1000.)
],
'u': [
np.divide(float(s[16]), 1000.),
np.divide(float(s[18]), 1000.)
]
},
'182.625': {
'n': [
np.divide(float(s[9]), 1000.),
np.divide(float(s[11]), 1000.)
],
'e': [
np.divide(float(s[13]), 1000.),
np.divide(float(s[15]), 1000.)
],
'u': [
np.divide(float(s[17]), 1000.),
np.divide(float(s[19]), 1000.)
]
}
}
return params
