def station_list(cnn, stations, dates): stations = process_stnlist(cnn, stations) stn_obj = StationCollection() # use the connection to the db to get the stations for Stn in tqdm(sorted(stations, key=lambda s: (s['NetworkCode'], s['StationCode'])), ncols=80, disable=None): NetworkCode = Stn['NetworkCode'] StationCode = Stn['StationCode'] rs = cnn.query( 'SELECT * FROM rinex_proc WHERE "NetworkCode" = \'%s\' AND "StationCode" = \'%s\' AND ' '("ObservationYear", "ObservationDOY") BETWEEN (%s) AND (%s)' % (NetworkCode, StationCode, dates[0].yyyy() + ', ' + dates[0].ddd(), dates[1].yyyy() + ', ' + dates[1].ddd())) if rs.ntuples() > 0: tqdm.write(' -- %s -> adding...' % stationID(Stn)) try: stn_obj.append(Station(cnn, NetworkCode, StationCode, dates)) except pyETMException: tqdm.write( ' %s -> station exists, but there was a problem initializing ETM.' % stationID(Stn)) else: tqdm.write(' -- %s -> no data for requested time window' % stationID(Stn)) return stn_obj
def to_dharp(self, record): """ function to convert the current height code to DHARP :return: DHARP height """ if record.HeightCode == 'DHARP': return record else: htc = self.cnn.query_float('SELECT * FROM gamit_htc WHERE "AntennaCode" = \'%s\' AND "HeightCode" = \'%s\'' % (record.AntennaCode, record.HeightCode), as_dict=True) if len(htc): record.AntennaHeight = np.sqrt(np.square(float(record.AntennaHeight)) - np.square(float(htc[0]['h_offset']))) - float(htc[0]['v_offset']) if record.Comments is not None: record.Comments = record.Comments + '\nChanged from %s to DHARP by pyStationInfo.\n' \ % record.HeightCode else: record.Comments = 'Changed from %s to DHARP by pyStationInfo.\n' % record.HeightCode record.HeightCode = 'DHARP' return record else: raise pyStationInfoHeightCodeNotFound('%s: %s -> Could not translate height code %s to DHARP. ' 'Check the height codes table.' % (stationID(self), record.AntennaCode, record.HeightCode))
def load_stationinfo_records(self): # function to load the station info records in the database # returns true if records found # returns false if none found, unless allow_empty = False in which case it raises an error. stninfo = self.cnn.query( 'SELECT * FROM stationinfo WHERE "NetworkCode" = \'' + self.NetworkCode + '\' AND "StationCode" = \'' + self.StationCode + '\' ORDER BY "DateStart"') if stninfo.ntuples() == 0: if not self.allow_empty: # allow no station info if explicitly requested by the user. # Purpose: insert a station info for a new station! raise pyStationInfoException( 'Could not find ANY valid station info entry for ' + stationID(self)) self.record_count = 0 return False else: for record in stninfo.dictresult(): self.records.append( StationInfoRecord(self.NetworkCode, self.StationCode, record)) self.record_count = stninfo.ntuples() return True
def plot_etm(cnn, stack, station, directory): try: ts = stack.get_station(station['NetworkCode'], station['StationCode']) ts = pyETM.GamitSoln(cnn, ts, station['NetworkCode'], station['StationCode'], stack.project) etm = pyETM.GamitETM(cnn, station['NetworkCode'], station['StationCode'], gamit_soln=ts) pngfile = os.path.join(directory, stationID(etm) + '_gamit.png') jsonfile = os.path.join(directory, stationID(etm) + '_gamit.json') etm.plot(pngfile, plot_missing=False) file_write( os.path.join(jsonfile), json.dumps(etm.todictionary(False), indent=4, sort_keys=False)) except pyETM.pyETMException as e: tqdm.write(str(e))
def __init__(self, cnn, NetworkCode=None, StationCode=None, date=None, allow_empty=False, h_tolerance=0): self.record_count = 0 self.NetworkCode = NetworkCode self.StationCode = StationCode self.allow_empty = allow_empty self.date = None self.records = [] self.currentrecord = StationInfoRecord(NetworkCode, StationCode) self.header = '*SITE Station Name Session Start Session Stop Ant Ht HtCod Ant N ' \ 'Ant E Receiver Type Vers SwVer Receiver SN ' \ 'Antenna Type Dome Antenna SN ' # connect to the db and load the station info table if NetworkCode is not None and StationCode is not None: self.cnn = cnn if self.load_stationinfo_records(): # find the record that matches the given date if date is not None: self.date = date pDate = date.datetime() for record in self.records: DateStart = record['DateStart'].datetime() DateEnd = record['DateEnd'].datetime() # make the gap-tolerant comparison tolerance = datetime.timedelta(hours=h_tolerance) if DateStart - tolerance <= pDate <= DateEnd + tolerance: # found the record that corresponds to this date self.currentrecord = record break if self.currentrecord.DateStart is None: raise pyStationInfoException( 'Could not find a matching station.info record for ' + stationID(self) + ' ' + date.yyyymmdd() + ' (' + date.yyyyddd() + ')')
def parse_sinex(self): for sinex in os.listdir(self.pwd_glbf): if sinex.endswith('.snx'): snx = snxParse.snxFileParser(os.path.join(self.pwd_glbf, sinex)) snx.parse() self.polyhedron = snx.stationDict self.VarianceFactor = snx.varianceFactor if self.polyhedron: # remame any aliases and change keys to net.stn for stn in self.StationInstances: # replace the key try: self.polyhedron[stationID(stn)] = self.polyhedron.pop(stn.StationAlias.upper()) except KeyError: # maybe the station didn't have a solution pass return self.polyhedron, self.VarianceFactor
def generate_kml(cnn, project, stations): stnlist = [stationID(s) for s in stations] tqdm.write( ' >> Generating KML for this run (see production directory)...') kml = simplekml.Kml() rs = cnn.query_float( 'SELECT * FROM stations WHERE "NetworkCode" NOT LIKE \'?%\' ' 'ORDER BY "NetworkCode", "StationCode" ', as_dict=True) tqdm.write(' >> Adding stations in database') folder1 = kml.newfolder(name=project) folder2 = kml.newfolder(name='all stations') ICON_CIRCLE = 'http://maps.google.com/mapfiles/kml/shapes/placemark_circle.png' ICON_SQUARE = 'http://maps.google.com/mapfiles/kml/shapes/placemark_square.png' stylec = simplekml.StyleMap() stylec.normalstyle.iconstyle.icon.href = ICON_CIRCLE stylec.normalstyle.iconstyle.scale = 1.5 stylec.normalstyle.labelstyle.scale = 0 stylec.highlightstyle.iconstyle.icon.href = ICON_CIRCLE stylec.highlightstyle.iconstyle.scale = 2 stylec.highlightstyle.labelstyle.scale = 2 styles_ok = simplekml.StyleMap() styles_ok.normalstyle.iconstyle.icon.href = ICON_SQUARE styles_ok.normalstyle.iconstyle.color = 'ff00ff00' styles_ok.normalstyle.iconstyle.scale = 1.5 styles_ok.normalstyle.labelstyle.scale = 0 styles_ok.highlightstyle.iconstyle.icon.href = ICON_SQUARE styles_ok.highlightstyle.iconstyle.color = 'ff00ff00' styles_ok.highlightstyle.iconstyle.scale = 2 styles_ok.highlightstyle.labelstyle.scale = 2 styles_nok = simplekml.StyleMap() styles_nok.normalstyle.iconstyle.icon.href = ICON_SQUARE styles_nok.normalstyle.iconstyle.color = 'ff0000ff' styles_nok.normalstyle.iconstyle.scale = 1.5 styles_nok.normalstyle.labelstyle.scale = 0 styles_nok.highlightstyle.iconstyle.icon.href = ICON_SQUARE styles_nok.highlightstyle.iconstyle.color = 'ff0000ff' styles_nok.highlightstyle.iconstyle.scale = 2 styles_nok.highlightstyle.labelstyle.scale = 2 for stn in tqdm(rs, ncols=80): stn_id = stationID(stn) count = cnn.query_float( 'SELECT count(*) as cc FROM rinex_proc WHERE "NetworkCode" = \'%s\' ' 'AND "StationCode" = \'%s\'' % (stn['NetworkCode'], stn['StationCode'])) ppp_s = cnn.query_float( 'SELECT count(*) as cc FROM ppp_soln WHERE "NetworkCode" = \'%s\' ' 'AND "StationCode" = \'%s\'' % (stn['NetworkCode'], stn['StationCode'])) try: stninfo = pyStationInfo.StationInfo(cnn, stn['NetworkCode'], stn['StationCode'], allow_empty=True) _ = stninfo.return_stninfo_short() except pyStationInfo.pyStationInfoHeightCodeNotFound as e: tqdm.write('Error: %s. Station will be skipped.' % str(e)) continue if count[0][0]: completion = '%.1f' % (float(ppp_s[0][0]) / float(count[0][0]) * 100) else: completion = 'NA' if stn['DateStart']: DS = '%.3f' % stn['DateStart'] DE = '%.3f' % stn['DateEnd'] else: DS = 'NA' DE = 'NA' if stn_id in stnlist: folder = folder1 # mark the stations with less than 100 observations or with less than 60% completion (PPP) if count[0][0] >= 100 and (float(ppp_s[0][0]) / float(count[0][0]) * 100) >= 60.0: style = styles_ok else: style = styles_nok else: folder = folder2 style = stylec plt = plot_station_info_rinex(cnn, stn['NetworkCode'], stn['StationCode'], stninfo) pt = folder.newpoint(name=stn_id, coords=[(stn['lon'], stn['lat'])]) pt.stylemap = style pt.description = """<strong>%s -> %s</strong> RINEX count: %i PPP soln: %s%%<br><br> <strong>Station Information:</strong><br> <table width="880" cellpadding="0" cellspacing="0"> <tr> <td align="left" valign="top"> <p style="font-family: monospace; font-size: 8pt;">%s<br><br> <strong>Observation distribution:</strong><br> </p> <img src="data:image/png;base64, %s" alt="Observation information" /> </tr> </td> </table>""" % (DS, DE, count[0][0], completion, stninfo.return_stninfo_short().replace('\n', '<br>'), plt) if not os.path.exists('production'): os.makedirs('production') kml.savekmz('production/' + project + '.kmz')
def generate_kmz(kmz, stations, discarded, deformation_type='interseismic', units='mm/yr'): tqdm.write(' >> Generating KML (see production directory)...') kml = simplekml.Kml() folder1 = kml.newfolder(name=deformation_type) folder2 = kml.newfolder(name='discarded') # define styles styles_ok = simplekml.StyleMap() styles_nok = simplekml.StyleMap() for (s, icon_color, label_scale) in ((styles_ok.normalstyle, 'ff00ff00', 0), (styles_ok.highlightstyle, 'ff00ff00', 3), (styles_nok.normalstyle, 'ff0000ff', 0), (styles_nok.highlightstyle, 'ff0000ff', 3)): s.iconstyle.icon.href = 'http://maps.google.com/mapfiles/kml/shapes/placemark_square.png' s.iconstyle.color = icon_color s.labelstyle.scale = label_scale for stn in tqdm(stations, ncols=160, disable=None, desc=' -- Included station list'): stn_id = stationID(stn) plot = plot_station_param(stn['NetworkCode'], stn['StationCode'], deformation_type, units, stn['n'], stn['e']) pt = folder1.newpoint(name=stn_id, coords=[(stn['lon'], stn['lat'])]) pt.stylemap = styles_ok pt.description = """<strong>NE (%s): %5.2f %5.2f [%s]</strong><br><br> <table width="880" cellpadding="0" cellspacing="0"> <tr> <td align="center" valign="top"> <strong>Parameters:</strong><br> <img src="data:image/png;base64, %s" alt="Observation information" height="300" width="300"/><br> <strong>Trajectory model:</strong><br> <img src="data:image/png;base64, %s" alt="Observation information" height="750" width="1100"/> </p> </tr> </td> </table> """ % (deformation_type, stn['n'] * 1000, stn['e'] * 1000, units, plot, stn['etm']) ls = folder1.newlinestring(name=stn_id) ls.coords = [ (stn['lon'], stn['lat']), (stn['lon'] + stn['e'] * 10, stn['lat'] + stn['n'] * 10 * np.cos(stn['lat'] * np.pi / 180)) ] ls.style.linestyle.width = 3 ls.style.linestyle.color = 'ff0000ff' for stn in tqdm(discarded, ncols=160, disable=None, desc=' -- Excluded station list'): stn_id = stationID(stn) plot = plot_station_param(stn['NetworkCode'], stn['StationCode'], deformation_type, units, stn['n'], stn['e']) pt = folder2.newpoint(name=stn_id, coords=[(stn['lon'], stn['lat'])]) pt.stylemap = styles_nok pt.description = """<strong>NE (%s): %5.2f %5.2f [%s]</strong><br><br> <table width="880" cellpadding="0" cellspacing="0"> <tr> <td align="center" valign="top"> <strong>Parameters:</strong><br> <img src="data:image/png;base64, %s" alt="Observation information" height="300" width="300"/><br> <strong>Trajectory model:</strong><br> <img src="data:image/png;base64, %s" alt="Observation information" height="750" width="1100"/> </p> </tr> </td> </table> """ % (deformation_type, stn['n'] * 1000, stn['e'] * 1000, units, plot, stn['etm']) ls = folder2.newlinestring(name=stn_id) ls.coords = [ (stn['lon'], stn['lat']), (stn['lon'] + stn['e'] * 10, stn['lat'] + stn['n'] * 10 * np.cos(stn['lat'] * np.pi / 180)) ] ls.style.linestyle.width = 3 ls.style.linestyle.color = 'ff0000ff' if not os.path.exists('production'): os.makedirs('production') tqdm.write(' >> Saving kmz...') kml.savekmz(kmz)
def process_postseismic(cnn, stnlist, force_stnlist, stack, interseimic_filename, event, prev_events, sigma_cutoff, lat_lim, filename, kmz): tqdm.write( ' >> Analyzing suitability of station list to participate in postseismic model...' ) tqdm.write(' -- output filename: %s' % filename) use_station = [] discarded = [] # load the interseismic model model = np.loadtxt(interseimic_filename) # model[:, 0] -= 360 params = [] def getpost(): return { 'NetworkCode': etm.NetworkCode, 'StationCode': etm.StationCode, 'lat': etm.gamit_soln.lat[0], 'lon': etm.gamit_soln.lon[0], 'n': eq.p.params[0, 0] if eq.p.jump_type is CO_SEISMIC_DECAY else eq.p.params[0, 1], 'e': eq.p.params[1, 0] if eq.p.jump_type is CO_SEISMIC_DECAY else eq.p.params[1, 1], 'etm': etm.plot(plot_missing=False, plot_outliers=True, fileio=BytesIO()) } for stn in tqdm(stnlist, ncols=160, disable=None): stn_id = stationID(stn) tqdm.write(' -- Processing station %s' % stn_id) try: lla = cnn.query_float( 'SELECT lat,lon FROM stations WHERE "NetworkCode" = \'%s\' AND "StationCode" = \'%s\'' % (stn['NetworkCode'], stn['StationCode']), as_dict=True)[0] ve = griddata(model[:, 0:2], model[:, 2] / 1000, (lla['lon'], lla['lat']), method='cubic') vn = griddata(model[:, 0:2], model[:, 3] / 1000, (lla['lon'], lla['lat']), method='cubic') etm = pyETM.GamitETM(cnn, stn['NetworkCode'], stn['StationCode'], stack_name=stack, interseismic=[vn, ve, 0.]) for eq in [ e for e in etm.Jumps.table if e.p.jump_type in ( CO_SEISMIC_DECAY, CO_SEISMIC_JUMP_DECAY) and e.fit and etm.A is not None ]: if eq.date == event: tqdm.write( ' co-seismic decay detected for event %s (years: %.3f; data points: %i)' % (str(eq.p.jump_date), eq.constrain_years, eq.constrain_data_points)) if (eq.constrain_years >= 2.5 and eq.constrain_data_points >= eq.constrain_years * 5) \ or stn in force_stnlist: params.append(getpost()) tqdm.write( ' co-seismic decay added to the list for interpolation' ) else: tqdm.write( ' co-seismic decay not added (conditions not met)' ) discarded.append(getpost()) break except pyETM.pyETMException as e: tqdm.write(' -- %s: %s' % (stn_id, str(e))) outvar = np.array([(v['NetworkCode'] + '.' + v['StationCode'], v['lon'], v['lat'], v['e'], v['n']) for v in params], dtype=[('stn', 'U8'), ('lon', 'float64'), ('lat', 'float64'), ('e', 'float64'), ('n', 'float64')]) np.savetxt(filename, outvar, fmt=("%s", "%13.8f", "%12.8f", "%12.8f", "%12.8f")) if kmz: generate_kmz(kmz, params, discarded, 'postseismic', 'mm')
def process_interseismic(cnn, stnlist, force_stnlist, stack, sigma_cutoff, vel_cutoff, lat_lim, filename, kmz): # start by checking that the stations in the list have a linear start (no post-seismic) # and more than 2 years of data until the first earthquake or non-linear behavior tqdm.write( ' >> Analyzing suitability of station list to participate in interseismic model...' ) tqdm.write(' -- latitude cutoff: south %.2f, north %.2f' % (lat_lim[0], lat_lim[1])) tqdm.write(' -- velocity cutoff: %.2f mm/yr; output filename: %s' % (vel_cutoff, filename)) use_station = [] discarded = [] velocities = [] # min_lon = 9999 # max_lon = -9999 # min_lat = 9999 # max_lat = -9999 for stn in tqdm(stnlist, ncols=160, disable=None): try: stn_id = stationID(stn) etm = pyETM.GamitETM(cnn, stn['NetworkCode'], stn['StationCode'], stack_name=stack) use = True if stn in force_stnlist: tqdm.write(' -- %s was forced to be included in the list' % stn_id) else: # only check everything is station not included in the force list # check that station is within latitude range if etm.gamit_soln.lat[0] < lat_lim[0] or \ etm.gamit_soln.lat[0] > lat_lim[1]: tqdm.write( ' -- %s excluded because it is outside of the latitude limit' % stn_id) use = False # check that station has at least 2 years of data if etm.gamit_soln.date[-1].fyear - etm.gamit_soln.date[ 0].fyear < 2 and use: tqdm.write( ' -- %s rejected because it has less than two years of observations %s -> %s' % (stn_id, etm.gamit_soln.date[0].yyyyddd(), etm.gamit_soln.date[-1].yyyyddd())) use = False # other checks if etm.A is not None: if len(etm.Jumps.table) > 0 and use: j = next((j for j in etm.Jumps.table if j.p.jump_type == pyETM.CO_SEISMIC_JUMP_DECAY and j.fit and \ j.magnitude >= 7 and j.date.fyear < etm.gamit_soln.date[0].fyear + 1.5 ), None) if j: tqdm.write( ' -- %s has a Mw %.1f in %s and data starts in %s' % (stn_id, j.magnitude, j.date.yyyyddd(), etm.gamit_soln.date[0].yyyyddd())) use = False else: has_eq_jumps = any( True for j in etm.Jumps.table if j.p.jump_type == pyETM.CO_SEISMIC_DECAY and j.fit) if has_eq_jumps: tqdm.write( ' -- %s has one or more earthquakes before data started in %s' % (stn_id, etm.gamit_soln.date[0].yyyyddd())) use = False if (etm.factor[0] * 1000 > sigma_cutoff or etm.factor[1] * 1000 > sigma_cutoff) and use: tqdm.write( ' -- %s rejected due to large wrms %5.2f %5.2f %5.2f' % (stn_id, etm.factor[0] * 1000, etm.factor[1] * 1000, etm.factor[2] * 1000)) use = False norm = np.sqrt( np.sum(np.square(etm.Linear.p.params[0:2, 1] * 1000))) if norm > vel_cutoff and use: tqdm.write( ' -- %s rejected due to large NEU velocity: %5.2f %5.2f %5.2f NE norm %5.2f' % (stn_id, etm.Linear.p.params[0, 1] * 1000, etm.Linear.p.params[1, 1] * 1000, etm.Linear.p.params[2, 1] * 1000, norm)) use = False elif use: tqdm.write(' -- %s too few solutions to calculate ETM' % stn_id) use = False def getvel(): return { 'NetworkCode': etm.NetworkCode, 'StationCode': etm.StationCode, 'lat': etm.gamit_soln.lat[0], 'lon': etm.gamit_soln.lon[0], 'n': etm.Linear.p.params[0, 1], 'e': etm.Linear.p.params[1, 1], 'etm': etm.plot(plot_missing=False, plot_outliers=True, fileio=BytesIO()) } if use: tqdm.write( ' -- %s added NEU wrms: %5.2f %5.2f %5.2f NEU vel: %5.2f %5.2f %5.2f' % (stn_id, etm.factor[0] * 1000, etm.factor[1] * 1000, etm.factor[2] * 1000, etm.Linear.p.params[0, 1] * 1000, etm.Linear.p.params[1, 1] * 1000, etm.Linear.p.params[2, 1] * 1000)) use_station.append(stn) v = getvel() velocities.append(v) #min_lon = min(v['lon'], min_lon) #max_lon = max(v['lon'], max_lon) #min_lat = min(v['lat'], min_lat) #max_lat = max(v['lat'], max_lat) elif etm.A is not None: discarded.append(getvel()) except pyETM.pyETMException as e: tqdm.write(' -- %s: %s' % (stn_id, str(e))) tqdm.write(' >> Total number of stations for linear model: %i' % len(use_station)) outvar = np.array([(v['NetworkCode'] + '.' + v['StationCode'], v['lon'], v['lat'], v['e'], v['n']) for v in velocities], dtype=[('stn', 'U8'), ('lon', 'float64'), ('lat', 'float64'), ('e', 'float64'), ('n', 'float64')]) np.savetxt(filename, outvar, fmt=("%s", "%13.8f", "%12.8f", "%12.8f", "%12.8f")) if kmz: generate_kmz(kmz, velocities, discarded, 'interseismic', 'mm/yr')
def start(self, dirname, year, doy, dry_run=False): monitor_open = False try: # copy the folder created by GamitSession in the solution_pwd to the remote_pwd (pwd) try: if not os.path.exists(os.path.dirname(self.pwd)): os.makedirs(os.path.dirname(self.pwd)) except OSError: # racing condition having several processes trying to create the same folder # if OSError occurs, ignore and continue pass # if the local folder exists (due to previous incomplete processing, erase it). if os.path.exists(self.pwd): shutil.rmtree(self.pwd) # ready to copy the shared solution_dir to pwd shutil.copytree(self.solution_pwd, self.pwd, symlinks=True) with file_open(os.path.join(self.pwd, 'monitor.log'), 'a') as monitor: monitor_open = True def log(s): monitor.write(now_str() + ' -> ' + s + '\n') log('%s %i %i executing on %s' % (dirname, year, doy, platform.node())) log('fetching orbits') try: Sp3 = pySp3.GetSp3Orbits(self.orbits['sp3_path'], self.date, self.orbits['sp3types'], self.pwd_igs, True) # type: pySp3.GetSp3Orbits except pySp3.pySp3Exception: log('could not find principal orbits, fetching alternative') # try alternative orbits if self.options['sp3altrn']: Sp3 = pySp3.GetSp3Orbits(self.orbits['sp3_path'], self.date, self.orbits['sp3altrn'], self.pwd_igs, True) # type: pySp3.GetSp3Orbits else: raise if Sp3.type != 'igs': # rename file shutil.copyfile(Sp3.file_path, Sp3.file_path.replace(Sp3.type, 'igs')) log('fetching broadcast orbits') pyBrdc.GetBrdcOrbits(self.orbits['brdc_path'], self.date, self.pwd_brdc, no_cleanup=True) # type: pyBrdc.GetBrdcOrbits for rinex in self.params['rinex']: log('fetching rinex for %s %s %s %s' % (stationID(rinex), rinex['StationAlias'], '{:10.6f} {:11.6f}'.format(rinex['lat'], rinex['lon']), 'tie' if rinex['is_tie'] else '')) try: with pyRinex.ReadRinex(rinex['NetworkCode'], rinex['StationCode'], rinex['source'], False) as Rinex: # type: pyRinex.ReadRinex # WARNING! some multiday RINEX were generating conflicts because the RINEX has a name, say, # tuc12302.10o and the program wants to rename it as tuc12030.10o but because it's a # multiday file, during __init__ it's already split and renamed as tuc12300.10o and # additional folders are generated with the information for each file. Therefore, find # the rinex that corresponds to the date being processed and use that one instead of the # original file. These files are not allowed by pyArchiveService, but the "start point" of # the database (i.e. the files already in the folders read by pyScanArchive) has such # problems. # figure out if this station has been affected by an earthquake # if so, window the data if rinex['jump'] is not None: monitor.write( ' -> RINEX file has been windowed: ETM detected jump on ' + rinex['jump'].datetime().strftime('%Y-%m-%d %H:%M:%S') + '\n') if Rinex.multiday: # find the rinex that corresponds to the session being processed for Rnx in Rinex.multiday_rnx_list: if Rnx.date == self.date: Rnx.rename(rinex['destiny']) if rinex['jump'] is not None: self.window_rinex(Rnx, rinex['jump']) # before creating local copy, decimate file Rnx.decimate(30) Rnx.purge_comments() Rnx.compress_local_copyto(self.pwd_rinex) break else: Rinex.rename(rinex['destiny']) if rinex['jump'] is not None: self.window_rinex(Rinex, rinex['jump']) # before creating local copy, decimate file Rinex.decimate(30) Rinex.purge_comments() Rinex.compress_local_copyto(self.pwd_rinex) except (OSError, IOError): log('An error occurred while trying to copy ' + rinex['source'] + ' to ' + rinex['destiny'] + ': File skipped.') except (pyRinex.pyRinexException, Exception) as e: log('An error occurred while trying to copy ' + rinex['source'] + ': ' + str(e)) log('executing GAMIT') # create the run script self.create_replace_links() self.create_run_script() self.create_finish_script() # run the script to replace the links of the tables directory self.p = subprocess.Popen('find ./tables ! -name "otl.grid" -type l -exec ./replace_links.sh {} +', shell=True, stdout=subprocess.PIPE,stderr=subprocess.PIPE, cwd=self.pwd) _, _ = self.p.communicate() # now execute the run script if not dry_run: self.p = subprocess.Popen('./run.sh', shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=self.pwd) self.stdout, self.stderr = self.p.communicate() self.p = subprocess.Popen('./finish.sh', shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=self.pwd) self.stdout, self.stderr = self.p.communicate() # check for any fatals self.p = subprocess.Popen('grep -q \'FATAL\' monitor.log', shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=self.pwd) _, _ = self.p.communicate() self.success = (self.p.returncode != 0) # output statistics to the parent to display result = self.parse_monitor(self.success) file_append(os.path.join(self.pwd, 'monitor.log'), now_str() + ' -> return to Parallel.GAMIT\n') # no matter the result of the processing, move folder to final destination if not dry_run: self.finish() return result except: msg = traceback.format_exc() + '\nProcessing %s date %s on node %s' \ % (self.params['NetName'], self.date.yyyyddd(), platform.node()) # DDG: do not attempt to write to monitor.log or do any file operations (maybe permission problem) # problem might occur during copytree or rmtree or some other operation before opening monitor.log if monitor_open: file_append(os.path.join(self.pwd, 'monitor.log'), now_str() + ' -> ERROR in pyGamitTask.start()\n%s' % msg) # the solution folder exists because it was created by GamitSession to start the processing. # erase it to upload the result if os.path.exists(self.solution_pwd): shutil.rmtree(self.solution_pwd) # execute final error step: copy to self.solution_pwd shutil.copytree(self.pwd, self.solution_pwd, symlinks=True) # remove the remote pwd shutil.rmtree(self.pwd) # output statistics to the parent to display result = self.parse_monitor(False) else: result = {'session' : '%s %s' % (self.date.yyyyddd(), self.params['DirName']), 'Project' : self.params['NetName'], 'subnet' : self.params['subnet'], 'Year' : self.date.year, 'DOY' : self.date.doy, 'FYear' : self.date.fyear, 'wl' : 0, 'nl' : 0, 'nrms' : 0, 'relaxed_constrains' : '', 'max_overconstrained' : '', 'node' : platform.node(), 'execution_time' : 0, 'execution_date' : 0, 'missing' : '', 'success' : False, 'fatals' : [] } result['error'] = msg # return useful information to the main node return result
def parse_monitor(self, success): lines = file_readlines(self.pwd + '/monitor.log') output = ''.join(lines) try: start_time = datetime.strptime( re.findall(r'run.sh \((\d+-\d+-\d+ \d+:\d+:\d+)\): Iteration depth: 1', output, re.MULTILINE)[0], '%Y-%m-%d %H:%M:%S') except: start_time = datetime(2001, 1, 1, 0, 0, 0) try: if success: end_time = datetime.strptime( re.findall(r'finish.sh \((\d+-\d+-\d+ \d+:\d+:\d+)\): Done processing h-files and generating SINEX.' , output, re.MULTILINE)[0], '%Y-%m-%d %H:%M:%S') else: end_time = datetime.now() except: end_time = datetime(2001, 1, 1, 0, 0, 0) try: if not success: fatals = set(re.findall(r'(.*?FATAL.*)', output, re.MULTILINE)) else: fatals = [] except Exception as e: fatals = ['Could not retrieve FATALS: ' + str(e)] try: iterations = int(re.findall(r'run.sh \(\d+-\d+-\d+ \d+:\d+:\d+\): Iteration depth: (\d+)', output, re.MULTILINE)[-1]) except: iterations = 0 try: nrms = float( re.findall(r'Prefit nrms:\s+\d+.\d+[eEdD]\+\d+\s+Postfit nrms:\s+(\d+.\d+[eEdD][+-]\d+)', output, re.MULTILINE)[-1]) except: # maybe GAMIT didn't finish nrms = 100 try: updated_apr = re.findall(r' (\w+).*?Updated from', output, re.MULTILINE)[0] updated_apr = [upd.replace('_GPS', '').lower() for upd in updated_apr] upd_stn = [] for stn in updated_apr: for rinex in self.params['rinex']: if rinex['StationAlias'].lower() == stn.lower(): upd_stn += [stationID(rinex)] upd_stn = ','.join(upd_stn) except: # maybe GAMIT didn't finish upd_stn = None try: wl = float(re.findall(r'WL fixed\s+(\d+.\d+)', output, re.MULTILINE)[0]) except: # maybe GAMIT didn't finish wl = 0 try: nl = float(re.findall(r'NL fixed\s+(\d+.\d+)', output, re.MULTILINE)[0]) except: # maybe GAMIT didn't finish nl = 0 try: oc = re.findall(r'relaxing over constrained stations (\w+.*)', output, re.MULTILINE)[0] oc = oc.replace('|', ',').replace('_GPS', '').lower() oc_stn = [] for stn in oc.split(','): for rinex in self.params['rinex']: if rinex['StationAlias'].lower() == stn.lower(): oc_stn += [stationID(rinex)] oc_stn = ','.join(oc_stn) except: # maybe GAMIT didn't finish oc_stn = None try: max_overconstrained = None overcons = re.findall(r'GCR APTOL (\w+).{10}\s+([-]?\d+.\d+)', output, re.MULTILINE) if len(overcons) > 0: vals = [float(o[1]) for o in overcons] i = vals.index(max(abs(v) for v in vals)) stn = overcons[i][0] for rinex in self.params['rinex']: if rinex['StationAlias'].lower() == stn.lower(): # get the real station code max_overconstrained = stationID(rinex) else: max_overconstrained = None except: # maybe GAMIT didn't finish max_overconstrained = None try: ms = re.findall(r'No data for site (\w+)', output, re.MULTILINE) ds = re.findall(r'.*deleting station (\w+)', output, re.MULTILINE) missing_sites = [] for stn in ms + ds: for rinex in self.params['rinex']: if rinex['StationAlias'].lower() == stn.lower() and \ stationID(rinex) not in missing_sites: if stn in ms: missing_sites += ['(' + stationID(rinex) + ')'] else: missing_sites += [stationID(rinex)] except: # maybe GAMIT didn't finish missing_sites = [] return {'session' : '%s %s' % (self.date.yyyyddd(), self.params['DirName']), 'Project' : self.params['NetName'], 'subnet' : self.params['subnet'], 'Year' : self.date.year, 'DOY' : self.date.doy, 'FYear' : self.date.fyear, 'wl' : wl, 'nl' : nl, 'nrms' : nrms, 'relaxed_constrains' : oc_stn, 'max_overconstrained' : max_overconstrained, 'updated_apr' : upd_stn, 'iterations' : iterations, 'node' : platform.node(), 'execution_time' : int((end_time - start_time).total_seconds() / 60.0), 'execution_date' : start_time, 'missing' : missing_sites, 'success' : success, 'fatals' : fatals }
def print_summary(stations, sessions, dates): # output a summary of each network print('') print(' >> Summary of stations in this project') print(' -- Selected stations (%i):' % (len(stations))) Utils.print_columns([stationID(item) for item in stations]) min_stn = 99999 min_date = pyDate.Date(year=1980, doy=1) for session in sessions: if min_stn > len(session.stations_dict): min_stn = len(session.stations_dict) min_date = session.date print('') print(' >> Minimum number of stations (%i) on day %s' % (min_stn, min_date.yyyyddd())) # output a summary of the missing days per station: print('') sys.stdout.write(' >> Summary of data per station (' + chr(0x258C) + ' = 1 DOY)\n') if (dates[1] - dates[0]) / 2.0 > 120: cut_len = int(math.ceil((dates[1] - dates[0]) / 4.0)) else: cut_len = dates[1] - dates[0] for stn in stations: # make a group per year for year in sorted(set(d.year for d in stn.good_rinex)): sys.stdout.write('\n -- %s:\n' % stationID(stn)) missing_dates = set(m.doy for m in stn.missing_rinex if m.year == year) p_doys = [m.doy for m in stn.good_rinex if m.year == year] sys.stdout.write('\n%i:\n %03i>' % (year, p_doys[0])) for i, doy in enumerate(zip(p_doys[0:-1:2], p_doys[1::2])): if doy[0] in missing_dates: if doy[1] in missing_dates: c = ' ' else: c = chr(0x2590) else: if doy[1] in missing_dates: c = chr(0x258C) else: c = chr(0x2588) sys.stdout.write(c) if i + 1 == cut_len: sys.stdout.write('<%03i\n' % doy[0]) sys.stdout.write(' %03i>' % (doy[0] + 1)) if len(p_doys) % 2 != 0: # last one missing if p_doys[-1] in missing_dates: sys.stdout.write(' ') else: sys.stdout.write(chr(0x258C)) if cut_len < len(p_doys): sys.stdout.write('< %03i\n' % (p_doys[-1])) else: sys.stdout.write('<%03i\n' % (p_doys[-1])) else: sys.stdout.write('<%03i\n' % (p_doys[-1])) return
def main(): parser = argparse.ArgumentParser(description='Program to perform weekly loosely-constrained solutions. Combination ' 'is performed using GLOBK. Result is output in SINEX format.') parser.add_argument('stnlist', type=str, nargs='+', metavar='all|net.stnm', help="List of networks/stations to include in the solution.") parser.add_argument('-s', '--session_config', type=str, nargs=1, metavar='session.cfg', help="Filename with the session configuration to run Parallel.GAMIT") parser.add_argument('-w', '--gpsweek', nargs=1, help="GPS week to combine.") parser.add_argument('-e', '--exclude', type=str, nargs='+', metavar='station', help="List of stations to exclude (e.g. -e igm1 lpgs vbca)") args = parser.parse_args() cnn = dbConnection.Cnn("gnss_data.cfg") # get the working dates date_s = pyDate.Date(gpsWeek=int(args.gpsweek[0]), gpsWeekDay=0) date_e = pyDate.Date(gpsWeek=int(args.gpsweek[0]), gpsWeekDay=6) print(' >> Working with GPS week ' + args.gpsweek[0] + ' (%s to %s)' % (date_s.yyyyddd(), date_e.yyyyddd())) exclude = args.exclude if exclude is not None: print(' >> User selected list of stations to exclude:') Utils.print_columns(exclude) args.stnlist += ['-' + exc for exc in exclude] # get the station list stnlist = Utils.process_stnlist(cnn, args.stnlist) # check that the selected stations have all different station codes # otherwise, exit with error for i in range(len(stnlist) - 1): for j in range(i + 1, len(stnlist)): if stnlist[i]['StationCode'] == stnlist[j]['StationCode']: print('During station selection, two identical station codes were found. ' 'Please remove one and try again.') exit() GamitConfig = pyGamitConfig.GamitConfiguration(args.session_config[0]) # type: pyGamitConfig.GamitConfiguration project = GamitConfig.NetworkConfig.network_id.lower() org = GamitConfig.gamitopt['org'] print(' >> REMINDER: To automatically remove outliers during the weekly combination, ' 'first run DRA.py to analyze the daily repetitivities') soln_pwd = GamitConfig.gamitopt['solutions_dir'] # create a globk directory in production if not os.path.exists('production/globk'): os.makedirs('production/globk') # check if week folder exists globk_pwd = 'production/globk/' + args.gpsweek[0] if os.path.exists(globk_pwd): rmtree(globk_pwd) # create the directory os.makedirs(globk_pwd) glx_list = [] # make a list of the h files that need to be combined for day in range(0, 7): date = pyDate.Date(gpsWeek = int(args.gpsweek[0]), gpsWeekDay = day) soln_dir = os.path.join(soln_pwd, "%s/%s/%s/glbf" % (date.yyyy(), date.ddd(), project)) if os.path.exists(soln_dir): glx = glob.glob(os.path.join(soln_dir, '*.GLX.*')) if not glx: glx = glob.glob(os.path.join(soln_dir, '*.glx')) glx_list.append({'file': glx[0], 'gpsweek': date.wwwwd()}) # create the earthquakes.txt file to remove outliers with file_open(globk_pwd + '/eq_rename.txt', 'w') as fd: rename = [] remove = [] use_site = [] fd.write('# LIST OF OUTLIERS DETECTED BY DRA\n') for stn in stnlist: # obtain the filtered solutions rm = cnn.query_float('SELECT * FROM gamit_soln_excl WHERE "Project" = \'%s\' AND "NetworkCode" = \'%s\'' ' AND "StationCode" = \'%s\' AND ("Year", "DOY") BETWEEN (%i, %i) AND (%i, %i) ' 'ORDER BY residual' % (project, stn['NetworkCode'], stn['StationCode'], date_s.year, date_s.doy, date_e.year, date_e.doy), as_dict=True) # obtain the total number of solutions sl = cnn.query_float('SELECT * FROM gamit_soln WHERE "Project" = \'%s\' AND "NetworkCode" = \'%s\'' ' AND "StationCode" = \'%s\' AND ("Year", "DOY") BETWEEN (%i, %i) AND (%i, %i) ' % (project, stn['NetworkCode'], stn['StationCode'], date_s.year, date_s.doy, date_e.year, date_e.doy), as_dict=True) for i, r in enumerate(rm): date = pyDate.Date(year=r['Year'], doy=r['DOY']) # if the number of rejected solutions is equal to the number of total solutions, # leave out the first one (i == 0) which is the one with the lowest residual (see ORDER BY in rm) if len(rm) < len(sl) or (len(rm) == len(sl) and i != 0): fd.write(' rename %s_gps %s_xcl %-20s %s %02i %02i 0 0 %s %02i %02i 24 0\n' % (stn['StationCode'], stn['StationCode'], org + date.wwwwd() + '.GLX', date.yyyy()[2:], date.month, date.day, date.yyyy()[2:], date.month, date.day)) # check for renames that might not agree between days mv = cnn.query_float('SELECT * FROM gamit_subnets WHERE "Project" = \'%s\' AND ("Year", "DOY") ' 'BETWEEN (%i, %i) AND (%i, %i) AND \'%s.%s\' = ANY(stations)' % (project, date_s.year, date_s.doy, date_e.year, date_e.doy, stn['NetworkCode'], stn['StationCode']), as_dict=True) for m in mv: date = pyDate.Date(year=m['Year'], doy=m['DOY']) # check on each day to see if alias agrees with station code for i, s in enumerate(m['stations']): if s.split('.')[1] != m['alias'][i] and s == stationID(stn): print(' -- %s alias for %s = %s: renaming' \ % (date.yyyyddd(), stationID(stn), m['alias'][i])) # change the name of the station to the original name rename.append(' rename %s_gps %s_dup %-20s %s %02i %02i 0 0 %s %02i %02i 24 0\n' % (m['alias'][i], stn['StationCode'], org + date.wwwwd() + '.GLX', date.yyyy()[2:], date.month, date.day, date.yyyy()[2:], date.month, date.day)) use_site.append('%s_dup' % stn['StationCode']) elif s not in [stationID(st) for st in stnlist]: # print ' -- Removing %s: not selected' % s # just in case, remove any other occurrences of this station code remove.append(' rename %s_gps %s_xcl %-20s %s %02i %02i 0 0 %s %02i %02i 24 0\n' % (m['alias'][i], m['alias'][i], org + date.wwwwd() + '.GLX', date.yyyy()[2:], date.month, date.day, date.yyyy()[2:], date.month, date.day)) else: use_site.append('%s_gps' % stn['StationCode']) fd.write('# LIST OF STATIONS TO BE REMOVED\n') fd.write(''.join(remove)) fd.write('# LIST OF STATIONS TO BE RENAMED\n') fd.write(''.join(rename)) print(' >> Converting to SINEX the daily solutions') for day, glx in enumerate(glx_list): date = pyDate.Date(gpsWeek = int(args.gpsweek[0]), gpsWeekDay = day) print(' -- Working on %s' % date.wwwwd()) # delete the existing GLX files for ff in glob.glob(globk_pwd + '/*.GLX'): os.remove(ff) Globk(globk_pwd, org, [glx], date.wwww(), date.gpsWeekDay + 8, ' '.join(set(use_site))) # convert the file to a valid gpsweek day move(globk_pwd + '/' + org + date.wwww() + '%i.snx' % (date.gpsWeekDay + 8), globk_pwd + '/' + org + date.wwww() + '%i.snx' % date.gpsWeekDay) process_sinex(cnn, project, [date, date], globk_pwd + '/' + org + date.wwww() + '%i.snx' % date.gpsWeekDay) # delete the existing GLX files: get ready for weekly combination for ff in glob.glob(globk_pwd + '/*.GLX'): os.remove(ff) # ready to pass list to globk object Globk(globk_pwd, org, glx_list, date_s.wwww(), 7, ' '.join(set(use_site))) print(' >> Formatting the SINEX file') process_sinex(cnn, project, [date_s, date_e], globk_pwd + '/' + org + date_s.wwww() + '7.snx')
def __init__(self, cnn, archive, name, org, subnet, date, GamitConfig, stations, ties=(), centroid=()): """ The GAMIT session object creates all the directory structure and configuration files according to the parameters set in GamitConfig. Two stations list are passed and merged to create the session :param cnn: connection to database object :param archive: archive object to find rinex files in archive structure :param name: name of the project/network :param org: name of the organization :param subnet: subnet number (may be None, in which case the directory name will not show ORGXX :param date: date that is being processed :param GamitConfig: configuration to run gamit :param stations: list of stations to be processed :param ties: tie stations as obtained by pyNetwork """ self.NetName = name self.org = org self.subnet = subnet if subnet is not None: self.DirName = '%s.%s%02i' % (self.NetName, self.org, self.subnet) else: self.DirName = self.NetName self.date = date self.GamitOpts = GamitConfig.gamitopt # type: pyGamitConfig.GamitConfiguration().gamitopt self.Config = GamitConfig # type: pyGamitConfig.GamitConfiguration self.frame = None self.params = None # to store the polyhedron read from the final SINEX self.polyhedron = None self.VarianceFactor = None # gamit task will be filled with the GamitTask object self.GamitTask = None self.solution_base = self.GamitOpts['solutions_dir'].rstrip('/') # tie station dictionary (to build KMLs, do not change) self.tie_dict = [{'name' : stationID(stn), 'coords': [(stn.lon, stn.lat)]} for stn in ties] # station dictionary (to build KMLs, do not change) self.stations_dict = [{'name' : stationID(stn), 'coords' : [(stn.lon, stn.lat)]} for stn in stations] # make StationInstances station_instances = [] for stn in stations: try: station_instances += [StationInstance(cnn, archive, stn, date, GamitConfig)] except pyRinexName.RinexNameException: tqdm.write(' -- WARNING (station instance): station %s on day %s appears to have a badly formed RINEX ' 'filename. Please check the archive and make sure all filenames follow the RINEX 2/3 ' 'convention. Station has been excluded from the GAMIT session.' % (stationID(stn), date.yyyyddd())) # do the same with ties for stn in ties: try: station_instances += [StationInstance(cnn, archive, stn, date, GamitConfig, is_tie=True)] except pyRinexName.RinexNameException: tqdm.write(' -- WARNING (tie instance): station %s on day %s appears to have a badly formed RINEX ' 'filename. Please check the archive and make sure all filenames follow the RINEX 2/3 ' 'convention. Station has been excluded from the GAMIT session.' % (stationID(stn), date.yyyyddd())) self.StationInstances = station_instances # create working dirs for this session last_path = '/%s/%s/%s' % (date.yyyy(), date.ddd(), self.DirName) self.solution_pwd = self.solution_base + last_path # the remote pwd is the directory where the processing will be performed self.remote_pwd = 'production/gamit' + last_path row_key = {'Year' : date.year, 'DOY' : date.doy, 'Project' : self.NetName, 'subnet' : 0 if subnet is None else subnet} try: # attempt to retrieve the session from the database. If error is raised, then the session has to be # reprocessed cnn.get('gamit_stats', row_key.copy()) self.ready = True except: self.ready = False try: # since ready == False, then try to delete record in subnets cnn.delete('gamit_subnets', row_key.copy()) except: pass # a list to report missing data for this session self.missing_data = [] if not os.path.exists(self.solution_pwd): # if the path does not exist, create it! os.makedirs(self.solution_pwd) # force ready = False, no matter what the database says self.ready = False try: cnn.delete('gamit_stats', row_key.copy()) cnn.delete('gamit_subnets', row_key.copy()) except: pass elif os.path.exists(self.solution_pwd) and not self.ready: # if the solution directory exists but the session is not ready, kill the directory rmtree(self.solution_pwd) if not self.ready: # insert the subnet in the database cnn.insert('gamit_subnets', {**row_key, 'stations' : '{%s}' % ','.join(stationID(s) for s in stations + list(ties)), 'alias' : '{%s}' % ','.join(s.StationAlias for s in stations + list(ties)), 'ties' : '{%s}' % ','.join(s['name'] for s in self.tie_dict), 'centroid' : '{%s}' % ','.join('%.1f' % c for c in centroid)}) self.pwd_igs = os.path.join(self.solution_pwd, 'igs') self.pwd_brdc = os.path.join(self.solution_pwd, 'brdc') self.pwd_rinex = os.path.join(self.solution_pwd, 'rinex') self.pwd_tables = os.path.join(self.solution_pwd, 'tables') self.pwd_glbf = os.path.join(self.solution_pwd, 'glbf') self.pwd_proc = os.path.join(self.solution_pwd, date.ddd()) if not self.ready: # only create folders, etc if it was determined the solution isn't ready if not os.path.exists(self.pwd_igs): os.makedirs(self.pwd_igs) if not os.path.exists(self.pwd_brdc): os.makedirs(self.pwd_brdc) if os.path.exists(self.pwd_rinex): # delete any possible rinex files from a truncated session rmtree(self.pwd_rinex) os.makedirs(self.pwd_rinex) if not os.path.exists(self.pwd_tables): os.makedirs(self.pwd_tables) # check that the processing directory doesn't exist. # if it does, remove (it has already been determined that the solution is not ready if os.path.exists(self.pwd_glbf): rmtree(self.pwd_glbf) if os.path.exists(self.pwd_proc): rmtree(self.pwd_proc) self.generate_kml()
def main(): parser = argparse.ArgumentParser(description='GNSS time series stacker') parser.add_argument( 'project', type=str, nargs=1, metavar='{project name}', help= "Specify the project name used to process the GAMIT solutions in Parallel.GAMIT." ) parser.add_argument( 'stack_name', type=str, nargs=1, metavar='{stack name}', help= "Specify a name for the stack: eg. itrf2014 or posgar07b. This name should be unique " "and cannot be repeated for any other solution project") parser.add_argument( '-max', '--max_iters', nargs=1, type=int, metavar='{max_iter}', help="Specify maximum number of iterations. Default is 4.") parser.add_argument( '-exclude', '--exclude_stations', nargs='+', type=str, metavar='{net.stnm}', help="Manually specify stations to remove from the stacking process.") parser.add_argument( '-use', '--use_stations', nargs='+', type=str, metavar='{net.stnm}', help="Manually specify stations to use for the stacking process.") parser.add_argument( '-dir', '--directory', type=str, help= "Directory to save the resulting PNG files. If not specified, assumed to be the " "production directory") parser.add_argument('-redo', '--redo_stack', action='store_true', help="Delete the stack and redo it from scratch") parser.add_argument('-plot', '--plot_stack_etms', action='store_true', default=False, help="Plot the stack ETMs after computation is done") parser.add_argument( '-constrains', '--external_constrains', nargs='+', help= "File with external constrains parameters (position, velocity and periodic). These may be " "from a parent frame such as ITRF. " "Inheritance will occur with stations on the list whenever a parameter exists. " "Example: -constrains itrf14.txt " "Format is: net.stn x y z epoch vx vy vz sn_1y sn_6m cn_1y cn_6m se_1y se_6m ce_1y ce_6m " "su_1y su_6m cu_1y cu_6m ") parser.add_argument( '-d', '--date_end', nargs=1, metavar='date', help= 'Limit the polyhedrons to the specified date. Can be in wwww-d, yyyy_ddd, yyyy/mm/dd ' 'or fyear format') parser.add_argument('-np', '--noparallel', action='store_true', help="Execute command without parallelization.") args = parser.parse_args() cnn = dbConnection.Cnn("gnss_data.cfg") Config = pyOptions.ReadOptions( "gnss_data.cfg") # type: pyOptions.ReadOptions JobServer = pyJobServer.JobServer( Config, run_parallel=not args.noparallel) # type: pyJobServer.JobServer if args.max_iters: max_iters = int(args.max_iters[0]) else: max_iters = 4 print(' >> Defaulting to 4 iterations') exclude_stn = args.exclude_stations if args.exclude_stations else [] use_stn = args.use_stations if args.use_stations else [] dates = [Date(year=1980, doy=1), Date(datetime=datetime.now())] if args.date_end is not None: try: dates = process_date( [str(Date(year=1980, doy=1).fyear), args.date_end[0]]) except ValueError as e: parser.error(str(e)) # create folder for plots if args.directory: if not os.path.exists(args.directory): os.mkdir(args.directory) else: if not os.path.exists('production'): os.mkdir('production') args.directory = 'production' # load the ITRF dat file with the periodic space components if args.external_constrains: constrains = load_constrains(args.external_constrains[0]) else: constrains = None # create the stack object stack = pyStack.Stack(cnn, args.project[0], args.stack_name[0], args.redo_stack, end_date=dates[1]) # stack.align_spaces(frame_params) # stack.to_json('alignment.json') # exit() for i in range(max_iters): # create the target polyhedrons based on iteration number (i == 0: PPP) target = calculate_etms(cnn, stack, JobServer, i) qbar = tqdm(total=len(stack), ncols=160, desc=' >> Aligning polyhedrons (%i of %i)' % (i + 1, max_iters), disable=None) # work on each polyhedron of the stack for j in range(len(stack)): qbar.update() if not stack[j].aligned: # do not move this if up one level: to speed up the target polyhedron loading process, the target is # set to an empty list when the polyhedron is already aligned if stack[j].date != target[j].date: # raise an error if dates don't agree! raise Exception( "Error processing %s: dates don't agree (target date %s)" % (stack[j].date.yyyyddd(), target[j].date.yyyyddd())) else: # should only attempt to align a polyhedron that is unaligned # do not set the polyhedron as aligned unless we are in the max iteration step stack[j].align(target[j], True if i == max_iters - 1 else False) # write info to the screen qbar.write( ' -- %s (%04i) %2i it: wrms: %4.1f T %5.1f %5.1f %5.1f ' 'R (%5.1f %5.1f %5.1f)*1e-9' % (stack[j].date.yyyyddd(), stack[j].stations_used, stack[j].iterations, stack[j].wrms * 1000, stack[j].helmert[-3] * 1000, stack[j].helmert[-2] * 1000, stack[j].helmert[-1] * 1000, stack[j].helmert[-6], stack[j].helmert[-5], stack[j].helmert[-4])) stack.transformations.append([poly.info() for poly in stack]) qbar.close() if args.redo_stack: # before removing common modes (or inheriting periodic terms), calculate ETMs with final aligned solutions calculate_etms(cnn, stack, JobServer, iterations=None, create_target=False) # only apply common mode removal if redoing the stack if args.external_constrains: stack.remove_common_modes(constrains) # here, we also align the stack in velocity and coordinate space stack.align_spaces(constrains) else: stack.remove_common_modes() # calculate the etms again, after removing or inheriting parameters calculate_etms(cnn, stack, JobServer, iterations=None, create_target=False) # save the json with the information about the alignment stack.to_json(args.stack_name[0] + '_alignment.json') # save polyhedrons to the database stack.save() if args.plot_stack_etms: qbar = tqdm(total=len(stack.stations), ncols=160, disable=None) for stn in stack.stations: # plot the ETMs qbar.update() qbar.postfix = stationID(stn) plot_etm(cnn, stack, stn, args.directory) qbar.close()