def main(): parser = argparse.ArgumentParser(description='GNSS time series stacker') parser.add_argument('project', type=str, nargs=1, metavar='{project name}', help="Specify the project name used to process the GAMIT solutions in Parallel.GAMIT.") parser.add_argument('-d', '--date_filter', nargs='+', metavar='date', help='Date range filter Can be specified in yyyy/mm/dd yyyy_doy wwww-d format') args = parser.parse_args() cnn = dbConnection.Cnn("gnss_data.cfg") Config = pyOptions.ReadOptions("gnss_data.cfg") # type: pyOptions.ReadOptions # create the execution log dates = [pyDate.Date(year=1980, doy=1), pyDate.Date(year=2100, doy=1)] try: dates = process_date(args.date_filter) except ValueError as e: parser.error(str(e)) # create folder for plots if not os.path.isdir(args.project[0]): os.makedirs(args.project[0]) ######################################## # load polyhedrons project = dra(cnn, args.project[0], dates)
def main(): parser = argparse.ArgumentParser(description='GNSS time series stacker') parser.add_argument('project', type=str, nargs=1, metavar='{project name}', help="Specify the project name used to process the GAMIT solutions in Parallel.GAMIT.") parser.add_argument('sinex', type=str, nargs=1, metavar='{project name}', help="SINEX file to update.") parser.add_argument('-d', '--date_filter', nargs='+', metavar='date', help='Date range filter can be specified in yyyy/mm/dd yyyy_doy wwww-d format') args = parser.parse_args() cnn = dbConnection.Cnn("gnss_data.cfg") Config = pyOptions.ReadOptions("gnss_data.cfg") # type: pyOptions.ReadOptions dates = [pyDate.Date(year=1980, doy=1), pyDate.Date(year=2100, doy=1)] try: dates = process_date(args.date_filter) except ValueError as e: parser.error(str(e)) sinex = args.sinex[0] project = args.project[0] process_sinex(cnn, project, dates, sinex)
def main(): parser = argparse.ArgumentParser(description='GNSS time series stacker') parser.add_argument('project', type=str, nargs=1, metavar='{project name}', help="Specify the project name used to process the GAMIT solutions in Parallel.GAMIT.") parser.add_argument('-d', '--date_filter', nargs='+', metavar='date', help='Date range filter Can be specified in yyyy/mm/dd yyyy_doy wwww-d format') args = parser.parse_args() cnn = dbConnection.Cnn("gnss_data.cfg") project = args.project[0] dates = [pyDate.Date(year=1980, doy=1), pyDate.Date(year=2100, doy=1)] try: dates = process_date(args.date_filter) except ValueError as e: parser.error(str(e)) # create folder for plots if not os.path.isdir(project + '_dra'): os.makedirs(project + '_dra') ######################################## # load polyhedrons dra = DRA(cnn, args.project[0], dates[1]) dra.stack_dra() for stn in tqdm(dra.stations): NetworkCode = stn['NetworkCode'] StationCode = stn['StationCode'] # load from the db ts = dra.get_station(NetworkCode, StationCode) if ts.size: try: if ts.shape[0] > 2: dts = np.append(np.diff(ts[:, 0:3], axis=0), ts[1:, -3:], axis=1) dra_ts = pyETM.GamitSoln(cnn, dts, NetworkCode, StationCode, project) etm = pyETM.DailyRep(cnn, NetworkCode, StationCode, False, False, dra_ts) etm.plot(pngfile='%s/%s.%s_DRA.png' % (project + '_dra', NetworkCode, StationCode), plot_missing=False) except Exception as e: tqdm.write(' -->' + str(e)) dra.to_json(project + '_dra.json')
def main(): parser = argparse.ArgumentParser( description='Plot ETM for stations in the database') parser.add_argument( 'stnlist', type=str, nargs='+', help= "List of networks/stations to plot given in [net].[stnm] format or just [stnm] " "(separated by spaces; if [stnm] is not unique in the database, all stations with that " "name will be plotted). Use keyword 'all' to plot all stations in all networks. " "If [net].all is given, all stations from network [net] will be plotted" ) parser.add_argument('-nop', '--no_plots', action='store_true', help="Do not produce plots", default=False) parser.add_argument('-nom', '--no_missing_data', action='store_true', help="Do not show missing days", default=False) parser.add_argument('-nm', '--no_model', action='store_true', help="Plot time series without fitting a model") parser.add_argument('-r', '--residuals', action='store_true', help="Plot time series residuals", default=False) parser.add_argument( '-dir', '--directory', type=str, help= "Directory to save the resulting PNG files. If not specified, assumed to be the " "production directory") parser.add_argument( '-json', '--json', type=int, help="Export ETM adjustment to JSON. Append '0' to just output " "the ETM parameters, '1' to export time series without " "model and '2' to export both time series and model.") parser.add_argument( '-gui', '--interactive', action='store_true', help="Interactive mode: allows to zoom and view the plot interactively" ) parser.add_argument( '-win', '--time_window', nargs='+', metavar='interval', help= 'Date range to window data. Can be specified in yyyy/mm/dd, yyyy.doy or as a single ' 'integer value (N) which shall be interpreted as last epoch-N') parser.add_argument( '-q', '--query', nargs=2, metavar='{type} {date}', type=str, help= 'Dates to query the ETM. Specify "model" or "solution" to get the ETM value or the value ' 'of the daily solution (if exists). Output is in XYZ.') parser.add_argument( '-gamit', '--gamit', type=str, nargs=1, metavar='{stack}', help="Plot the GAMIT time series specifying which stack name to plot.") parser.add_argument( '-lang', '--language', type=str, help="Change the language of the plots. Default is English. " "Use ESP to select Spanish. To add more languages, " "include the ISO 639-1 code in pyETM.py", default='ENG') parser.add_argument('-hist', '--histogram', action='store_true', help="Plot histogram of residuals") parser.add_argument( '-file', '--filename', type=str, help= "Obtain data from an external source (filename). Format should be specified with -format." ) parser.add_argument( '-format', '--format', nargs='+', type=str, help= "To be used together with --filename. Specify order of the fields as found in the input " "file. Format strings are gpsWeek, gpsWeekDay, year, doy, fyear, month, day, mjd, " "x, y, z, na. Use 'na' to specify a field that should be ignored. If fields to be ignored " "are at the end of the line, then there is no need to specify those.") parser.add_argument('-outliers', '--plot_outliers', action='store_true', help="Plot an additional panel with the outliers") parser.add_argument('-vel', '--velocity', action='store_true', help="During query, output the velocity in XYZ.") parser.add_argument('-seasonal', '--seasonal_terms', action='store_true', help="During query, output the seasonal terms in NEU.") parser.add_argument('-quiet', '--suppress_messages', action='store_true', help="Quiet mode: suppress information messages") args = parser.parse_args() cnn = dbConnection.Cnn('gnss_data.cfg') stnlist = Utils.process_stnlist(cnn, args.stnlist) # define the language pyETM.LANG = args.language.lower() # set the logging level if not args.suppress_messages: pyETM.logger.setLevel(pyETM.INFO) ##################################### # date filter dates = None if args.time_window is not None: if len(args.time_window) == 1: try: dates = process_date(args.time_window, missing_input=None, allow_days=False) dates = (dates[0].fyear, ) except ValueError: # an integer value dates = float(args.time_window[0]) else: dates = process_date(args.time_window) dates = (dates[0].fyear, dates[1].fyear) if stnlist: # do the thing if args.directory: if not os.path.exists(args.directory): os.mkdir(args.directory) else: if not os.path.exists('production'): os.mkdir('production') args.directory = 'production' for stn in stnlist: try: if args.gamit is None and args.filename is None: etm = pyETM.PPPETM(cnn, stn['NetworkCode'], stn['StationCode'], False, args.no_model) elif args.filename is not None: etm = from_file(args, cnn, stn) else: polyhedrons = cnn.query_float( 'SELECT "X", "Y", "Z", "Year", "DOY" FROM stacks ' 'WHERE "name" = \'%s\' AND "NetworkCode" = \'%s\' AND ' '"StationCode" = \'%s\' ' 'ORDER BY "Year", "DOY", "NetworkCode", "StationCode"' % (args.gamit[0], stn['NetworkCode'], stn['StationCode'])) soln = pyETM.GamitSoln(cnn, polyhedrons, stn['NetworkCode'], stn['StationCode'], args.gamit[0]) etm = pyETM.GamitETM(cnn, stn['NetworkCode'], stn['StationCode'], False, args.no_model, gamit_soln=soln) # print ' > %5.2f %5.2f %5.2f %i %i' % \ # (etm.factor[0]*1000, etm.factor[1]*1000, etm.factor[2]*1000, etm.soln.t.shape[0], # etm.soln.t.shape[0] - # np.sum(np.logical_and(np.logical_and(etm.F[0], etm.F[1]), etm.F[2]))) # print two largest outliers if etm.A is not None: lres = np.sqrt(np.sum(np.square(etm.R), axis=0)) slres = lres[np.argsort(-lres)] print ' >> Two largest residuals:' for i in [0, 1]: print(' %s %6.3f %6.3f %6.3f' % (pyDate.Date(mjd=etm.soln.mjd[ lres == slres[i]]).yyyyddd(), etm.R[0, lres == slres[i]], etm.R[1, lres == slres[i]], etm.R[2, lres == slres[i]])) if args.interactive: xfile = None else: if args.gamit is None: if args.filename is None: xfile = os.path.join( args.directory, '%s.%s_ppp' % (etm.NetworkCode, etm.StationCode)) else: xfile = os.path.join( args.directory, '%s.%s_file' % (etm.NetworkCode, etm.StationCode)) else: xfile = os.path.join( args.directory, '%s.%s_gamit' % (etm.NetworkCode, etm.StationCode)) # leave pngfile empty to enter interactive mode (GUI) if not args.no_plots: etm.plot(xfile + '.png', t_win=dates, residuals=args.residuals, plot_missing=not args.no_missing_data, plot_outliers=args.plot_outliers) if args.histogram: etm.plot_hist(xfile + '_hist.png') if args.json is not None: with open(xfile + '.json', 'w') as f: if args.json == 1: json.dump(etm.todictionary(time_series=True), f, indent=4, sort_keys=False) elif args.json == 2: json.dump(etm.todictionary(time_series=True, model=True), f, indent=4, sort_keys=False) else: json.dump(etm.todictionary(False), f, indent=4, sort_keys=False) if args.query is not None: model = True if args.query[0] == 'model' else False q_date = pyDate.Date(fyear=float(args.query[1])) xyz, _, _, txt = etm.get_xyz_s(q_date.year, q_date.doy, force_model=model) strp = '' # if user requests velocity too, output it if args.velocity: if etm.A is not None: vxyz = etm.rotate_2xyz(etm.Linear.p.params[:, 1]) strp = '%8.5f %8.5f %8.5f ' \ % (vxyz[0, 0], vxyz[1, 0], vxyz[2, 0]) # also output seasonal terms, if requested if args.seasonal_terms: if etm.Periodic.frequency_count > 0: strp += ' '.join([ '%8.5f' % (x * 1000) for x in etm.Periodic.p.params.flatten().tolist() ]) print ' %s.%s %14.5f %14.5f %14.5f %8.3f %s -> %s' \ % (etm.NetworkCode, etm.StationCode, xyz[0], xyz[1], xyz[2], q_date.fyear, strp, txt) print 'Successfully plotted ' + stn['NetworkCode'] + '.' + stn[ 'StationCode'] except pyETM.pyETMException as e: print str(e) except Exception: print 'Error during processing of ' + stn[ 'NetworkCode'] + '.' + stn['StationCode'] print traceback.format_exc() pass
def main(): parser = argparse.ArgumentParser( description='Archive operations Main Program') parser.add_argument( 'stnlist', type=str, nargs='+', metavar='all|net.stnm', help= "List of networks/stations to process given in [net].[stnm] format or just [stnm] " "(separated by spaces; if [stnm] is not unique in the database, all stations with that " "name will be processed). Use keyword 'all' to process all stations in the database. " "If [net].all is given, all stations from network [net] will be processed. " "Alternatevily, a file with the station list can be provided.") parser.add_argument( '-date', '--date_range', nargs='+', action=required_length(1, 2), metavar='date_start|date_end', help="Date range to check given as [date_start] or [date_start] " "and [date_end]. Allowed formats are yyyy.doy or yyyy/mm/dd..") parser.add_argument( '-win', '--window', nargs=1, metavar='days', type=int, help= "Download data from a given time window determined by today - {days}.") try: args = parser.parse_args() cnn = dbConnection.Cnn('gnss_data.cfg') Config = pyOptions.ReadOptions('gnss_data.cfg') stnlist = Utils.process_stnlist(cnn, args.stnlist) print ' >> Selected station list:' print_columns([ item['NetworkCode'] + '.' + item['StationCode'] for item in stnlist ]) dates = [] try: if args.window: # today - ndays d = pyDate.Date(year=datetime.now().year, month=datetime.now().month, day=datetime.now().day) dates = [d - int(args.window[0]), d] else: dates = process_date(args.date_range) except ValueError as e: parser.error(str(e)) if dates[0] < pyDate.Date(gpsWeek=650, gpsWeekDay=0): dates = [ pyDate.Date(gpsWeek=650, gpsWeekDay=0), pyDate.Date(year=datetime.now().year, month=datetime.now().month, day=datetime.now().day) ] # go through the dates drange = np.arange(dates[0].mjd, dates[1].mjd + 1, 1) download_data(cnn, Config, stnlist, drange) except argparse.ArgumentTypeError as e: parser.error(str(e))
def main(): parser = argparse.ArgumentParser( description= 'Database integrity tools, metadata check and fixing tools program') parser.add_argument( 'stnlist', type=str, nargs='+', metavar='all|net.stnm', help= "List of networks/stations to process given in [net].[stnm] format or just [stnm] " "(separated by spaces; if [stnm] is not unique in the database, all stations with that " "name will be processed). Use keyword 'all' to process all stations in the database. " "If [net].all is given, all stations from network [net] will be processed. " "Alternatevily, a file with the station list can be provided.") parser.add_argument( '-d', '--date_filter', nargs='+', metavar='date', help='Date range filter for all operations. ' 'Can be specified in wwww-d, yyyy_ddd, yyyy/mm/dd or fyear format') parser.add_argument( '-rinex', '--check_rinex', choices=['fix', 'report'], type=str, nargs=1, help= 'Check the RINEX integrity of the archive-database by verifying that the RINEX files ' 'reported in the rinex table exist in the archive. If argument = "fix" and a RINEX file ' 'does not exist, remove the record. PPP records or gamit_soln are deleted. If argument = ' '"report" then just list the missing files.') parser.add_argument( '-rnx_count', '--rinex_count', action='store_true', help='Count the total number of RINEX files (unique station-days) ' 'per day for a given time interval.') parser.add_argument( '-stnr', '--station_info_rinex', action='store_true', help= 'Check that the receiver serial number in the rinex headers agrees with the station info ' 'receiver serial number.') parser.add_argument( '-stns', '--station_info_solutions', action='store_true', help='Check that the PPP hash values match the station info hash.') parser.add_argument( '-stnp', '--station_info_proposed', metavar='ignore_days', const=0, type=int, nargs='?', help= 'Output a proposed station.info using the RINEX metadata. Optional, specify [ignore_days] ' 'to ignore station.info records <= days.') parser.add_argument( '-stnc', '--station_info_check', action='store_true', help= 'Check the consistency of the station information records in the database. Date range ' 'does not apply. Also, check that the RINEX files fall within a valid station information ' 'record.') parser.add_argument( '-g', '--data_gaps', metavar='ignore_days', const=0, type=int, nargs='?', help= 'Check the RINEX files in the database and look for gaps (missing days). ' 'Optional, [ignore_days] with the smallest gap to display.') parser.add_argument('-gg', '--graphical_gaps', action='store_true', help='Visually output RINEX gaps for stations.') parser.add_argument( '-sc', '--spatial_coherence', choices=['exclude', 'delete', 'noop'], type=str, nargs=1, help= 'Check that the RINEX files correspond to the stations they are linked to using their ' 'PPP coordinate. If keyword [exclude] or [delete], add the PPP solution to the excluded ' 'table or delete the PPP solution. If [noop], then only report but do not ' 'exlude or delete.') parser.add_argument( '-print', '--print_stninfo', choices=['long', 'short'], type=str, nargs=1, help= 'Output the station info to stdout. [long] outputs the full line of the station info. ' '[short] outputs a short version (better for screen visualization).') parser.add_argument( '-r', '--rename', metavar='net.stnm', nargs=1, help= "Takes the data from the station list and renames (merges) it to net.stnm. " "It also changes the rinex filenames in the archive to match those of the new destiny " "station. Only a single station can be given as the origin and destiny. " "Limit the date range using the -d option.") parser.add_argument( '-es', '--exclude_solutions', metavar=('{start_date}', '{end_date}'), nargs=2, help= 'Exclude PPP solutions (by adding them to the excluded table) between {start_date} ' 'and {end_date}') parser.add_argument( '-del', '--delete_rinex', metavar=('{start_date}', '{end_date}', '{completion}'), nargs=3, help='Delete RINEX files (and associated solutions, PPP and GAMIT) ' 'from archive between {start_date} and {end_date} with completion <= {completion}. ' 'Completion ranges form 1.0 to 0.0. Use 1.0 to delete all data. ' 'Operation cannot be undone!') parser.add_argument('-np', '--noparallel', action='store_true', help="Execute command without parallelization.") args = parser.parse_args() cnn = dbConnection.Cnn("gnss_data.cfg") # type: dbConnection.Cnn # create the execution log cnn.insert('executions', script='pyIntegrityCheck.py') Config = pyOptions.ReadOptions( "gnss_data.cfg") # type: pyOptions.ReadOptions stnlist = Utils.process_stnlist(cnn, args.stnlist) JobServer = pyJobServer.JobServer( Config, run_parallel=not args.noparallel) # type: pyJobServer.JobServer ##################################### # date filter dates = [pyDate.Date(year=1980, doy=1), pyDate.Date(year=2100, doy=1)] try: dates = process_date(args.date_filter) except ValueError as e: parser.error(str(e)) ##################################### if args.check_rinex: CheckRinexIntegrity(cnn, Config, stnlist, dates[0], dates[1], args.check_rinex[0], JobServer) ##################################### if args.rinex_count: RinexCount(cnn, stnlist, dates[0], dates[1]) ##################################### if args.station_info_rinex: StnInfoRinexIntegrity(cnn, stnlist, dates[0], dates[1], JobServer) ##################################### if args.station_info_check: StnInfoCheck(cnn, stnlist, Config) ##################################### if args.data_gaps is not None: GetStnGaps(cnn, stnlist, args.data_gaps, dates[0], dates[1]) if args.graphical_gaps: VisualizeGaps(cnn, stnlist, dates[0], dates[1]) ##################################### if args.spatial_coherence is not None: CheckSpatialCoherence(cnn, stnlist, dates[0], dates[1]) ##################################### if args.exclude_solutions is not None: try: dates = process_date(args.exclude_solutions) except ValueError as e: parser.error(str(e)) ExcludeSolutions(cnn, stnlist, dates[0], dates[1]) ##################################### if args.print_stninfo is not None: if args.print_stninfo[0] == 'short': PrintStationInfo(cnn, stnlist, True) elif args.print_stninfo[0] == 'long': PrintStationInfo(cnn, stnlist, False) else: parser.error( 'Argument for print_stninfo has to be either long or short') ##################################### if args.station_info_proposed is not None: for stn in stnlist: stninfo = pyStationInfo.StationInfo(cnn, stn['NetworkCode'], stn['StationCode'], allow_empty=True) sys.stdout.write( stninfo.rinex_based_stninfo(args.station_info_proposed)) ##################################### if args.delete_rinex is not None: try: dates = process_date(args.delete_rinex[0:2]) except ValueError as e: parser.error(str(e)) DeleteRinex(cnn, stnlist, dates[0], dates[1], float(args.delete_rinex[2])) ##################################### if args.rename: if len(stnlist) > 1: parser.error( 'Only a single station should be given for the origin station') if '.' not in args.rename[0]: parser.error('Format for destiny station should be net.stnm') else: DestNetworkCode = args.rename[0].split('.')[0] DestStationCode = args.rename[0].split('.')[1] RenameStation(cnn, stnlist[0]['NetworkCode'], stnlist[0]['StationCode'], DestNetworkCode, DestStationCode, dates[0], dates[1], Config.archive_path) JobServer.close_cluster()
def main(): parser = argparse.ArgumentParser(description='GNSS time series stacker') parser.add_argument( 'project', type=str, nargs=1, metavar='{project name}', help= "Specify the project name used to process the GAMIT solutions in Parallel.GAMIT." ) parser.add_argument( 'stack_name', type=str, nargs=1, metavar='{stack name}', help= "Specify a name for the stack: eg. itrf2014 or posgar07b. This name should be unique " "and cannot be repeated for any other solution project") parser.add_argument( '-max', '--max_iters', nargs=1, type=int, metavar='{max_iter}', help="Specify maximum number of iterations. Default is 4.") parser.add_argument( '-exclude', '--exclude_stations', nargs='+', type=str, metavar='{net.stnm}', help="Manually specify stations to remove from the stacking process.") parser.add_argument( '-use', '--use_stations', nargs='+', type=str, metavar='{net.stnm}', help="Manually specify stations to use for the stacking process.") parser.add_argument( '-dir', '--directory', type=str, help= "Directory to save the resulting PNG files. If not specified, assumed to be the " "production directory") parser.add_argument('-redo', '--redo_stack', action='store_true', help="Delete the stack and redo it from scratch") parser.add_argument('-plot', '--plot_stack_etms', action='store_true', default=False, help="Plot the stack ETMs after computation is done") parser.add_argument( '-constrains', '--external_constrains', nargs='+', help= "File with external constrains parameters (position, velocity and periodic). These may be " "from a parent frame such as ITRF. " "Inheritance will occur with stations on the list whenever a parameter exists. " "Example: -constrains itrf14.txt " "Format is: net.stn x y z epoch vx vy vz sn_1y sn_6m cn_1y cn_6m se_1y se_6m ce_1y ce_6m " "su_1y su_6m cu_1y cu_6m ") parser.add_argument( '-d', '--date_end', nargs=1, metavar='date', help= 'Limit the polyhedrons to the specified date. Can be in wwww-d, yyyy_ddd, yyyy/mm/dd ' 'or fyear format') parser.add_argument('-np', '--noparallel', action='store_true', help="Execute command without parallelization.") args = parser.parse_args() cnn = dbConnection.Cnn("gnss_data.cfg") Config = pyOptions.ReadOptions( "gnss_data.cfg") # type: pyOptions.ReadOptions JobServer = pyJobServer.JobServer( Config, run_parallel=not args.noparallel) # type: pyJobServer.JobServer if args.max_iters: max_iters = int(args.max_iters[0]) else: max_iters = 4 print ' >> Defaulting to 4 iterations' if args.exclude_stations: exclude_stn = args.exclude_stations else: exclude_stn = [] if args.use_stations: use_stn = args.use_stations else: use_stn = [] dates = [Date(year=1980, doy=1), Date(datetime=datetime.now())] if args.date_end is not None: try: dates = process_date( [str(Date(year=1980, doy=1).fyear), args.date_end[0]]) except ValueError as e: parser.error(str(e)) # create folder for plots if args.directory: if not os.path.exists(args.directory): os.mkdir(args.directory) else: if not os.path.exists('production'): os.mkdir('production') args.directory = 'production' # load the ITRF dat file with the periodic space components if args.external_constrains: constrains = load_constrains(args.external_constrains[0]) else: constrains = None # create the stack object stack = pyStack.Stack(cnn, args.project[0], args.stack_name[0], args.redo_stack, end_date=dates[1]) # stack.align_spaces(frame_params) # stack.to_json('alignment.json') # exit() for i in range(max_iters): # create the target polyhedrons based on iteration number (i == 0: PPP) target = calculate_etms(cnn, stack, JobServer, i) qbar = tqdm(total=len(stack), ncols=160, desc=' >> Aligning polyhedrons (%i of %i)' % (i + 1, max_iters)) # work on each polyhedron of the stack for j in range(len(stack)): qbar.update() if not stack[j].aligned: # do not move this if up one level: to speed up the target polyhedron loading process, the target is # set to an empty list when the polyhedron is already aligned if stack[j].date != target[j].date: # raise an error if dates don't agree! raise StandardError( 'Error processing %s: dates don\'t agree (target date %s)' % (stack[j].date.yyyyddd(), target[j].date.yyyyddd())) else: # should only attempt to align a polyhedron that is unaligned # do not set the polyhedron as aligned unless we are in the max iteration step stack[j].align(target[j], True if i == max_iters - 1 else False) # write info to the screen qbar.write( ' -- %s (%3i) %2i it: wrms: %4.1f T %5.1f %5.1f %5.1f ' 'R (%5.1f %5.1f %5.1f)*1e-9' % (stack[j].date.yyyyddd(), stack[j].stations_used, stack[j].iterations, stack[j].wrms * 1000, stack[j].helmert[-3] * 1000, stack[j].helmert[-2] * 1000, stack[j].helmert[-1] * 1000, stack[j].helmert[-6], stack[j].helmert[-5], stack[j].helmert[-4])) stack.transformations.append([poly.info() for poly in stack]) qbar.close() if args.redo_stack: # before removing common modes (or inheriting periodic terms), calculate ETMs with final aligned solutions calculate_etms(cnn, stack, JobServer, iterations=None, create_target=False) # only apply common mode removal if redoing the stack if args.external_constrains: stack.remove_common_modes(constrains) else: stack.remove_common_modes() # here, we also align the stack in velocity and coordinate space stack.align_spaces(constrains) # calculate the etms again, after removing or inheriting parameters calculate_etms(cnn, stack, JobServer, iterations=None, create_target=False) # save the json with the information about the alignment stack.to_json(args.stack_name[0] + '_alignment.json') # save polyhedrons to the database stack.save() if args.plot_stack_etms: qbar = tqdm(total=len(stack.stations), ncols=160) for stn in stack.stations: # plot the ETMs qbar.update() qbar.postfix = '%s.%s' % (stn['NetworkCode'], stn['StationCode']) plot_etm(cnn, stack, stn, args.directory) qbar.close()
def main(): parser = argparse.ArgumentParser( description='Plot ETM for stations in the database') parser.add_argument( 'stnlist', type=str, nargs='+', help= "List of networks/stations to plot given in [net].[stnm] format or just [stnm] " "(separated by spaces; if [stnm] is not unique in the database, all stations with that " "name will be plotted). Use keyword 'all' to plot all stations in all networks. " "If [net].all is given, all stations from network [net] will be plotted" ) parser.add_argument('-nop', '--no_plots', action='store_true', help="Do not produce plots", default=False) parser.add_argument('-nom', '--no_missing_data', action='store_true', help="Do not show missing days", default=False) parser.add_argument('-nm', '--no_model', action='store_true', help="Plot time series without fitting a model") parser.add_argument('-r', '--residuals', action='store_true', help="Plot time series residuals", default=False) parser.add_argument( '-dir', '--directory', type=str, help= "Directory to save the resulting PNG files. If not specified, assumed to be the " "production directory") parser.add_argument( '-json', '--json', type=int, help="Export ETM adjustment to JSON. Append '1' to export time " "series or append '0' to just output the ETM parameters.") parser.add_argument( '-gui', '--interactive', action='store_true', help="Interactive mode: allows to zoom and view the plot interactively" ) parser.add_argument( '-win', '--time_window', nargs='+', metavar='interval', help= 'Date range to window data. Can be specified in yyyy/mm/dd, yyyy.doy or as a single ' 'integer value (N) which shall be interpreted as last epoch-N') parser.add_argument( '-gamit', '--gamit', type=str, nargs=2, metavar='{project} {type}', help= "Plot the GAMIT time series. Specify project and type = \'stack\' to plot the time " "series after stacking or \'gamit\' to just plot the coordinates of the polyhedron" ) args = parser.parse_args() Config = pyOptions.ReadOptions( "gnss_data.cfg") # type: pyOptions.ReadOptions cnn = dbConnection.Cnn('gnss_data.cfg') if len(args.stnlist) == 1 and os.path.isfile(args.stnlist[0]): print ' >> Station list read from ' + args.stnlist[0] stnlist = [line.strip() for line in open(args.stnlist[0], 'r')] stnlist = [{ 'NetworkCode': item.split('.')[0], 'StationCode': item.split('.')[1] } for item in stnlist] else: stnlist = Utils.process_stnlist(cnn, args.stnlist) ##################################### # date filter dates = None if args.time_window is not None: if len(args.time_window) == 1: try: dates = process_date(args.time_window, missing_input=None, allow_days=False) dates = (dates[0].fyear, ) except ValueError: # an integer value dates = float(args.time_window[0]) else: dates = process_date(args.time_window) dates = (dates[0].fyear, dates[1].fyear) if stnlist: # do the thing if args.directory: if not os.path.exists(args.directory): os.mkdir(args.directory) else: if not os.path.exists('production'): os.mkdir('production') args.directory = 'production' for stn in stnlist: try: if args.gamit is None: etm = pyETM.PPPETM(cnn, stn['NetworkCode'], stn['StationCode'], False, args.no_model) else: if args.gamit[1] == 'stack': polyhedrons = cnn.query_float( 'SELECT "X", "Y", "Z", "Year", "DOY" FROM stacks ' 'WHERE "Project" = \'%s\' AND "NetworkCode" = \'%s\' AND ' '"StationCode" = \'%s\' ' 'ORDER BY "Year", "DOY", "NetworkCode", "StationCode"' % (args.gamit[0], stn['NetworkCode'], stn['StationCode'])) soln = pyETM.GamitSoln(cnn, polyhedrons, stn['NetworkCode'], stn['StationCode'], args.gamit[0]) etm = pyETM.GamitETM(cnn, stn['NetworkCode'], stn['StationCode'], False, args.no_model, gamit_soln=soln) # print ' > %5.2f %5.2f %5.2f %i %i' % \ # (etm.factor[0]*1000, etm.factor[1]*1000, etm.factor[2]*1000, etm.soln.t.shape[0], # etm.soln.t.shape[0] - np.sum(np.logical_and(np.logical_and(etm.F[0], etm.F[1]), etm.F[2]))) # print two largest outliers if etm.A is not None: lres = np.sqrt(np.sum(np.square(etm.R), axis=0)) slres = lres[np.argsort(-lres)] print ' >> Two largest residuals:' for i in [0, 1]: print(' %s %6.3f %6.3f %6.3f' % (pyDate.Date(mjd=etm.soln.mjd[ lres == slres[i]]).yyyyddd(), etm.R[0, lres == slres[i]], etm.R[1, lres == slres[i]], etm.R[2, lres == slres[i]])) elif args.gamit[1] == 'gamit': etm = pyETM.GamitETM(cnn, stn['NetworkCode'], stn['StationCode'], False, args.no_model, project=args.gamit[1]) else: parser.error('Invalid option for -gamit switch') etm = None if args.interactive: xfile = None else: if args.gamit is None: xfile = os.path.join( args.directory, '%s.%s_ppp' % (etm.NetworkCode, etm.StationCode)) else: xfile = os.path.join( args.directory, '%s.%s_gamit' % (etm.NetworkCode, etm.StationCode)) # leave pngfile empty to enter interactive mode (GUI) if not args.no_plots: etm.plot(xfile + '.png', t_win=dates, residuals=args.residuals, plot_missing=not args.no_missing_data) if args.json is not None: with open(xfile + '.json', 'w') as f: if args.json != 0: json.dump(etm.todictionary(True), f, indent=4, sort_keys=False) else: json.dump(etm.todictionary(False), f, indent=4, sort_keys=False) print 'Successfully plotted ' + stn['NetworkCode'] + '.' + stn[ 'StationCode'] except pyETM.pyETMException as e: print str(e) except Exception: print 'Error during processing of ' + stn[ 'NetworkCode'] + '.' + stn['StationCode'] print traceback.format_exc() pass
def main(): parser = argparse.ArgumentParser( description='Archive operations Main Program') parser.add_argument( '-date', '--date_range', nargs='+', action=required_length(1, 2), metavar='date_start|date_end', help= "Date range to check given as [date_start] or [date_start] and [date_end]. " "Allowed formats are yyyy.doy or yyyy/mm/dd..") parser.add_argument( '-win', '--window', nargs=1, metavar='days', type=int, help= "Download data from a given time window determined by today - {days}.") try: args = parser.parse_args() Config = pyOptions.ReadOptions('gnss_data.cfg') dates = () now = datetime.now() try: if args.window: # today - ndays d = pyDate.Date(year=now.year, month=now.month, day=now.day) dates = (d - int(args.window[0]), d) else: dates = process_date(args.date_range) except ValueError as e: parser.error(str(e)) if dates[0] < pyDate.Date(gpsWeek=650, gpsWeekDay=0): dates = (pyDate.Date(gpsWeek=650, gpsWeekDay=0), pyDate.Date(year=now.year, month=now.month, day=now.day)) # go through the dates drange = np.arange(dates[0].mjd, dates[1].mjd, 1) pbar = tqdm(desc='%-30s' % ' >> Synchronizing orbit files', total=len(drange), ncols=160) # connect to ftp ftp = ftplib.FTP_TLS(FTP_HOST, FTP_USER, FTP_PASS) ftp.set_pasv(True) ftp.prot_p() def downloadIfMissing(ftp_list, ftp_filename, local_filename, local_dir, desc): mark_path = os.path.join(local_dir, local_filename) if not os.path.isfile(mark_path) and ftp_filename in ftp_list: tqdm.write('%-31s: %s' % (' -- trying to download ' + desc, filename)) down_path = os.path.join(local_dir, ftp_filename) with open(down_path, 'wb') as f: ftp.retrbinary("RETR " + ftp_filename, f.write) return True def get_archive_path(archive, date): return archive.replace('$year', str(date.year)) \ .replace('$doy', str(date.doy).zfill(3)) \ .replace('$gpsweek', str(date.gpsWeek).zfill(4)) \ .replace('$gpswkday', str(date.gpsWeekDay)) for date in (pyDate.Date(mjd=mdj) for mdj in drange): sp3_archive = get_archive_path(Config.sp3_path, date) if not os.path.exists(sp3_archive): os.makedirs(sp3_archive) for repro in ('', '/repro2'): # try both in the repro and / folders folder = "/pub/gps/products/" + date.wwww() + repro try: tqdm.write(' -- Changing folder to ' + folder) ftp.cwd(folder) ftp_list = set(ftp.nlst()) except: # folder not present, skip continue for orbit in Config.sp3types + Config.sp3altrn: for ext in ('.sp3', '.clk', '.erp', '7.erp'): try: filename = orbit + date.wwwwd() + ext + '.Z' downloadIfMissing( ftp_list, filename, filename, sp3_archive, 'EOP' if ext == '7.erp' else ext.upper()) except: pass ###### now the brdc files ######### try: folder = "/pub/gps/data/daily/%s/%s/%sn" % ( date.yyyy(), date.ddd(), date.yyyy()[2:]) tqdm.write(' -- Changing folder to ' + folder) ftp.cwd(folder) ftp_list = set(ftp.nlst()) except: continue brdc_archive = get_archive_path(Config.brdc_path, date) if not os.path.exists(brdc_archive): os.makedirs(brdc_archive) try: filename = 'brdc%s0.%sn' % (str( date.doy).zfill(3), str(date.year)[2:4]) ftp_filename = filename + '.Z' if downloadIfMissing(ftp_list, ftp_filename, filename, brdc_archive, 'BRDC'): # decompress file pyRunWithRetry.RunCommand('gunzip -f ' + ftp_filename, 15).run_shell() except: continue pbar.set_postfix(gpsWeek='%i %i' % (date.gpsWeek, date.gpsWeekDay)) pbar.update() pbar.close() ftp.quit() except argparse.ArgumentTypeError as e: parser.error(str(e))
def __init__(self, configfile): self.options = { 'path': None, 'repository': None, 'parallel': False, 'cups': None, 'node_list': None, 'ip_address': None, 'brdc': None, 'sp3_type_1': None, 'sp3_type_2': None, 'sp3_type_3': None, 'sp3_altr_1': None, 'sp3_altr_2': None, 'sp3_altr_3': None, 'grdtab': None, 'otlgrid': None, 'otlmodel': 'FES2014b', 'ppp_path': None, 'institution': None, 'info': None, 'sp3': None, 'frames': None, 'atx': None, 'height_codes': None, 'ppp_exe': None, 'ppp_remote_local': () } config = ConfigParser.ConfigParser() config.readfp(open(configfile)) # get the archive config for iconfig, val in dict(config.items('archive')).iteritems(): self.options[iconfig] = val # get the otl config for iconfig, val in dict(config.items('otl')).iteritems(): self.options[iconfig] = val # get the ppp config for iconfig, val in dict(config.items('ppp')).iteritems(): self.options[iconfig] = os.path.expandvars(val).replace('//', '/') # frames and dates frames = [item.strip() for item in self.options['frames'].split(',')] atx = [item.strip() for item in self.options['atx'].split(',')] self.Frames = [] for frame, atx in zip(frames, atx): date = process_date(self.options[frame.lower()].split(',')) self.Frames += [{ 'name': frame, 'atx': atx, 'dates': (Date(year=date[0].year, doy=date[0].doy, hour=0, minute=0, second=0), Date(year=date[1].year, doy=date[1].doy, hour=23, minute=59, second=59)) }] self.options['frames'] = self.Frames self.archive_path = self.options['path'] self.sp3_path = self.options['sp3'] self.brdc_path = self.options['brdc'] self.repository = self.options['repository'] self.repository_data_in = os.path.join(self.repository, 'data_in') self.repository_data_in_retry = os.path.join(self.repository, 'data_in_retry') self.repository_data_reject = os.path.join(self.repository, 'data_rejected') self.sp3types = [ self.options['sp3_type_1'], self.options['sp3_type_2'], self.options['sp3_type_3'] ] self.sp3types = [ sp3type for sp3type in self.sp3types if sp3type is not None ] # alternative sp3 types self.sp3altrn = [ self.options['sp3_altr_1'], self.options['sp3_altr_2'], self.options['sp3_altr_3'] ] self.sp3altrn = [ sp3alter for sp3alter in self.sp3altrn if sp3alter is not None ] if self.options['parallel'] == 'True': self.run_parallel = True else: self.run_parallel = False return
def main(): parser = argparse.ArgumentParser(description='Archive operations Main Program') parser.add_argument('-date', '--date_range', nargs='+', action=required_length(1, 2), metavar='date_start|date_end', help="Date range to check given as [date_start] or [date_start] and [date_end]. " "Allowed formats are yyyy.doy or yyyy/mm/dd..") parser.add_argument('-win', '--window', nargs=1, metavar='days', type=int, help="Download data from a given time window determined by today - {days}.") try: args = parser.parse_args() Config = pyOptions.ReadOptions('gnss_data.cfg') dates = [] try: if args.window: # today - ndays d = pyDate.Date(year=datetime.now().year, month=datetime.now().month, day=datetime.now().day) dates = [d-int(args.window[0]), d] else: dates = process_date(args.date_range) except ValueError as e: parser.error(str(e)) if dates[0] < pyDate.Date(gpsWeek=650, gpsWeekDay=0): dates = [pyDate.Date(gpsWeek=650, gpsWeekDay=0), pyDate.Date(year=datetime.now().year, month=datetime.now().month, day=datetime.now().day)] # go through the dates drange = np.arange(dates[0].mjd, dates[1].mjd, 1) pbar = tqdm(desc='%-30s' % ' >> Synchronizing orbit files', total=len(drange), ncols=160) # connect to ftp ftp = ftplib.FTP('198.118.242.40', 'Anonymous', '*****@*****.**') for date in [pyDate.Date(mjd=mdj) for mdj in drange]: sp3_archive = get_archive_path(Config.sp3_path, date) if not os.path.exists(sp3_archive): os.makedirs(sp3_archive) for repro in ['', '/repro2']: # try both in the repro and / folders folder = "/pub/gps/products/" + date.wwww() + repro try: ftp.cwd(folder) except Exception: # folder not present, skip continue tqdm.write(' -- Changing folder to ' + folder) ftp_list = ftp.nlst() for orbit in Config.sp3types + Config.sp3altrn: for ext in ['.sp3.Z', '.clk.Z', '.erp.Z']: filename = orbit + date.wwwwd() + ext if not os.path.isfile(os.path.join(sp3_archive, filename)) and filename in ftp_list: tqdm.write('%-31s: %s' % (' -- trying to download ' + ext.replace('.Z', '').upper(), filename)) try: ftp.retrbinary("RETR " + filename, open(os.path.join(sp3_archive, filename), 'wb').write) except Exception: continue # now the eop file filename = orbit + date.wwww() + '7.erp.Z' if not os.path.isfile(os.path.join(sp3_archive, filename)) and filename in ftp_list: tqdm.write('%-31s: %s' % (' -- trying to download EOP', filename)) try: ftp.retrbinary("RETR " + filename, open(os.path.join(sp3_archive, filename), 'wb').write) except Exception: continue ###### now the brdc files ######### try: folder = "/pub/gps/data/daily/%s/%s/%sn" % (date.yyyy(), date.ddd(), date.yyyy()[2:]) tqdm.write(' -- Changing folder to ' + folder) ftp.cwd(folder) ftp_list = ftp.nlst() except Exception: continue brdc_archive = get_archive_path(Config.brdc_path, date) if not os.path.exists(brdc_archive): os.makedirs(brdc_archive) filename = 'brdc' + str(date.doy).zfill(3) + '0.' + str(date.year)[2:4] + 'n' if not os.path.isfile(os.path.join(brdc_archive, filename)) and filename + '.Z' in ftp_list: tqdm.write('%-31s: %s' % (' -- trying to download BRDC', filename)) try: ftp.retrbinary("RETR " + filename + '.Z', open(os.path.join(brdc_archive, filename + '.Z'), 'wb').write) # decompress file cmd = pyRunWithRetry.RunCommand('gunzip -f ' + os.path.join(brdc_archive, filename + '.Z'), 15) cmd.run_shell() except Exception: continue pbar.set_postfix(gpsWeek='%i %i' % (date.gpsWeek, date.gpsWeekDay)) pbar.update() pbar.close() ftp.quit() except argparse.ArgumentTypeError as e: parser.error(str(e))
def main(): parser = argparse.ArgumentParser( description='Parallel.GAMIT main execution program') parser.add_argument( 'session_cfg', type=str, nargs=1, metavar='session.cfg', help="Filename with the session configuration to run Parallel.GAMIT") parser.add_argument( '-d', '--date', type=str, nargs=2, metavar='{date}', help= "Date range to process. Can be specified in yyyy/mm/dd yyyy_doy wwww-d format" ) parser.add_argument( '-dp', '--date_parser', type=str, nargs=2, metavar='{year} {doys}', help="Parse date using ranges and commas (e.g. 2018 1,3-6). " "Cannot cross year boundaries") parser.add_argument( '-e', '--exclude', type=str, nargs='+', metavar='{station}', help= "List of stations to exclude from this processing (e.g. -e igm1 lpgs vbca)" ) parser.add_argument( '-c', '--check_mode', type=str, nargs='+', metavar='{station}', help= "Check station(s) mode. If station(s) are not present in the GAMIT polyhedron, " "(i.e. the RINEX file(s) were missing at the time of the processing) Parallel.GAMIT will " "add the station to the closest subnetwork(s) and reprocess them. If station(s) were " "present at the time of the processing but failed to process (i.e. they are in the " "missing stations list), these subnetworks will be reprocessed to try to obtain a " "solution. Station list provided in the cfg is ignored in this mode. Therefore, changes " "in the station list will not produce any changes in network configuration. Purge not " "allowed when using this mode. (Syntax: -c igm1 lpgs rms.vbca)") parser.add_argument( '-i', '--ignore_missing', action='store_true', help= "When using check mode or processing existing sessions, ignore missing stations. In other " "words, do not try to reprocess sessions that have missing solutions.") parser.add_argument( '-p', '--purge', action='store_true', default=False, help= "Purge year doys from the database and directory structure and re-run the solution." ) parser.add_argument( '-dry', '--dry_run', action='store_true', help="Generate the directory structures (locally) but do not run GAMIT. " "Output is left in the production directory.") parser.add_argument( '-kml', '--create_kml', action='store_true', help="Create a KML with everything processed in this run.") parser.add_argument('-np', '--noparallel', action='store_true', help="Execute command without parallelization.") args = parser.parse_args() cnn = dbConnection.Cnn('gnss_data.cfg') # type: dbConnection.Cnn dates = None drange = None try: if args.date_parser: year = int(args.date_parser[0]) doys = parseIntSet(args.date_parser[1]) if any(doy for doy in doys if doy < 1): parser.error( 'DOYs cannot start with zero. Please selected a DOY range between 1-365/366' ) if 366 in doys: if year % 4 != 0: parser.error( 'Year ' + str(year) + ' is not a leap year: DOY 366 does not exist.') dates = [pyDate.Date(year=year, doy=i) for i in doys] drange = [dates[0], dates[-1]] else: drange = process_date(args.date, missing_input=None) if not all(drange): parser.error( 'Must specify a start and end date for the processing.') # get the dates to purge dates = [ pyDate.Date(mjd=i) for i in range(drange[0].mjd, drange[1].mjd + 1) ] except ValueError as e: parser.error(str(e)) print( ' >> Reading configuration files and creating project network, please wait...' ) GamitConfig = pyGamitConfig.GamitConfiguration( args.session_cfg[0]) # type: pyGamitConfig.GamitConfiguration print( ' >> Checking GAMIT tables for requested config and year, please wait...' ) JobServer = pyJobServer.JobServer( GamitConfig, check_gamit_tables=(pyDate.Date(year=drange[1].year, doy=drange[1].doy), GamitConfig.gamitopt['eop_type']), run_parallel=not args.noparallel, software_sync=GamitConfig.gamitopt['gamit_remote_local']) # to exclude stations, append them to GamitConfig.NetworkConfig with a - in front exclude = args.exclude if exclude is not None: print(' >> User selected list of stations to exclude:') Utils.print_columns(exclude) GamitConfig.NetworkConfig['stn_list'] += ',-' + ',-'.join(exclude) # initialize stations in the project stations = station_list(cnn, GamitConfig.NetworkConfig['stn_list'].split(','), drange) check_station_list = args.check_mode if check_station_list is not None: print(' >> Check mode. List of stations to check for selected days:') Utils.print_columns(check_station_list) check_stations = station_list(cnn, check_station_list, drange) else: check_stations = StationCollection() dry_run = False if args.dry_run is None else args.dry_run if not dry_run and not len(check_stations): # ignore if calling a dry run # purge solutions if requested purge_solutions(JobServer, args, dates, GamitConfig) elif args.purge: tqdm.write( ' >> Dry run or check mode activated. Cannot purge solutions in these modes.' ) # run the job server sessions = ExecuteGamit(cnn, JobServer, GamitConfig, stations, check_stations, args.ignore_missing, dates, args.dry_run, args.create_kml) # execute globk on doys that had to be divided into subnets if not args.dry_run: ExecuteGlobk(cnn, JobServer, GamitConfig, sessions, dates) # parse the zenith delay outputs ParseZTD(GamitConfig.NetworkConfig.network_id.lower(), dates, sessions, GamitConfig, JobServer) tqdm.write(' >> %s Successful exit from Parallel.GAMIT' % print_datetime())
def main(): global cnn parser = argparse.ArgumentParser(description='Parallel.GAMIT main execution program') parser.add_argument('session_cfg', type=str, nargs=1, metavar='session.cfg', help="Filename with the session configuration to run Parallel.GAMIT") parser.add_argument('-d', '--date', type=str, nargs=2, metavar='{date}', help="Date range to process. Can be specified in yyyy/mm/dd yyyy_doy wwww-d format") parser.add_argument('-dp', '--date_parser', type=str, nargs=2, metavar='{year} {doys}', help="Parse date using ranges and commas (e.g. 2018 1,3-6). " "Cannot cross year boundaries") parser.add_argument('-e', '--exclude', type=str, nargs='+', metavar='station', help="List of stations to exclude from this processing (e.g. -e igm1 lpgs vbca)") parser.add_argument('-p', '--purge', action='store_true', help="Purge year doys from the database and directory structure and re-run the solution.") parser.add_argument('-dry', '--dry_run', action='store_true', help="Generate the directory structures (locally) but do not run GAMIT. " "Output is left in the production directory.") parser.add_argument('-kml', '--generate_kml', action='store_true', help="Generate KML and exit without running GAMIT.") parser.add_argument('-np', '--noparallel', action='store_true', help="Execute command without parallelization.") args = parser.parse_args() dates = None drange = None try: if args.date_parser: year = int(args.date_parser[0]) doys = parseIntSet(args.date_parser[1]) if any([doy for doy in doys if doy < 1]): parser.error('DOYs cannot start with zero. Please selected a DOY range between 1-365/366') if 366 in doys: if year % 4 != 0: parser.error('Year ' + str(year) + ' is not a leap year: DOY 366 does not exist.') dates = [pyDate.Date(year=year, doy=i) for i in doys] drange = [dates[0], dates[-1]] else: drange = process_date(args.date, missing_input=None) if not all(drange): parser.error('Must specify a start and end date for the processing.') # get the dates to purge dates = [pyDate.Date(mjd=i) for i in range(drange[0].mjd, drange[1].mjd + 1)] except ValueError as e: parser.error(str(e)) print(' >> Reading configuration files and creating project network, please wait...') GamitConfig = pyGamitConfig.GamitConfiguration(args.session_cfg[0]) # type: pyGamitConfig.GamitConfiguration print(' >> Checing GAMIT tables for requested config and year, please wait...') JobServer = pyJobServer.JobServer(GamitConfig, check_gamit_tables=(pyDate.Date(year=drange[1].year, doy=drange[1].doy), GamitConfig.gamitopt['eop_type']), run_parallel=not args.noparallel, software_sync=GamitConfig.gamitopt['gamit_remote_local']) cnn = dbConnection.Cnn(GamitConfig.gamitopt['gnss_data']) # type: dbConnection.Cnn # to exclude stations, append them to GamitConfig.NetworkConfig with a - in front exclude = args.exclude if exclude is not None: print(' >> User selected list of stations to exclude:') Utils.print_columns(exclude) GamitConfig.NetworkConfig['stn_list'] += ',-' + ',-'.join(exclude) if args.dry_run is not None: dry_run = args.dry_run else: dry_run = False if not dry_run: # ignore if calling a dry run # purge solutions if requested purge_solutions(JobServer, args, dates, GamitConfig) # initialize stations in the project stations = station_list(cnn, GamitConfig.NetworkConfig, drange) tqdm.write(' >> Creating GAMIT session instances, please wait...') sessions = [] archive = pyArchiveStruct.RinexStruct(cnn) # type: pyArchiveStruct.RinexStruct for date in tqdm(dates, ncols=80): # make the dir for these sessions # this avoids a racing condition when starting each process pwd = GamitConfig.gamitopt['solutions_dir'].rstrip('/') + '/' + date.yyyy() + '/' + date.ddd() if not os.path.exists(pwd): os.makedirs(pwd) net_object = Network(cnn, archive, GamitConfig, stations, date) sessions += net_object.sessions if args.generate_kml: # generate a KML of the sessions generate_kml(dates, sessions, GamitConfig) exit() # print a summary of the current project (NOT VERY USEFUL AFTER ALL) # print_summary(stations, sessions, drange) # run the job server ExecuteGamit(sessions, JobServer, dry_run) # execute globk on doys that had to be divided into subnets if not args.dry_run: ExecuteGlobk(GamitConfig, sessions, dates) # parse the zenith delay outputs ParseZTD(GamitConfig.NetworkConfig.network_id, sessions, GamitConfig) print(' >> Done processing and parsing information. Successful exit from Parallel.GAMIT')
def main(): global wrms_n, wrms_e, wrms_u, project parser = argparse.ArgumentParser( description='GNSS daily repetitivities analysis (DRA)') parser.add_argument( 'project', type=str, nargs=1, metavar='{project name}', help= "Specify the project name used to process the GAMIT solutions in Parallel.GAMIT." ) parser.add_argument( '-d', '--date_filter', nargs='+', metavar='date', help= 'Date range filter. Can be specified in yyyy/mm/dd yyyy_doy wwww-d format' ) parser.add_argument( '-w', '--plot_window', nargs='+', metavar='date', help= 'Date window range to plot. Can be specified in yyyy/mm/dd yyyy_doy wwww-d format' ) parser.add_argument('-hist', '--histogram', action='store_true', help="Plot a histogram of the daily repetitivities") parser.add_argument( '-v', '--verbose', action='store_true', help= "Provide additional information during the alignment process (for debugging purposes)" ) parser.add_argument('-np', '--noparallel', action='store_true', help="Execute command without parallelization.") args = parser.parse_args() cnn = dbConnection.Cnn("gnss_data.cfg") project = args.project[0] dates = [pyDate.Date(year=1980, doy=1), pyDate.Date(year=2100, doy=1)] Config = pyOptions.ReadOptions( "gnss_data.cfg") # type: pyOptions.ReadOptions JobServer = pyJobServer.JobServer( Config, run_parallel=not args.noparallel) # type: pyJobServer.JobServer try: dates = process_date(args.date_filter) except ValueError as e: parser.error(str(e)) pdates = None if args.plot_window is not None: if len(args.plot_window) == 1: try: pdates = process_date(args.plot_window, missing_input=None, allow_days=False) pdates = (pdates[0].fyear, ) except ValueError: # an integer value pdates = float(args.plot_window[0]) else: pdates = process_date(args.plot_window) pdates = (pdates[0].fyear, pdates[1].fyear) # create folder for plots path_plot = project + '_dra' if not os.path.isdir(path_plot): os.makedirs(path_plot) ######################################## # load polyhedrons # create the DRA object dra = DRA(cnn, args.project[0], dates[0], dates[1], args.verbose) dra.stack_dra() dra.to_json(project + '_dra.json') missing_doys = [] tqdm.write( ' >> Daily repetitivity analysis done. DOYs with wrms > 8 mm are shown below:' ) for i, d in enumerate(dra): if d.wrms is not None: if d.wrms > 0.008: tqdm.write( ' -- %s (%04i) %2i it wrms: %4.1f D-W: %5.3f IQR: %4.1f' % (d.date.yyyyddd(), d.stations_used, d.iterations, d.wrms * 1000, d.down_frac, d.iqr * 1000)) qbar = tqdm(total=len(dra.stations), desc=' >> Computing DRAs', ncols=160, disable=None) modules = ('pyETM', 'dbConnection', 'traceback', 'io', 'numpy') JobServer.create_cluster(compute_dra, progress_bar=qbar, callback=callback_handler, modules=modules) # plot each DRA for stn in dra.stations: NetworkCode = stn['NetworkCode'] StationCode = stn['StationCode'] ts = dra.get_station(NetworkCode, StationCode) JobServer.submit(ts, NetworkCode, StationCode, pdates, project, args.histogram) JobServer.wait() qbar.close() JobServer.close_cluster() wrms_n = np.array(wrms_n) wrms_e = np.array(wrms_e) wrms_u = np.array(wrms_u) # plot the WRM of the DRA stack and number of stations f, axis = plt.subplots(nrows=3, ncols=2, figsize=(15, 10)) # type: plt.subplots # WRMS ax = axis[0][0] ax.plot([t['fyear'] for t in dra.transformations[0]], [t['wrms'] * 1000 for t in dra.transformations[0]], 'ob', markersize=2) ax.set_ylabel('WRMS [mm]') ax.grid(True) ax.set_ylim(0, 10) # station count ax = axis[1][0] ax.plot([t['fyear'] for t in dra.transformations[0]], [t['stations_used'] for t in dra.transformations[0]], 'ob', markersize=2) ax.set_ylabel('Station count') ax.grid(True) # d-w fraction ax = axis[2][0] ax.plot([t['fyear'] for t in dra.transformations[0]], [t['downweighted_fraction'] for t in dra.transformations[0]], 'ob', markersize=2) ax.set_ylabel('DW fraction') ax.grid(True) ax = axis[0][1] ax.hist(wrms_n[wrms_n <= 8], 40, alpha=0.75, facecolor='blue') ax.grid(True) ax.set_ylabel('# stations') ax.set_xlabel('WRMS misfit N [mm]') ax.set_title('Daily repetitivities NEU') ax = axis[1][1] ax.hist(wrms_e[wrms_e <= 8], 40, alpha=0.75, facecolor='blue') ax.grid(True) ax.set_xlim(0, 8) ax.set_ylabel('# stations') ax.set_xlabel('WRMS misfit E [mm]') ax = axis[2][1] ax.hist(wrms_u[wrms_u <= 10], 40, alpha=0.75, facecolor='blue') ax.grid(True) ax.set_xlim(0, 10) ax.set_ylabel('# stations') ax.set_xlabel('WRMS misfit U [mm]') f.suptitle('Daily repetitivity analysis for project %s\n' 'Solutions with WRMS > 10 mm are not shown' % project, fontsize=12, family='monospace') plt.savefig(project + '_dra.png') plt.close() ax.set_xlim(0, 8)