def main(): parser = argparse.ArgumentParser(description='GNSS time series stacker') parser.add_argument('project', type=str, nargs=1, metavar='{project name}', help="Specify the project name used to process the GAMIT solutions in Parallel.GAMIT.") parser.add_argument('-np', '--noparallel', action='store_true', help="Execute command without parallelization.") args = parser.parse_args() cnn = dbConnection.Cnn("gnss_data.cfg") Config = pyOptions.ReadOptions("gnss_data.cfg") # type: pyOptions.ReadOptions if not args.noparallel: JobServer = pyJobServer.JobServer(Config, run_node_test=False) # type: pyJobServer.JobServer else: JobServer = None Config.run_parallel = False # create the execution log # load polyhedrons project = Project(cnn, args.project[0]) # plot initial state tqdm.write(' -- Plotting initial ETMs (unaligned)...') #for etm in tqdm(project.etms, ncols=160): # etm.plot(pngfile=args.project[0] + '/' + etm.NetworkCode + '.' + etm.StationCode + '_0.png', residuals=True) project.align_stack() tqdm.write(' -- Plotting intermediate step ETMs (aligned)...')
def main(): Config = pyOptions.ReadOptions('gnss_data.cfg') JobServer = pyJobServer.JobServer(Config) # type: pyJobServer.JobServer cnn = dbConnection.Cnn('gnss_data.cfg') archive = pyArchiveStruct.RinexStruct(cnn) rinex = cnn.query_float( 'SELECT * FROM rinex WHERE "ObservationYear" <= 1995 ORDER BY "NetworkCode", ' '"StationCode", "ObservationYear", "ObservationDOY"', as_dict=True) pbar = tqdm(desc='%-30s' % ' >> Processing rinex files', total=len(rinex), ncols=160) modules = ('os', 'pyRinex') callback = [] for rnx in rinex: filename = archive.build_rinex_path(rnx['NetworkCode'], rnx['StationCode'], rnx['ObservationYear'], rnx['ObservationDOY'], filename=rnx['Filename']) arguments = (rnx['NetworkCode'], rnx['StationCode'], Config.archive_path, filename) JobServer.SubmitJob(check_rinex, arguments, (), modules, callback, callback_class(pbar), 'callbackfunc') if JobServer.process_callback: # handle any output messages during this batch callback = output_handle(callback) JobServer.process_callback = False tqdm.write(' >> waiting for jobs to finish...') JobServer.job_server.wait() tqdm.write(' >> Done.') # process the errors and the new stations output_handle(callback) pbar.close()
def main(): parser = argparse.ArgumentParser(description='Script to synchronize AWS with OSU\'s archive database') parser.add_argument('date', type=str, nargs=1, help="Check the sync state for this given date. Format can be fyear or yyyy_ddd.") parser.add_argument('-mark', '--mark_uploaded', nargs='+', type=str, help="Pass net.stnm to mark these files as transferred to the AWS", metavar='{net.stnm}') parser.add_argument('-pull', '--pull_rinex', action='store_true', help="Get all the unsynchronized RINEX files in the local dir") parser.add_argument('-np', '--noparallel', action='store_true', help="Execute command without parallelization.") args = parser.parse_args() Config = pyOptions.ReadOptions("gnss_data.cfg") # type: pyOptions.ReadOptions cnn = dbConnection.Cnn('gnss_data.cfg') # before attempting anything, check aliases!! print ' >> Checking GAMIT aliases' check_aliases(cnn) # initialize the PP job server if not args.noparallel: JobServer = pyJobServer.JobServer(Config, 1500) # type: pyJobServer.JobServer else: JobServer = None Config.run_parallel = False dd = args.date[0] if '_' in dd: date = pyDate.Date(year=int(dd.split('_')[0]), doy=int(dd.split('_')[1])) elif dd == 'all': # run all dates (1994 to 2018) ts = range(pyDate.Date(year=2004, doy=20).mjd, pyDate.Date(year=2018, doy=87).mjd, 1) ts = [pyDate.Date(mjd=tts) for tts in ts] for date in ts: print ' >> Processing ' + str(date) pull_rinex(cnn, date, Config, JobServer) return else: date = pyDate.Date(fyear=float(dd)) if args.pull_rinex: pull_rinex(cnn, date, Config, JobServer) if args.mark_uploaded is not None: print 'Processing %i for day %s' % (len(args.mark_uploaded), date.yyyyddd()) # mark the list of stations as transferred to the AWS mark_uploaded(cnn, date, args.mark_uploaded)
def main(): parser = argparse.ArgumentParser( description= 'Database integrity tools, metadata check and fixing tools program') parser.add_argument( 'stnlist', type=str, nargs='+', metavar='all|net.stnm', help= "List of networks/stations to process given in [net].[stnm] format or just [stnm] " "(separated by spaces; if [stnm] is not unique in the database, all stations with that " "name will be processed). Use keyword 'all' to process all stations in the database. " "If [net].all is given, all stations from network [net] will be processed. " "Alternatevily, a file with the station list can be provided.") parser.add_argument( '-d', '--date_filter', nargs='+', metavar='date', help='Date range filter for all operations. ' 'Can be specified in wwww-d, yyyy_ddd, yyyy/mm/dd or fyear format') parser.add_argument( '-rinex', '--check_rinex', choices=['fix', 'report'], type=str, nargs=1, help= 'Check the RINEX integrity of the archive-database by verifying that the RINEX files ' 'reported in the rinex table exist in the archive. If argument = "fix" and a RINEX file ' 'does not exist, remove the record. PPP records or gamit_soln are deleted. If argument = ' '"report" then just list the missing files.') parser.add_argument( '-rnx_count', '--rinex_count', action='store_true', help='Count the total number of RINEX files (unique station-days) ' 'per day for a given time interval.') parser.add_argument( '-stnr', '--station_info_rinex', action='store_true', help= 'Check that the receiver serial number in the rinex headers agrees with the station info ' 'receiver serial number.') parser.add_argument( '-stns', '--station_info_solutions', action='store_true', help='Check that the PPP hash values match the station info hash.') parser.add_argument( '-stnp', '--station_info_proposed', metavar='ignore_days', const=0, type=int, nargs='?', help= 'Output a proposed station.info using the RINEX metadata. Optional, specify [ignore_days] ' 'to ignore station.info records <= days.') parser.add_argument( '-stnc', '--station_info_check', action='store_true', help= 'Check the consistency of the station information records in the database. Date range ' 'does not apply. Also, check that the RINEX files fall within a valid station information ' 'record.') parser.add_argument( '-g', '--data_gaps', metavar='ignore_days', const=0, type=int, nargs='?', help= 'Check the RINEX files in the database and look for gaps (missing days). ' 'Optional, [ignore_days] with the smallest gap to display.') parser.add_argument('-gg', '--graphical_gaps', action='store_true', help='Visually output RINEX gaps for stations.') parser.add_argument( '-sc', '--spatial_coherence', choices=['exclude', 'delete', 'noop'], type=str, nargs=1, help= 'Check that the RINEX files correspond to the stations they are linked to using their ' 'PPP coordinate. If keyword [exclude] or [delete], add the PPP solution to the excluded ' 'table or delete the PPP solution. If [noop], then only report but do not ' 'exlude or delete.') parser.add_argument( '-print', '--print_stninfo', choices=['long', 'short'], type=str, nargs=1, help= 'Output the station info to stdout. [long] outputs the full line of the station info. ' '[short] outputs a short version (better for screen visualization).') parser.add_argument( '-r', '--rename', metavar='net.stnm', nargs=1, help= "Takes the data from the station list and renames (merges) it to net.stnm. " "It also changes the rinex filenames in the archive to match those of the new destiny " "station. Only a single station can be given as the origin and destiny. " "Limit the date range using the -d option.") parser.add_argument( '-es', '--exclude_solutions', metavar=('{start_date}', '{end_date}'), nargs=2, help= 'Exclude PPP solutions (by adding them to the excluded table) between {start_date} ' 'and {end_date}') parser.add_argument( '-del', '--delete_rinex', metavar=('{start_date}', '{end_date}', '{completion}'), nargs=3, help='Delete RINEX files (and associated solutions, PPP and GAMIT) ' 'from archive between {start_date} and {end_date} with completion <= {completion}. ' 'Completion ranges form 1.0 to 0.0. Use 1.0 to delete all data. ' 'Operation cannot be undone!') parser.add_argument('-np', '--noparallel', action='store_true', help="Execute command without parallelization.") args = parser.parse_args() cnn = dbConnection.Cnn("gnss_data.cfg") # type: dbConnection.Cnn # create the execution log cnn.insert('executions', script='pyIntegrityCheck.py') Config = pyOptions.ReadOptions( "gnss_data.cfg") # type: pyOptions.ReadOptions stnlist = Utils.process_stnlist(cnn, args.stnlist) JobServer = pyJobServer.JobServer( Config, run_parallel=not args.noparallel) # type: pyJobServer.JobServer ##################################### # date filter dates = [pyDate.Date(year=1980, doy=1), pyDate.Date(year=2100, doy=1)] try: dates = process_date(args.date_filter) except ValueError as e: parser.error(str(e)) ##################################### if args.check_rinex: CheckRinexIntegrity(cnn, Config, stnlist, dates[0], dates[1], args.check_rinex[0], JobServer) ##################################### if args.rinex_count: RinexCount(cnn, stnlist, dates[0], dates[1]) ##################################### if args.station_info_rinex: StnInfoRinexIntegrity(cnn, stnlist, dates[0], dates[1], JobServer) ##################################### if args.station_info_check: StnInfoCheck(cnn, stnlist, Config) ##################################### if args.data_gaps is not None: GetStnGaps(cnn, stnlist, args.data_gaps, dates[0], dates[1]) if args.graphical_gaps: VisualizeGaps(cnn, stnlist, dates[0], dates[1]) ##################################### if args.spatial_coherence is not None: CheckSpatialCoherence(cnn, stnlist, dates[0], dates[1]) ##################################### if args.exclude_solutions is not None: try: dates = process_date(args.exclude_solutions) except ValueError as e: parser.error(str(e)) ExcludeSolutions(cnn, stnlist, dates[0], dates[1]) ##################################### if args.print_stninfo is not None: if args.print_stninfo[0] == 'short': PrintStationInfo(cnn, stnlist, True) elif args.print_stninfo[0] == 'long': PrintStationInfo(cnn, stnlist, False) else: parser.error( 'Argument for print_stninfo has to be either long or short') ##################################### if args.station_info_proposed is not None: for stn in stnlist: stninfo = pyStationInfo.StationInfo(cnn, stn['NetworkCode'], stn['StationCode'], allow_empty=True) sys.stdout.write( stninfo.rinex_based_stninfo(args.station_info_proposed)) ##################################### if args.delete_rinex is not None: try: dates = process_date(args.delete_rinex[0:2]) except ValueError as e: parser.error(str(e)) DeleteRinex(cnn, stnlist, dates[0], dates[1], float(args.delete_rinex[2])) ##################################### if args.rename: if len(stnlist) > 1: parser.error( 'Only a single station should be given for the origin station') if '.' not in args.rename[0]: parser.error('Format for destiny station should be net.stnm') else: DestNetworkCode = args.rename[0].split('.')[0] DestStationCode = args.rename[0].split('.')[1] RenameStation(cnn, stnlist[0]['NetworkCode'], stnlist[0]['StationCode'], DestNetworkCode, DestStationCode, dates[0], dates[1], Config.archive_path) JobServer.close_cluster()
def main(): parser = argparse.ArgumentParser(description='GNSS time series stacker') parser.add_argument( 'project', type=str, nargs=1, metavar='{project name}', help= "Specify the project name used to process the GAMIT solutions in Parallel.GAMIT." ) parser.add_argument( 'stack_name', type=str, nargs=1, metavar='{stack name}', help= "Specify a name for the stack: eg. itrf2014 or posgar07b. This name should be unique " "and cannot be repeated for any other solution project") parser.add_argument( '-max', '--max_iters', nargs=1, type=int, metavar='{max_iter}', help="Specify maximum number of iterations. Default is 4.") parser.add_argument( '-exclude', '--exclude_stations', nargs='+', type=str, metavar='{net.stnm}', help="Manually specify stations to remove from the stacking process.") parser.add_argument( '-use', '--use_stations', nargs='+', type=str, metavar='{net.stnm}', help="Manually specify stations to use for the stacking process.") parser.add_argument( '-dir', '--directory', type=str, help= "Directory to save the resulting PNG files. If not specified, assumed to be the " "production directory") parser.add_argument('-redo', '--redo_stack', action='store_true', help="Delete the stack and redo it from scratch") parser.add_argument('-plot', '--plot_stack_etms', action='store_true', default=False, help="Plot the stack ETMs after computation is done") parser.add_argument( '-constrains', '--external_constrains', nargs='+', help= "File with external constrains parameters (position, velocity and periodic). These may be " "from a parent frame such as ITRF. " "Inheritance will occur with stations on the list whenever a parameter exists. " "Example: -constrains itrf14.txt " "Format is: net.stn x y z epoch vx vy vz sn_1y sn_6m cn_1y cn_6m se_1y se_6m ce_1y ce_6m " "su_1y su_6m cu_1y cu_6m ") parser.add_argument( '-d', '--date_end', nargs=1, metavar='date', help= 'Limit the polyhedrons to the specified date. Can be in wwww-d, yyyy_ddd, yyyy/mm/dd ' 'or fyear format') parser.add_argument('-np', '--noparallel', action='store_true', help="Execute command without parallelization.") args = parser.parse_args() cnn = dbConnection.Cnn("gnss_data.cfg") Config = pyOptions.ReadOptions( "gnss_data.cfg") # type: pyOptions.ReadOptions JobServer = pyJobServer.JobServer( Config, run_parallel=not args.noparallel) # type: pyJobServer.JobServer if args.max_iters: max_iters = int(args.max_iters[0]) else: max_iters = 4 print ' >> Defaulting to 4 iterations' if args.exclude_stations: exclude_stn = args.exclude_stations else: exclude_stn = [] if args.use_stations: use_stn = args.use_stations else: use_stn = [] dates = [Date(year=1980, doy=1), Date(datetime=datetime.now())] if args.date_end is not None: try: dates = process_date( [str(Date(year=1980, doy=1).fyear), args.date_end[0]]) except ValueError as e: parser.error(str(e)) # create folder for plots if args.directory: if not os.path.exists(args.directory): os.mkdir(args.directory) else: if not os.path.exists('production'): os.mkdir('production') args.directory = 'production' # load the ITRF dat file with the periodic space components if args.external_constrains: constrains = load_constrains(args.external_constrains[0]) else: constrains = None # create the stack object stack = pyStack.Stack(cnn, args.project[0], args.stack_name[0], args.redo_stack, end_date=dates[1]) # stack.align_spaces(frame_params) # stack.to_json('alignment.json') # exit() for i in range(max_iters): # create the target polyhedrons based on iteration number (i == 0: PPP) target = calculate_etms(cnn, stack, JobServer, i) qbar = tqdm(total=len(stack), ncols=160, desc=' >> Aligning polyhedrons (%i of %i)' % (i + 1, max_iters)) # work on each polyhedron of the stack for j in range(len(stack)): qbar.update() if not stack[j].aligned: # do not move this if up one level: to speed up the target polyhedron loading process, the target is # set to an empty list when the polyhedron is already aligned if stack[j].date != target[j].date: # raise an error if dates don't agree! raise StandardError( 'Error processing %s: dates don\'t agree (target date %s)' % (stack[j].date.yyyyddd(), target[j].date.yyyyddd())) else: # should only attempt to align a polyhedron that is unaligned # do not set the polyhedron as aligned unless we are in the max iteration step stack[j].align(target[j], True if i == max_iters - 1 else False) # write info to the screen qbar.write( ' -- %s (%3i) %2i it: wrms: %4.1f T %5.1f %5.1f %5.1f ' 'R (%5.1f %5.1f %5.1f)*1e-9' % (stack[j].date.yyyyddd(), stack[j].stations_used, stack[j].iterations, stack[j].wrms * 1000, stack[j].helmert[-3] * 1000, stack[j].helmert[-2] * 1000, stack[j].helmert[-1] * 1000, stack[j].helmert[-6], stack[j].helmert[-5], stack[j].helmert[-4])) stack.transformations.append([poly.info() for poly in stack]) qbar.close() if args.redo_stack: # before removing common modes (or inheriting periodic terms), calculate ETMs with final aligned solutions calculate_etms(cnn, stack, JobServer, iterations=None, create_target=False) # only apply common mode removal if redoing the stack if args.external_constrains: stack.remove_common_modes(constrains) else: stack.remove_common_modes() # here, we also align the stack in velocity and coordinate space stack.align_spaces(constrains) # calculate the etms again, after removing or inheriting parameters calculate_etms(cnn, stack, JobServer, iterations=None, create_target=False) # save the json with the information about the alignment stack.to_json(args.stack_name[0] + '_alignment.json') # save polyhedrons to the database stack.save() if args.plot_stack_etms: qbar = tqdm(total=len(stack.stations), ncols=160) for stn in stack.stations: # plot the ETMs qbar.update() qbar.postfix = '%s.%s' % (stn['NetworkCode'], stn['StationCode']) plot_etm(cnn, stack, stn, args.directory) qbar.close()
# function to print any error that are encountered during parallel execution for msg in messages: if msg: file_append( 'errors_amend.log', 'ON ' + datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') + ' an unhandled error occurred:\n' + msg + '\n' + 'END OF ERROR =================== \n\n') return [] cnn = dbConnection.Cnn('gnss_data.cfg') options = pyOptions.ReadOptions('gnss_data.cfg') JobServer = pyJobServer.JobServer(options) archive = pyArchiveStruct.RinexStruct(cnn) for table in ['rinex']: print(" >> Processing " + table) tbl = cnn.query('SELECT * FROM ' + table + ' WHERE "Completion" is null') rnx = tbl.dictresult() callback = [] pbar = tqdm(total=len(rnx), ncols=80) depfuncs = (verify_rinex_date_multiday, ) modules = ('pyRinex', 'dbConnection', 'traceback', 'platform', 'pyDate',
def main(): # put connection and config in global variable to use inside callback_handle global cnn global repository_data_in # bind to the repository directory parser = argparse.ArgumentParser( description='Archive operations Main Program') parser.add_argument( '-purge', '--purge_locks', action='store_true', help= "Delete any network starting with '?' from the stations table and purge the contents of " "the locks table, deleting the associated files from data_in.") parser.add_argument('-np', '--noparallel', action='store_true', help="Execute command without parallelization.") args = parser.parse_args() Config = pyOptions.ReadOptions('gnss_data.cfg') repository_data_in = Config.repository_data_in if not os.path.isdir(Config.repository): print "the provided repository path in gnss_data.cfg is not a folder" exit() JobServer = pyJobServer.JobServer(Config, run_parallel=not args.noparallel, software_sync=[ Config.options['ppp_remote_local'] ]) # type: pyJobServer.JobServer cnn = dbConnection.Cnn('gnss_data.cfg') # create the execution log cnn.insert('executions', script='ArchiveService.py') # set the data_xx directories data_in = os.path.join(Config.repository, 'data_in') data_in_retry = os.path.join(Config.repository, 'data_in_retry') data_reject = os.path.join(Config.repository, 'data_rejected') # if if the subdirs exist if not os.path.isdir(data_in): os.makedirs(data_in) if not os.path.isdir(data_in_retry): os.makedirs(data_in_retry) if not os.path.isdir(data_reject): os.makedirs(data_reject) # delete any locks with a NetworkCode != '?%' cnn.query('delete from locks where "NetworkCode" not like \'?%\'') # get the locks to avoid reprocessing files that had no metadata in the database locks = cnn.query('SELECT * FROM locks') locks = locks.dictresult() if args.purge_locks: # first, delete all associated files for lock in tqdm(locks, ncols=160, unit='crz', desc='%-30s' % ' >> Purging locks', disable=None): try: os.remove( os.path.join(Config.repository_data_in, lock['filename'])) except Exception: sys.exc_clear() # purge the contents of stations. This will automatically purge the locks table cnn.query('delete from stations where "NetworkCode" like \'?%\'') # purge the networks cnn.query('delete from networks where "NetworkCode" like \'?%\'') # purge the locks already taken care of (just in case) cnn.query('delete from locks where "NetworkCode" not like \'?%\'') # get the locks to avoid reprocessing files that had no metadata in the database locks = cnn.query('SELECT * FROM locks') locks = locks.dictresult() # look for data in the data_in_retry and move it to data_in archive = pyArchiveStruct.RinexStruct(cnn) pbar = tqdm(desc='%-30s' % ' >> Scanning data_in_retry', ncols=160, unit='crz', disable=None) rfiles, paths, _ = archive.scan_archive_struct(data_in_retry, pbar) pbar.close() pbar = tqdm(desc='%-30s' % ' -- Moving files to data_in', total=len(rfiles), ncols=160, unit='crz', disable=None) for rfile, path in zip(rfiles, paths): dest_file = os.path.join(data_in, rfile) # move the file into the folder Utils.move(path, dest_file) pbar.set_postfix(crinez=rfile) pbar.update() # remove folder from data_in_retry (also removes the log file) try: # remove the log file that accompanies this Z file os.remove(path.replace('d.Z', '.log')) except Exception: sys.exc_clear() pbar.close() tqdm.write(' -- Cleaning data_in_retry') remove_empty_folders(data_in_retry) # take a break to allow the FS to finish the task time.sleep(5) files_path = [] files_list = [] pbar = tqdm(desc='%-30s' % ' >> Repository crinez scan', ncols=160, disable=None) rpaths, _, files = archive.scan_archive_struct(data_in, pbar) pbar.close() pbar = tqdm(desc='%-30s' % ' -- Checking the locks table', total=len(files), ncols=130, unit='crz', disable=None) for file, path in zip(files, rpaths): pbar.set_postfix(crinez=file) pbar.update() if path not in [lock['filename'] for lock in locks]: files_path.append(path) files_list.append(file) pbar.close() tqdm.write(" -- Found %i files in the lock list..." % (len(locks))) tqdm.write( " -- Found %i files (matching format [stnm][doy][s].[yy]d.Z) to process..." % (len(files_list))) pbar = tqdm(desc='%-30s' % ' >> Processing repository', total=len(files_path), ncols=160, unit='crz', disable=None) # dependency functions depfuncs = (check_rinex_timespan_int, write_error, error_handle, insert_data, verify_rinex_multiday) # import modules modules = ('pyRinex', 'pyArchiveStruct', 'pyOTL', 'pyPPP', 'pyStationInfo', 'dbConnection', 'Utils', 'os', 'uuid', 'datetime', 'pyDate', 'numpy', 'traceback', 'platform', 'pyBrdc', 'pyProducts', 'pyOptions', 'pyEvents') JobServer.create_cluster(process_crinex_file, depfuncs, callback_handle, pbar, modules=modules) for file_to_process, sfile in zip(files_path, files_list): JobServer.submit(file_to_process, sfile, data_reject, data_in_retry) JobServer.wait() pbar.close() JobServer.close_cluster() print_archive_service_summary() # iterate to delete empty folders remove_empty_folders(data_in)
def main(): parser = argparse.ArgumentParser( description='Parallel.GAMIT main execution program') parser.add_argument( 'session_cfg', type=str, nargs=1, metavar='session.cfg', help="Filename with the session configuration to run Parallel.GAMIT") parser.add_argument( '-d', '--date', type=str, nargs=2, metavar='{date}', help= "Date range to process. Can be specified in yyyy/mm/dd yyyy_doy wwww-d format" ) parser.add_argument( '-dp', '--date_parser', type=str, nargs=2, metavar='{year} {doys}', help="Parse date using ranges and commas (e.g. 2018 1,3-6). " "Cannot cross year boundaries") parser.add_argument( '-e', '--exclude', type=str, nargs='+', metavar='{station}', help= "List of stations to exclude from this processing (e.g. -e igm1 lpgs vbca)" ) parser.add_argument( '-c', '--check_mode', type=str, nargs='+', metavar='{station}', help= "Check station(s) mode. If station(s) are not present in the GAMIT polyhedron, " "(i.e. the RINEX file(s) were missing at the time of the processing) Parallel.GAMIT will " "add the station to the closest subnetwork(s) and reprocess them. If station(s) were " "present at the time of the processing but failed to process (i.e. they are in the " "missing stations list), these subnetworks will be reprocessed to try to obtain a " "solution. Station list provided in the cfg is ignored in this mode. Therefore, changes " "in the station list will not produce any changes in network configuration. Purge not " "allowed when using this mode. (Syntax: -c igm1 lpgs rms.vbca)") parser.add_argument( '-i', '--ignore_missing', action='store_true', help= "When using check mode or processing existing sessions, ignore missing stations. In other " "words, do not try to reprocess sessions that have missing solutions.") parser.add_argument( '-p', '--purge', action='store_true', default=False, help= "Purge year doys from the database and directory structure and re-run the solution." ) parser.add_argument( '-dry', '--dry_run', action='store_true', help="Generate the directory structures (locally) but do not run GAMIT. " "Output is left in the production directory.") parser.add_argument( '-kml', '--create_kml', action='store_true', help="Create a KML with everything processed in this run.") parser.add_argument('-np', '--noparallel', action='store_true', help="Execute command without parallelization.") args = parser.parse_args() cnn = dbConnection.Cnn('gnss_data.cfg') # type: dbConnection.Cnn dates = None drange = None try: if args.date_parser: year = int(args.date_parser[0]) doys = parseIntSet(args.date_parser[1]) if any(doy for doy in doys if doy < 1): parser.error( 'DOYs cannot start with zero. Please selected a DOY range between 1-365/366' ) if 366 in doys: if year % 4 != 0: parser.error( 'Year ' + str(year) + ' is not a leap year: DOY 366 does not exist.') dates = [pyDate.Date(year=year, doy=i) for i in doys] drange = [dates[0], dates[-1]] else: drange = process_date(args.date, missing_input=None) if not all(drange): parser.error( 'Must specify a start and end date for the processing.') # get the dates to purge dates = [ pyDate.Date(mjd=i) for i in range(drange[0].mjd, drange[1].mjd + 1) ] except ValueError as e: parser.error(str(e)) print( ' >> Reading configuration files and creating project network, please wait...' ) GamitConfig = pyGamitConfig.GamitConfiguration( args.session_cfg[0]) # type: pyGamitConfig.GamitConfiguration print( ' >> Checking GAMIT tables for requested config and year, please wait...' ) JobServer = pyJobServer.JobServer( GamitConfig, check_gamit_tables=(pyDate.Date(year=drange[1].year, doy=drange[1].doy), GamitConfig.gamitopt['eop_type']), run_parallel=not args.noparallel, software_sync=GamitConfig.gamitopt['gamit_remote_local']) # to exclude stations, append them to GamitConfig.NetworkConfig with a - in front exclude = args.exclude if exclude is not None: print(' >> User selected list of stations to exclude:') Utils.print_columns(exclude) GamitConfig.NetworkConfig['stn_list'] += ',-' + ',-'.join(exclude) # initialize stations in the project stations = station_list(cnn, GamitConfig.NetworkConfig['stn_list'].split(','), drange) check_station_list = args.check_mode if check_station_list is not None: print(' >> Check mode. List of stations to check for selected days:') Utils.print_columns(check_station_list) check_stations = station_list(cnn, check_station_list, drange) else: check_stations = StationCollection() dry_run = False if args.dry_run is None else args.dry_run if not dry_run and not len(check_stations): # ignore if calling a dry run # purge solutions if requested purge_solutions(JobServer, args, dates, GamitConfig) elif args.purge: tqdm.write( ' >> Dry run or check mode activated. Cannot purge solutions in these modes.' ) # run the job server sessions = ExecuteGamit(cnn, JobServer, GamitConfig, stations, check_stations, args.ignore_missing, dates, args.dry_run, args.create_kml) # execute globk on doys that had to be divided into subnets if not args.dry_run: ExecuteGlobk(cnn, JobServer, GamitConfig, sessions, dates) # parse the zenith delay outputs ParseZTD(GamitConfig.NetworkConfig.network_id.lower(), dates, sessions, GamitConfig, JobServer) tqdm.write(' >> %s Successful exit from Parallel.GAMIT' % print_datetime())
def main(): global cnn parser = argparse.ArgumentParser(description='Parallel.GAMIT main execution program') parser.add_argument('session_cfg', type=str, nargs=1, metavar='session.cfg', help="Filename with the session configuration to run Parallel.GAMIT") parser.add_argument('-d', '--date', type=str, nargs=2, metavar='{date}', help="Date range to process. Can be specified in yyyy/mm/dd yyyy_doy wwww-d format") parser.add_argument('-dp', '--date_parser', type=str, nargs=2, metavar='{year} {doys}', help="Parse date using ranges and commas (e.g. 2018 1,3-6). " "Cannot cross year boundaries") parser.add_argument('-e', '--exclude', type=str, nargs='+', metavar='station', help="List of stations to exclude from this processing (e.g. -e igm1 lpgs vbca)") parser.add_argument('-p', '--purge', action='store_true', help="Purge year doys from the database and directory structure and re-run the solution.") parser.add_argument('-dry', '--dry_run', action='store_true', help="Generate the directory structures (locally) but do not run GAMIT. " "Output is left in the production directory.") parser.add_argument('-kml', '--generate_kml', action='store_true', help="Generate KML and exit without running GAMIT.") parser.add_argument('-np', '--noparallel', action='store_true', help="Execute command without parallelization.") args = parser.parse_args() dates = None drange = None try: if args.date_parser: year = int(args.date_parser[0]) doys = parseIntSet(args.date_parser[1]) if any([doy for doy in doys if doy < 1]): parser.error('DOYs cannot start with zero. Please selected a DOY range between 1-365/366') if 366 in doys: if year % 4 != 0: parser.error('Year ' + str(year) + ' is not a leap year: DOY 366 does not exist.') dates = [pyDate.Date(year=year, doy=i) for i in doys] drange = [dates[0], dates[-1]] else: drange = process_date(args.date, missing_input=None) if not all(drange): parser.error('Must specify a start and end date for the processing.') # get the dates to purge dates = [pyDate.Date(mjd=i) for i in range(drange[0].mjd, drange[1].mjd + 1)] except ValueError as e: parser.error(str(e)) print(' >> Reading configuration files and creating project network, please wait...') GamitConfig = pyGamitConfig.GamitConfiguration(args.session_cfg[0]) # type: pyGamitConfig.GamitConfiguration print(' >> Checing GAMIT tables for requested config and year, please wait...') JobServer = pyJobServer.JobServer(GamitConfig, check_gamit_tables=(pyDate.Date(year=drange[1].year, doy=drange[1].doy), GamitConfig.gamitopt['eop_type']), run_parallel=not args.noparallel, software_sync=GamitConfig.gamitopt['gamit_remote_local']) cnn = dbConnection.Cnn(GamitConfig.gamitopt['gnss_data']) # type: dbConnection.Cnn # to exclude stations, append them to GamitConfig.NetworkConfig with a - in front exclude = args.exclude if exclude is not None: print(' >> User selected list of stations to exclude:') Utils.print_columns(exclude) GamitConfig.NetworkConfig['stn_list'] += ',-' + ',-'.join(exclude) if args.dry_run is not None: dry_run = args.dry_run else: dry_run = False if not dry_run: # ignore if calling a dry run # purge solutions if requested purge_solutions(JobServer, args, dates, GamitConfig) # initialize stations in the project stations = station_list(cnn, GamitConfig.NetworkConfig, drange) tqdm.write(' >> Creating GAMIT session instances, please wait...') sessions = [] archive = pyArchiveStruct.RinexStruct(cnn) # type: pyArchiveStruct.RinexStruct for date in tqdm(dates, ncols=80): # make the dir for these sessions # this avoids a racing condition when starting each process pwd = GamitConfig.gamitopt['solutions_dir'].rstrip('/') + '/' + date.yyyy() + '/' + date.ddd() if not os.path.exists(pwd): os.makedirs(pwd) net_object = Network(cnn, archive, GamitConfig, stations, date) sessions += net_object.sessions if args.generate_kml: # generate a KML of the sessions generate_kml(dates, sessions, GamitConfig) exit() # print a summary of the current project (NOT VERY USEFUL AFTER ALL) # print_summary(stations, sessions, drange) # run the job server ExecuteGamit(sessions, JobServer, dry_run) # execute globk on doys that had to be divided into subnets if not args.dry_run: ExecuteGlobk(GamitConfig, sessions, dates) # parse the zenith delay outputs ParseZTD(GamitConfig.NetworkConfig.network_id, sessions, GamitConfig) print(' >> Done processing and parsing information. Successful exit from Parallel.GAMIT')
def main(): parser = argparse.ArgumentParser(description='GNSS time series stacker') parser.add_argument( 'project', type=str, nargs=1, metavar='{project name}', help= "Specify the project name used to process the GAMIT solutions in Parallel.GAMIT." ) parser.add_argument( '-max', '--max_iters', nargs=1, type=int, metavar='{max_iter}', help="Specify maximum number of iterations. Default is 4.") parser.add_argument( '-exclude', '--exclude_stations', nargs='+', type=str, metavar='{net.stnm}', help="Manually specify stations to remove from the stacking process.") parser.add_argument( '-use', '--use_stations', nargs='+', type=str, metavar='{net.stnm}', help="Manually specify stations to use for the stacking process.") parser.add_argument( '-dir', '--directory', type=str, help= "Directory to save the resulting PNG files. If not specified, assumed to be the " "production directory") parser.add_argument('-redo', '--redo_stack', action='store_true', help="Delete the stack and redo it from scratch") parser.add_argument( '-itrf', '--itrf', nargs='+', help= "File with the ITRF periodic space parameters as given by Zuheir Altamimi and the list " "of stations to inherit the periodic terms from. Example: -itrf periodic.dat igs.braz " "rms.autf rms.igm1 rms.sant ...") parser.add_argument('-np', '--noparallel', action='store_true', help="Execute command without parallelization.") args = parser.parse_args() cnn = dbConnection.Cnn("gnss_data.cfg") Config = pyOptions.ReadOptions( "gnss_data.cfg") # type: pyOptions.ReadOptions JobServer = pyJobServer.JobServer( Config, run_parallel=not args.noparallel) # type: pyJobServer.JobServer if args.max_iters: max_iters = int(args.max_iters[0]) else: max_iters = 4 print ' >> Defaulting to 4 iterations' if args.exclude_stations: exclude_stn = args.exclude_stations else: exclude_stn = [] if args.use_stations: use_stn = args.use_stations else: use_stn = [] # create folder for plots if args.directory: if not os.path.exists(args.directory): os.mkdir(args.directory) else: if not os.path.exists('production'): os.mkdir('production') args.directory = 'production' # load the ITRF dat file with the periodic space components if args.itrf: periodic = load_periodic_space(args.itrf[0]) else: periodic = None # create the stack object stack = pyStack.Stack(cnn, args.project[0], args.redo_stack) for i in range(max_iters): # create the target polyhedrons based on iteration number (i == 0: PPP) target = calculate_etms(cnn, stack, JobServer, i) qbar = tqdm(total=len(stack), ncols=160, desc=' >> Aligning polyhedrons (%i of %i)' % (i + 1, max_iters)) # work on each polyhedron of the stack for j in range(len(stack)): qbar.update() if stack[j].date != target[j].date: # raise an error if dates don't agree! raise StandardError( 'Error processing %s: dates don\'t agree (target date %s)' % (stack[j].date.yyyyddd(), target[j].date.yyyyddd())) else: if not stack[j].aligned: # should only attempt to align a polyhedron that is unaligned # do not set the polyhedron as aligned unless we are in the max iteration step stack[j].align(target[j], True if i == max_iters - 1 else False) # write info to the screen qbar.write( ' -- %s (%3i) %2i it: wrms: %4.1f T %5.1f %5.1f %5.1f ' 'R (%5.1f %5.1f %5.1f)*1e-9' % (stack[j].date.yyyyddd(), stack[j].stations_used, stack[j].iterations, stack[j].wrms * 1000, stack[j].helmert[-3] * 1000, stack[j].helmert[-2] * 1000, stack[j].helmert[-1] * 1000, stack[j].helmert[-6], stack[j].helmert[-5], stack[j].helmert[-4])) qbar.close() # before removing common modes (or inheriting periodic terms), calculate ETMs with final aligned solutions calculate_etms(cnn, stack, JobServer, iterations=None) if args.redo_stack: # only apply common mode removal if redoing the stack if args.itrf: stack.remove_common_modes(periodic, args.itrf[1:]) else: stack.remove_common_modes() # here, we also align the stack in velocity and coordinate space # TODO: include alignment to velocity and coordinate space # calculate the etms again, after removing or inheriting parameters calculate_etms(cnn, stack, JobServer, iterations=None) # save polyhedrons to the database stack.save() qbar = tqdm(total=len(stack.stations), ncols=160) for stn in stack.stations: # plot the ETMs qbar.update() qbar.postfix = '%s.%s' % (stn['NetworkCode'], stn['StationCode']) plot_etm(cnn, stack, stn, args.directory) qbar.close()
def main(): global wrms_n, wrms_e, wrms_u, project parser = argparse.ArgumentParser( description='GNSS daily repetitivities analysis (DRA)') parser.add_argument( 'project', type=str, nargs=1, metavar='{project name}', help= "Specify the project name used to process the GAMIT solutions in Parallel.GAMIT." ) parser.add_argument( '-d', '--date_filter', nargs='+', metavar='date', help= 'Date range filter. Can be specified in yyyy/mm/dd yyyy_doy wwww-d format' ) parser.add_argument( '-w', '--plot_window', nargs='+', metavar='date', help= 'Date window range to plot. Can be specified in yyyy/mm/dd yyyy_doy wwww-d format' ) parser.add_argument('-hist', '--histogram', action='store_true', help="Plot a histogram of the daily repetitivities") parser.add_argument( '-v', '--verbose', action='store_true', help= "Provide additional information during the alignment process (for debugging purposes)" ) parser.add_argument('-np', '--noparallel', action='store_true', help="Execute command without parallelization.") args = parser.parse_args() cnn = dbConnection.Cnn("gnss_data.cfg") project = args.project[0] dates = [pyDate.Date(year=1980, doy=1), pyDate.Date(year=2100, doy=1)] Config = pyOptions.ReadOptions( "gnss_data.cfg") # type: pyOptions.ReadOptions JobServer = pyJobServer.JobServer( Config, run_parallel=not args.noparallel) # type: pyJobServer.JobServer try: dates = process_date(args.date_filter) except ValueError as e: parser.error(str(e)) pdates = None if args.plot_window is not None: if len(args.plot_window) == 1: try: pdates = process_date(args.plot_window, missing_input=None, allow_days=False) pdates = (pdates[0].fyear, ) except ValueError: # an integer value pdates = float(args.plot_window[0]) else: pdates = process_date(args.plot_window) pdates = (pdates[0].fyear, pdates[1].fyear) # create folder for plots path_plot = project + '_dra' if not os.path.isdir(path_plot): os.makedirs(path_plot) ######################################## # load polyhedrons # create the DRA object dra = DRA(cnn, args.project[0], dates[0], dates[1], args.verbose) dra.stack_dra() dra.to_json(project + '_dra.json') missing_doys = [] tqdm.write( ' >> Daily repetitivity analysis done. DOYs with wrms > 8 mm are shown below:' ) for i, d in enumerate(dra): if d.wrms is not None: if d.wrms > 0.008: tqdm.write( ' -- %s (%04i) %2i it wrms: %4.1f D-W: %5.3f IQR: %4.1f' % (d.date.yyyyddd(), d.stations_used, d.iterations, d.wrms * 1000, d.down_frac, d.iqr * 1000)) qbar = tqdm(total=len(dra.stations), desc=' >> Computing DRAs', ncols=160, disable=None) modules = ('pyETM', 'dbConnection', 'traceback', 'io', 'numpy') JobServer.create_cluster(compute_dra, progress_bar=qbar, callback=callback_handler, modules=modules) # plot each DRA for stn in dra.stations: NetworkCode = stn['NetworkCode'] StationCode = stn['StationCode'] ts = dra.get_station(NetworkCode, StationCode) JobServer.submit(ts, NetworkCode, StationCode, pdates, project, args.histogram) JobServer.wait() qbar.close() JobServer.close_cluster() wrms_n = np.array(wrms_n) wrms_e = np.array(wrms_e) wrms_u = np.array(wrms_u) # plot the WRM of the DRA stack and number of stations f, axis = plt.subplots(nrows=3, ncols=2, figsize=(15, 10)) # type: plt.subplots # WRMS ax = axis[0][0] ax.plot([t['fyear'] for t in dra.transformations[0]], [t['wrms'] * 1000 for t in dra.transformations[0]], 'ob', markersize=2) ax.set_ylabel('WRMS [mm]') ax.grid(True) ax.set_ylim(0, 10) # station count ax = axis[1][0] ax.plot([t['fyear'] for t in dra.transformations[0]], [t['stations_used'] for t in dra.transformations[0]], 'ob', markersize=2) ax.set_ylabel('Station count') ax.grid(True) # d-w fraction ax = axis[2][0] ax.plot([t['fyear'] for t in dra.transformations[0]], [t['downweighted_fraction'] for t in dra.transformations[0]], 'ob', markersize=2) ax.set_ylabel('DW fraction') ax.grid(True) ax = axis[0][1] ax.hist(wrms_n[wrms_n <= 8], 40, alpha=0.75, facecolor='blue') ax.grid(True) ax.set_ylabel('# stations') ax.set_xlabel('WRMS misfit N [mm]') ax.set_title('Daily repetitivities NEU') ax = axis[1][1] ax.hist(wrms_e[wrms_e <= 8], 40, alpha=0.75, facecolor='blue') ax.grid(True) ax.set_xlim(0, 8) ax.set_ylabel('# stations') ax.set_xlabel('WRMS misfit E [mm]') ax = axis[2][1] ax.hist(wrms_u[wrms_u <= 10], 40, alpha=0.75, facecolor='blue') ax.grid(True) ax.set_xlim(0, 10) ax.set_ylabel('# stations') ax.set_xlabel('WRMS misfit U [mm]') f.suptitle('Daily repetitivity analysis for project %s\n' 'Solutions with WRMS > 10 mm are not shown' % project, fontsize=12, family='monospace') plt.savefig(project + '_dra.png') plt.close() ax.set_xlim(0, 8)
def main(): parser = argparse.ArgumentParser(description='GNSS time series stacker') parser.add_argument( 'project', type=str, nargs=1, metavar='{project name}', help= "Specify the project name used to process the GAMIT solutions in Parallel.GAMIT." ) parser.add_argument( '-max', '--max_iters', nargs=1, type=int, metavar='{max_iter}', help="Specify maximum number of iterations. Default is 4.") parser.add_argument( '-exclude', '--exclude_stations', nargs='+', type=str, metavar='{net.stnm}', help="Manually specify stations to remove from the stacking process.") parser.add_argument( '-use', '--use_stations', nargs='+', type=str, metavar='{net.stnm}', help="Manually specify stations to use for the stacking process.") parser.add_argument('-np', '--noparallel', action='store_true', help="Execute command without parallelization.") args = parser.parse_args() cnn = dbConnection.Cnn("gnss_data.cfg") Config = pyOptions.ReadOptions( "gnss_data.cfg") # type: pyOptions.ReadOptions if not args.noparallel: JobServer = pyJobServer.JobServer( Config, run_node_test=False) # type: pyJobServer.JobServer else: JobServer = None Config.run_parallel = False if args.max_iters: max_iters = int(args.max_iters[0]) else: max_iters = 4 if args.exclude_stations: exclude_stn = args.exclude_stations else: exclude_stn = [] if args.use_stations: use_stn = args.use_stations else: use_stn = [] # create folder for plots if not os.path.isdir(args.project[0]): os.makedirs(args.project[0]) ######################################## # load polyhedrons project = Project(cnn, args.project[0], max_iters, use=use_stn, exclude=exclude_stn) #project.remove_common_modes(cnn) #exit() calculate_etms(cnn, project, JobServer) align_stack(cnn, project, JobServer) # remove common modes updated_poly = project.remove_common_modes(cnn) updated_poly.sort(key=lambda k: k['FYear']) # replace with new polyhedrons project.polyhedrons = updated_poly # last call to calculate ETMs calculate_etms(cnn, project, JobServer) tqdm.write(' -- Plotting final ETMs (aligned)...') project.plot_etms()