def purge_solution(pwd, project, date): cnn = dbConnection.Cnn('gnss_data.cfg') # delete the main solution dir (may be entire GAMIT run or combination directory) if os.path.isdir(os.path.join(pwd, project)): shutil.rmtree(os.path.join(pwd, project)) # possible subnetworks for sub in glob.glob(os.path.join(pwd, project + '.*')): shutil.rmtree(sub) # now remove the database entries cnn.query('DELETE FROM gamit_soln_excl WHERE "Year" = %i AND "DOY" = %i ' 'AND "Project" = \'%s\'' % (date.year, date.doy, project)) cnn.query('DELETE FROM stacks WHERE "Year" = %i AND "DOY" = %i ' 'AND "Project" = \'%s\'' % (date.year, date.doy, project)) cnn.query('DELETE FROM gamit_soln WHERE "Year" = %i AND "DOY" = %i ' 'AND "Project" = \'%s\'' % (date.year, date.doy, project)) cnn.query('DELETE FROM gamit_stats WHERE "Year" = %i AND "DOY" = %i ' 'AND "Project" = \'%s\'' % (date.year, date.doy, project)) cnn.query('DELETE FROM gamit_subnets WHERE "Year" = %i AND "DOY" = %i ' 'AND "Project" = \'%s\'' % (date.year, date.doy, project)) cnn.query('DELETE FROM gamit_ztd WHERE "Year" = %i AND "DOY" = %i ' 'AND "Project" = \'%s\'' % (date.year, date.doy, project)) cnn.close()
def main(): parser = argparse.ArgumentParser(description='GNSS time series stacker') parser.add_argument( 'project_file', type=str, nargs=1, metavar='{project cfg file}', help= "Project CFG file with all the stations being processed in Parallel.GAMIT" ) args = parser.parse_args() cnn = dbConnection.Cnn("gnss_data.cfg") GamitConfig = GamitConfiguration( args.project_file[0], check_config=False) # type: GamitConfiguration stations = station_list( cnn, GamitConfig.NetworkConfig, [Date(year=1999, doy=100), Date(year=1999, doy=128)]) # split the stations into subnet_count subnetworks archive = pyArchiveStruct.RinexStruct( cnn) # type: pyArchiveStruct.RinexStruct net_object = pyNetwork.Network(cnn, archive, GamitConfig, stations, Date(year=1999, doy=128)) generate_kml([Date(year=1999, doy=128)], net_object.sessions, GamitConfig)
def main(): parser = argparse.ArgumentParser(description='GNSS time series stacker') parser.add_argument('project', type=str, nargs=1, metavar='{project name}', help="Specify the project name used to process the GAMIT solutions in Parallel.GAMIT.") parser.add_argument('-np', '--noparallel', action='store_true', help="Execute command without parallelization.") args = parser.parse_args() cnn = dbConnection.Cnn("gnss_data.cfg") Config = pyOptions.ReadOptions("gnss_data.cfg") # type: pyOptions.ReadOptions if not args.noparallel: JobServer = pyJobServer.JobServer(Config, run_node_test=False) # type: pyJobServer.JobServer else: JobServer = None Config.run_parallel = False # create the execution log # load polyhedrons project = Project(cnn, args.project[0]) # plot initial state tqdm.write(' -- Plotting initial ETMs (unaligned)...') #for etm in tqdm(project.etms, ncols=160): # etm.plot(pngfile=args.project[0] + '/' + etm.NetworkCode + '.' + etm.StationCode + '_0.png', residuals=True) project.align_stack() tqdm.write(' -- Plotting intermediate step ETMs (aligned)...')
def station_etm(station, stn_ts, stack_name, iteration=0): cnn = dbConnection.Cnn("gnss_data.cfg") vertices = None try: # save the time series ts = pyETM.GamitSoln(cnn, stn_ts, station['NetworkCode'], station['StationCode'], stack_name) # create the ETM object etm = pyETM.GamitETM(cnn, station['NetworkCode'], station['StationCode'], False, False, ts) if etm.A is not None: if iteration == 0: # if iteration is == 0, then the target frame has to be the PPP ETMs vertices = etm.get_etm_soln_list(use_ppp_model=True, cnn=cnn) else: # on next iters, the target frame is the inner geometry of the stack vertices = etm.get_etm_soln_list() except pyETM.pyETMException: vertices = None return vertices if vertices else None
def compare_stninfo_rinex(NetworkCode, StationCode, STime, ETime, rinex_serial): try: cnn = dbConnection.Cnn("gnss_data.cfg") except Exception: return traceback.format_exc() + ' open de database when processing ' \ 'processing %s.%s' % (NetworkCode, StationCode), None try: # get the center of the session date = STime + (ETime - STime) / 2 date = pyDate.Date(datetime=date) stninfo = pyStationInfo.StationInfo(cnn, NetworkCode, StationCode, date) except pyStationInfo.pyStationInfoException as e: return "Station Information error: " + str(e), None if stninfo.currentrecord.ReceiverSerial.lower() != rinex_serial.lower(): return None, [ date, rinex_serial, stninfo.currentrecord.ReceiverSerial.lower() ] return None, None
def main(): parser = argparse.ArgumentParser(description='GNSS time series stacker') parser.add_argument('project', type=str, nargs=1, metavar='{project name}', help="Specify the project name used to process the GAMIT solutions in Parallel.GAMIT.") parser.add_argument('sinex', type=str, nargs=1, metavar='{project name}', help="SINEX file to update.") parser.add_argument('-d', '--date_filter', nargs='+', metavar='date', help='Date range filter can be specified in yyyy/mm/dd yyyy_doy wwww-d format') args = parser.parse_args() cnn = dbConnection.Cnn("gnss_data.cfg") Config = pyOptions.ReadOptions("gnss_data.cfg") # type: pyOptions.ReadOptions dates = [pyDate.Date(year=1980, doy=1), pyDate.Date(year=2100, doy=1)] try: dates = process_date(args.date_filter) except ValueError as e: parser.error(str(e)) sinex = args.sinex[0] project = args.project[0] process_sinex(cnn, project, dates, sinex)
def main(): parser = argparse.ArgumentParser(description='GNSS time series stacker') parser.add_argument('project', type=str, nargs=1, metavar='{project name}', help="Specify the project name used to process the GAMIT solutions in Parallel.GAMIT.") parser.add_argument('-d', '--date_filter', nargs='+', metavar='date', help='Date range filter Can be specified in yyyy/mm/dd yyyy_doy wwww-d format') args = parser.parse_args() cnn = dbConnection.Cnn("gnss_data.cfg") Config = pyOptions.ReadOptions("gnss_data.cfg") # type: pyOptions.ReadOptions # create the execution log dates = [pyDate.Date(year=1980, doy=1), pyDate.Date(year=2100, doy=1)] try: dates = process_date(args.date_filter) except ValueError as e: parser.error(str(e)) # create folder for plots if not os.path.isdir(args.project[0]): os.makedirs(args.project[0]) ######################################## # load polyhedrons project = dra(cnn, args.project[0], dates)
def check_rinex_stn(NetworkCode, StationCode, start_date, end_date): # load the connection try: # try to open a connection to the database cnn = dbConnection.Cnn("gnss_data.cfg") Config = pyOptions.ReadOptions("gnss_data.cfg") except Exception: return traceback.format_exc() + ' processing: (' + NetworkCode + '.' + StationCode \ + ') using node ' + platform.node(), None try: Archive = pyArchiveStruct.RinexStruct(cnn) rs = cnn.query('SELECT * FROM rinex WHERE "NetworkCode" = \'%s\' AND ' '"StationCode" = \'%s\' AND ' '"ObservationSTime" BETWEEN \'%s\' AND \'%s\' ' 'ORDER BY "ObservationSTime"' % (NetworkCode, StationCode, start_date.yyyymmdd(), end_date.yyyymmdd())) rnxtbl = rs.dictresult() missing_files = [] for rnx in rnxtbl: crinex_path = os.path.join( Config.archive_path, Archive.build_rinex_path(NetworkCode, StationCode, rnx['ObservationYear'], rnx['ObservationDOY'], filename=rnx['Filename'])) if not os.path.exists(crinex_path): # problem with file! does not appear to be in the archive Archive.remove_rinex(rnx) event = pyEvents.Event( Description= 'A missing RINEX file was found during RINEX integrity check: ' + crinex_path + '. It has been removed from the database. Consider rerunning PPP for this station.', NetworkCode=NetworkCode, StationCode=StationCode, Year=rnx['ObservationYear'], DOY=rnx['ObservationDOY']) cnn.insert_event(event) missing_files += [crinex_path] return None, missing_files except Exception: return traceback.format_exc() + ' processing: ' + NetworkCode + '.' + \ StationCode + ' using node ' + platform.node(), None
def station_etm(project, station, stn_ts, exclude, iteration=0): msg = None add_exclude = [] cnn = dbConnection.Cnn("gnss_data.cfg") sql_r = 'INSERT INTO stack_residuals ' \ '("NetworkCode", "StationCode", "Project", x, y, z, sigmax, sigmay, sigmaz, "Year", "DOY") ' \ 'VALUES (%s, %s, \'' + project + '\', %f, %f, %f, %f, %f, %f, %i, %i)' sql_s = 'INSERT INTO stacks ' \ '("NetworkCode", "StationCode", "Project", "X", "Y", "Z", sigmax, sigmay, sigmaz, "Year", "DOY", "FYear") ' \ 'VALUES (\'' + station.NetworkCode + '\', \'' + station.StationCode + '\', \'' \ + project + '\', %f, %f, %f, 0, 0, 0, %i, %i, %f)' # make sure it is sorted by date stn_ts.sort(key=lambda k: (k[3], k[4])) try: # save the time series ts = pyETM.GamitSoln(cnn, stn_ts, station.NetworkCode, station.StationCode) cnn.executemany( sql_s, zip(ts.x.tolist(), ts.y.tolist(), ts.z.tolist(), [t.year for t in ts.date], [t.doy for t in ts.date], [t.fyear for t in ts.date])) if not exclude: # create the ETM object etm = pyETM.GamitETM(cnn, station.NetworkCode, station.StationCode, False, False, ts) if etm.A is None: # no contribution to stack, remove from the station list add_exclude = [station.dictionary] else: # insert the residuals for the station in stack_residuals # these values will be used later on in helmert_stack if iteration == 0: # if iteration is == 0, then the target frame has to be the PPP ETMs cnn.executemany( sql_r, etm.get_residuals_dict(use_ppp_model=True, cnn=cnn)) else: # on next iters, the target frame is the inner geometry of the stack cnn.executemany(sql_r, etm.get_residuals_dict()) except Exception as e: add_exclude = [station.dictionary] msg = 'Error while producing ETM for %s.%s: ' % ( station.NetworkCode, station.StationCode) + str(e) return add_exclude, msg
def gamit_callback(job): result = job.result if result is not None: msg = [] if 'error' not in result.keys(): if result['nrms'] > 1: msg.append(' > NRMS > 1.0 (%.3f)' % result['nrms']) if result['wl'] < 60: msg.append(' > WL fixed < 60 (%.1f)' % result['wl']) if result['missing']: msg.append(' > Missing sites in solution: ' + ', '.join(result['missing'])) # DDG: only show sessions with problems to facilitate debugging. if result['success']: if len(msg) > 0: tqdm.write( ' -- %s Done processing: %s -> WARNINGS:\n%s' % (print_datetime(), result['session'], '\n'.join(msg))) # insert information in gamit_stats try: cnn = dbConnection.Cnn( 'gnss_data.cfg') # type: dbConnection.Cnn cnn.insert('gamit_stats', result) cnn.close() except dbConnection.dbErrInsert as e: tqdm.write( ' -- %s Error while inserting GAMIT stat for %s: ' % (print_datetime(), result['session'] + ' ' + str(e))) else: tqdm.write(' -- %s Done processing: %s -> FATAL:\n' ' > Failed to complete. Check monitor.log:\n%s' % (print_datetime(), result['session'], indent('\n'.join(result['fatals']), 4))) # write FATAL to file f = open('FATAL.log', 'a') f.write( 'ON %s session %s -> FATAL: Failed to complete. Check monitor.log\n%s\n' % (print_datetime(), result['session'], indent('\n'.join(result['fatals']), 4))) f.close() else: tqdm.write( ' -- %s Error in session %s message from node follows -> \n%s' % (print_datetime(), result['session'], result['error'])) else: tqdm.write( ' -- %s Fatal error on node %s message from node follows -> \n%s' % (print_datetime(), job.ip_addr, job.exception))
def main(): parser = argparse.ArgumentParser(description='GNSS time series stacker') parser.add_argument('project', type=str, nargs=1, metavar='{project name}', help="Specify the project name used to process the GAMIT solutions in Parallel.GAMIT.") parser.add_argument('-d', '--date_filter', nargs='+', metavar='date', help='Date range filter Can be specified in yyyy/mm/dd yyyy_doy wwww-d format') args = parser.parse_args() cnn = dbConnection.Cnn("gnss_data.cfg") project = args.project[0] dates = [pyDate.Date(year=1980, doy=1), pyDate.Date(year=2100, doy=1)] try: dates = process_date(args.date_filter) except ValueError as e: parser.error(str(e)) # create folder for plots if not os.path.isdir(project + '_dra'): os.makedirs(project + '_dra') ######################################## # load polyhedrons dra = DRA(cnn, args.project[0], dates[1]) dra.stack_dra() for stn in tqdm(dra.stations): NetworkCode = stn['NetworkCode'] StationCode = stn['StationCode'] # load from the db ts = dra.get_station(NetworkCode, StationCode) if ts.size: try: if ts.shape[0] > 2: dts = np.append(np.diff(ts[:, 0:3], axis=0), ts[1:, -3:], axis=1) dra_ts = pyETM.GamitSoln(cnn, dts, NetworkCode, StationCode, project) etm = pyETM.DailyRep(cnn, NetworkCode, StationCode, False, False, dra_ts) etm.plot(pngfile='%s/%s.%s_DRA.png' % (project + '_dra', NetworkCode, StationCode), plot_missing=False) except Exception as e: tqdm.write(' -->' + str(e)) dra.to_json(project + '_dra.json')
def load_harpos(header, otl): cnn = dbConnection.Cnn("gnss_data.cfg") # begin removing the network code from the OTL NetStn = re.findall('S\s+(\w+.\w+)\s+', ''.join(otl)) NetworkCode, StationCode = NetStn[0].split('.') OTL = (''.join(header) + ''.join(otl)).replace(NetStn[0], StationCode + ' ') + 'HARPOS Format version of 2002.12.12' print(' >> updating %s.%s' % (NetworkCode, StationCode)) cnn.query('UPDATE stations SET "Harpos_coeff_otl" = \'%s\' WHERE "NetworkCode" = \'%s\' AND "StationCode" = \'%s\'' % (OTL, NetworkCode, StationCode))
def UpdateRecord(rinex, path): cnn = dbConnection.Cnn('gnss_data.cfg') Config = pyOptions.ReadOptions('gnss_data.cfg') try: rnxobj = pyRinex.ReadRinex(rinex['NetworkCode'], rinex['StationCode'], path) date = pyDate.Date(year=rinex['ObservationYear'], doy=rinex['ObservationDOY']) if not verify_rinex_date_multiday(date, rnxobj, Config): cnn.begin_transac() # propagate the deletes cnn.query( 'DELETE FROM gamit_soln WHERE "NetworkCode" = \'%s\' AND "StationCode" = \'%s\' AND "Year" = %i AND "DOY" = %i' % (rinex['NetworkCode'], rinex['StationCode'], rinex['ObservationYear'], rinex['ObservationDOY'])) cnn.query( 'DELETE FROM ppp_soln WHERE "NetworkCode" = \'%s\' AND "StationCode" = \'%s\' AND "Year" = %i AND "DOY" = %i' % (rinex['NetworkCode'], rinex['StationCode'], rinex['ObservationYear'], rinex['ObservationDOY'])) cnn.query( 'DELETE FROM rinex WHERE "NetworkCode" = \'%s\' AND "StationCode" = \'%s\' AND "ObservationYear" = %i AND "ObservationDOY" = %i' % (rinex['NetworkCode'], rinex['StationCode'], rinex['ObservationYear'], rinex['ObservationDOY'])) cnn.commit_transac() return 'Multiday rinex file moved out of the archive: ' + rinex[ 'NetworkCode'] + '.' + rinex['StationCode'] + ' ' + str( rinex['ObservationYear']) + ' ' + str( rinex['ObservationDOY'] ) + ' using node ' + platform.node() else: cnn.update('rinex', rinex, Completion=rnxobj.completion) except pyRinex.pyRinexExceptionBadFile: # empty file or problem with crinex format, move out archive = pyArchiveStruct.RinexStruct(cnn) archive.remove_rinex( rinex, os.path.join( Config.repository_data_reject, 'bad_rinex/%i/%03i' % (rinex['ObservationYear'], rinex['ObservationDOY']))) except Exception: return traceback.format_exc( ) + ' processing rinex: ' + rinex['NetworkCode'] + '.' + rinex[ 'StationCode'] + ' ' + str(rinex['ObservationYear']) + ' ' + str( rinex['ObservationDOY']) + ' using node ' + platform.node()
def main(): parser = argparse.ArgumentParser(description='Program to alter the default ETM parameters for each station. ' 'The command can be executed on several stations at the same time. ' 'It is also possible to alter parameters for PPP and GAMIT ' 'simultaneously.') parser.add_argument('stnlist', type=str, nargs='+', metavar='all|net.stnm', help="List of networks/stations to process given in [net].[stnm] format or just [stnm] " "(separated by spaces; if [stnm] is not unique in the database, all stations with that " "name will be processed). Use keyword 'all' to process all stations in the database. " "If [net].all is given, all stations from network [net] will be processed. " "Alternatively, a file with the station list can be provided.") parser.add_argument('-fun', '--function_type', nargs='+', metavar=('function', 'argument'), default=[], help="Specifies the type of function to work with. Can be polynomial (p), jump (j), " "periodic (q) or bulk earthquake jump removal (t). Each one accepts a list of arguments. " "p {terms} where terms equals the number of polynomial terms in the ETM, i.e. " "terms = 2 is constant velocity and terms = 3 is velocity + acceleration, etc.\n" "j {action} {type} {date} {relax} where action can be + or -. A + indicates that a jump " "should be added while a - means that an existing jump should be removed; " "type = 0 is a mechanic jump and 1 is a geophysical jump; " "date is the date of the event in all the accepted formats " "(yyyy/mm/dd yyyy_doy gpswk-wkday fyear); and relax is a list of relaxation times for the " "logarithmic decays (only used when type = 1, they are ignored when type = 0).\n" "q {periods} where periods is a list expressed in days (1 yr = 365.25). " "t {max_magnitude} {stack_name} removes any earthquake Mw <= max_magnitude from " "the specified stations' trajectory models; if GAMIT solutions are invoked, provide the " "stack_name to obtain the ETMs of the stations.") parser.add_argument('-soln', '--solution_type', nargs='+', choices=['ppp', 'gamit'], default=['ppp', 'gamit'], action=required_length(1, 2), help="Specifies the type of solution that this command will affect. If left empty, the ETMs " "for both PPP and GAMIT will be affected. Otherwise, specify gamit to insert or " "remove the function on GAMIT ETMs only or ppp to insert or remove the function on PPP " "ETMs only.") parser.add_argument('-print', '--print_params', action='store_true', help="Print the parameters present in the database for the selected stations.") args = parser.parse_args() cnn = dbConnection.Cnn("gnss_data.cfg") # get the station list stnlist = Utils.process_stnlist(cnn, args.stnlist) if args.print_params: print_params(cnn, stnlist) else: insert_modify_param(parser, cnn, stnlist, args)
def create_files(): cnn = dbConnection.Cnn("gnss_data.cfg") rs = cnn.query( 'SELECT * FROM stations WHERE "NetworkCode" NOT LIKE \'?%%\' AND "Harpos_coeff_otl" LIKE \'%%HARPOS%%\' ORDER BY "NetworkCode", "StationCode"' ) # rs = cnn.query( # 'SELECT * FROM stations WHERE "NetworkCode" NOT LIKE \'?%%\' ORDER BY "NetworkCode", "StationCode"') stations = rs.dictresult() print ' >> Cantidad de estaciones a procesar en Chalmers: ' + str( len(stations)) stnlist = [] index = 0 for stn in stations: stnlist += [ '%-24s %16.3f%16.3f%16.3f' % (stn['NetworkCode'] + '.' + stn['StationCode'], float( stn['auto_x']), float(stn['auto_y']), float(stn['auto_z'])) ] if len(stnlist) == 99: body = '\n'.join(stnlist) with open('otl_%i.list' % index, 'w') as otl_list: otl_list.write(body) index += 1 # msg = MIMEText(body) # msg['Subject'] = 'Subject: Ocean Loading Tides' # msg['From'] = '*****@*****.**' # msg['To'] = '*****@*****.**' # # s = smtplib.SMTP_SSL('64.233.190.108', 465) # s.ehlo() # s.login('*****@*****.**', 'demostenes0624') # s.sendmail('*****@*****.**', '*****@*****.**', msg.as_string()) # s.close() stnlist = [] if len(stnlist) > 0: body = '\n'.join(stnlist) with open('otl_%i.list' % index, 'w') as otl_list: otl_list.write(body)
def main(): Config = pyOptions.ReadOptions('gnss_data.cfg') JobServer = pyJobServer.JobServer(Config) # type: pyJobServer.JobServer cnn = dbConnection.Cnn('gnss_data.cfg') archive = pyArchiveStruct.RinexStruct(cnn) rinex = cnn.query_float( 'SELECT * FROM rinex WHERE "ObservationYear" <= 1995 ORDER BY "NetworkCode", ' '"StationCode", "ObservationYear", "ObservationDOY"', as_dict=True) pbar = tqdm(desc='%-30s' % ' >> Processing rinex files', total=len(rinex), ncols=160) modules = ('os', 'pyRinex') callback = [] for rnx in rinex: filename = archive.build_rinex_path(rnx['NetworkCode'], rnx['StationCode'], rnx['ObservationYear'], rnx['ObservationDOY'], filename=rnx['Filename']) arguments = (rnx['NetworkCode'], rnx['StationCode'], Config.archive_path, filename) JobServer.SubmitJob(check_rinex, arguments, (), modules, callback, callback_class(pbar), 'callbackfunc') if JobServer.process_callback: # handle any output messages during this batch callback = output_handle(callback) JobServer.process_callback = False tqdm.write(' >> waiting for jobs to finish...') JobServer.job_server.wait() tqdm.write(' >> Done.') # process the errors and the new stations output_handle(callback) pbar.close()
def main(): parser = argparse.ArgumentParser(description='Script to synchronize AWS with OSU\'s archive database') parser.add_argument('date', type=str, nargs=1, help="Check the sync state for this given date. Format can be fyear or yyyy_ddd.") parser.add_argument('-mark', '--mark_uploaded', nargs='+', type=str, help="Pass net.stnm to mark these files as transferred to the AWS", metavar='{net.stnm}') parser.add_argument('-pull', '--pull_rinex', action='store_true', help="Get all the unsynchronized RINEX files in the local dir") parser.add_argument('-np', '--noparallel', action='store_true', help="Execute command without parallelization.") args = parser.parse_args() Config = pyOptions.ReadOptions("gnss_data.cfg") # type: pyOptions.ReadOptions cnn = dbConnection.Cnn('gnss_data.cfg') # before attempting anything, check aliases!! print ' >> Checking GAMIT aliases' check_aliases(cnn) # initialize the PP job server if not args.noparallel: JobServer = pyJobServer.JobServer(Config, 1500) # type: pyJobServer.JobServer else: JobServer = None Config.run_parallel = False dd = args.date[0] if '_' in dd: date = pyDate.Date(year=int(dd.split('_')[0]), doy=int(dd.split('_')[1])) elif dd == 'all': # run all dates (1994 to 2018) ts = range(pyDate.Date(year=2004, doy=20).mjd, pyDate.Date(year=2018, doy=87).mjd, 1) ts = [pyDate.Date(mjd=tts) for tts in ts] for date in ts: print ' >> Processing ' + str(date) pull_rinex(cnn, date, Config, JobServer) return else: date = pyDate.Date(fyear=float(dd)) if args.pull_rinex: pull_rinex(cnn, date, Config, JobServer) if args.mark_uploaded is not None: print 'Processing %i for day %s' % (len(args.mark_uploaded), date.yyyyddd()) # mark the list of stations as transferred to the AWS mark_uploaded(cnn, date, args.mark_uploaded)
def main(): parser = argparse.ArgumentParser(description='GNSS time series stacker') parser.add_argument('project_file', type=str, nargs=1, metavar='{project cfg file}', help="Project CFG file with all the stations being processed in Parallel.GAMIT") args = parser.parse_args() cnn = dbConnection.Cnn("gnss_data.cfg") GamitConfig = GamitConfiguration(args.project_file[0], check_config=False) # type: GamitConfiguration stations = process_stnlist(cnn, GamitConfig.NetworkConfig['stn_list'].split(',')) generate_kml(cnn, GamitConfig.NetworkConfig.network_id.lower(), stations)
def run_globk(globk_task, project, date): from datetime import datetime polyhedron, variance = globk_task.execute() # open a database connection (this is on the node) cnn = dbConnection.Cnn('gnss_data.cfg') err = [] # kill the existing polyhedron to make sure all the new vertices get in # this is because when adding a station, only the cnn.query( 'DELETE FROM gamit_soln WHERE "Project" = \'%s\' AND "Year" = %i AND "DOY" = %i' % (project, date.year, date.doy)) # insert polyherdon in gamit_soln table for key, value in polyhedron.items(): if '.' in key: try: sqrt_variance = math.sqrt(variance) cnn.insert('gamit_soln', NetworkCode=key.split('.')[0], StationCode=key.split('.')[1], Project=project, Year=date.year, DOY=date.doy, FYear=date.fyear, X=value.X, Y=value.Y, Z=value.Z, sigmax=value.sigX * sqrt_variance, sigmay=value.sigY * sqrt_variance, sigmaz=value.sigZ * sqrt_variance, sigmaxy=value.sigXY * sqrt_variance, sigmaxz=value.sigXZ * sqrt_variance, sigmayz=value.sigYZ * sqrt_variance, VarianceFactor=variance) except dbConnection.dbErrInsert as e: # tqdm.write(' --> Error inserting ' + key + ' -> ' + str(e)) pass else: err.append( ' -- %s Error while combining with GLOBK -> Invalid key found in session %s -> %s ' 'polyhedron in database may be incomplete.' % (datetime.now().strftime('%Y-%m-%d %H:%M:%S'), date.yyyyddd(), key)) cnn.close() return err
def main(): print(' >> Loading g08d APRs...') mat = hdf5storage.loadmat('PRIORS_from_g08d.mat') # stn_index = np.where(mat['pv_stnm'] == rnx['NetworkCode'].uppper() + '_' + rnx['StationCode'].upper())[0][0] # ydm_index = np.where((mat['pv_Epoch']['iyear'] == date.year) & (mat['pv_Epoch']['doy'] == date.doy)) cnn = dbConnection.Cnn('gnss_data.cfg') for stnm in mat['pv_stnm'].tolist(): NetworkCode = stnm[0].split('_')[0].lower() StationCode = stnm[0].split('_')[1].lower() station_id = NetworkCode + '.' + StationCode print(' -- inserting ' + station_id) if cnn.query( 'SELECT * FROM stations WHERE "NetworkCode" = \'%s\' AND "StationCode" = \'%s\'' % (NetworkCode, StationCode)).ntuples() != 0: # get the rows for this station stn_index = np.where(mat['pv_stnm'] == stnm[0])[0][0] xyz = mat['pv_xyz'][stn_index * 3:stn_index * 3 + 3] enu = mat['pv_sig_enu'][stn_index * 3:stn_index * 3 + 3] # loop through the epochs for i, fyear in enumerate(mat['pv_Epoch']['fyear'][0][0]): date = pyDate.Date(fyear=fyear) if enu[0][i] < 10: # print ' -- ' + station_id + ' ' + date.yyyyddd() # actual sigma value, otherwise it's a super unconstrained (should not be inserted) try: cnn.query( 'INSERT INTO apr_coords ' '("NetworkCode", "StationCode", "Year", "DOY", "FYear", "x", "y", "z", "sn", "se", "su", "ReferenceFrame") VALUES ' '(\'%s\', \'%s\', %i, %i, %f, %f, %f, %f, %f, %f, %f, \'g08d\')' % (NetworkCode, StationCode, date.year, date.doy, date.fyear, xyz[0][i], xyz[1][i], xyz[2][i], enu[0][i], enu[1][i], enu[2][i])) except pg.IntegrityError: print(' -- ' + station_id + ' ' + date.yyyyddd() + ' already exists!') else: print(' -- COULD NOT FIND STATION ' + station_id)
def load_blq(header, otl): cnn = dbConnection.Cnn("gnss_data.cfg") # begin removing the network code from the OTL NetStn = re.findall('\s{2}(\w{3}\.\w{4})', ''.join(otl)) NetworkCode, StationCode = NetStn[0].split('.') OTL = (''.join(header) + ''.join(otl)).replace(' ' + NetStn[0], ' ' + StationCode) OTL = OTL.replace('$$ ' + NetStn[0], '$$ %-8s' % StationCode) OTL = OTL.replace('$$ END TABLE', '$$') OTL = OTL.replace("'", "") print(' >> updating %s.%s' % (NetworkCode, StationCode)) cnn.query('UPDATE stations SET "Harpos_coeff_otl" = \'%s\' WHERE "NetworkCode" = \'%s\' AND "StationCode" = \'%s\'' % (OTL, NetworkCode, StationCode))
def compute_dra(ts, NetworkCode, StationCode, pdates, project, histogram=False): try: # load from the db cnn = dbConnection.Cnn('gnss_data.cfg') # to pass the filename back to the callback_handler filename = project + '_dra/' + NetworkCode + '.' + StationCode if ts.size: if ts.shape[0] > 2: dts = numpy.append(numpy.diff(ts[:, 0:3], axis=0), ts[1:, -3:], axis=1) dra_ts = pyETM.GamitSoln(cnn, dts, NetworkCode, StationCode, project) etm = pyETM.DailyRep(cnn, NetworkCode, StationCode, False, False, dra_ts) figfile = '' hisfile = '' if etm.A is not None: figfile = etm.plot(fileio=io.BytesIO(), plot_missing=False, t_win=pdates) if histogram: hisfile = etm.plot_hist(fileio=io.BytesIO()) # save the wrms return etm.factor[0] * 1000, etm.factor[1] * 1000, etm.factor[2] * 1000, figfile, hisfile, \ filename, NetworkCode, StationCode else: return None, None, None, figfile, hisfile, filename, NetworkCode, StationCode except Exception as e: raise Exception('While working on %s.%s' % (NetworkCode, StationCode) + '\n') from e
def main(): cnn = dbConnection.Cnn("gnss_data.cfg") stack = Stack(cnn, 'igs-sirgas', redo=True) for i in tqdm(range(1, len(stack)), ncols=160): stack[i].align(stack[i - 1]) net = 'igs' stn = 'braz' ts = stack.get_station(net, stn) dts = np.append(np.diff(ts[:, 0:3], axis=0), ts[1:, -3:], axis=1) ts = pyETM.GamitSoln(cnn, dts, net, stn, 'igs-sirgas') pyETM.GamitETM(cnn, net, stn, True, gamit_soln=ts)
def main(): cnn = dbConnection.Cnn('gnss_data.cfg') yrs = glob.glob('/data/[1-2]*') for dir in yrs: days = glob.glob(dir + '/*') for day in days: sessions = glob.glob(day + '/*') for sess in sessions: parts = os.path.split(sess) path = sess + '/monitor.log' if '.' in parts[1]: # gamit session, parse monitor insert data parse_monitor(cnn, path) elif os.path.isfile(path): parse_monitor(cnn, path)
def main(): parser = argparse.ArgumentParser( description='Plot ETM for stations in the database') parser.add_argument( 'stnlist', type=str, nargs='+', help= "List of networks/stations to plot given in [net].[stnm] format or just [stnm] " "(separated by spaces; if [stnm] is not unique in the database, all stations with that " "name will be plotted). Use keyword 'all' to plot all stations in all networks. " "If [net].all is given, all stations from network [net] will be plotted" ) parser.add_argument('-nop', '--no_plots', action='store_true', help="Do not produce plots", default=False) parser.add_argument('-nom', '--no_missing_data', action='store_true', help="Do not show missing days", default=False) parser.add_argument('-nm', '--no_model', action='store_true', help="Plot time series without fitting a model") parser.add_argument('-r', '--residuals', action='store_true', help="Plot time series residuals", default=False) parser.add_argument( '-dir', '--directory', type=str, help= "Directory to save the resulting PNG files. If not specified, assumed to be the " "production directory") parser.add_argument( '-json', '--json', type=int, help="Export ETM adjustment to JSON. Append '0' to just output " "the ETM parameters, '1' to export time series without " "model and '2' to export both time series and model.") parser.add_argument( '-gui', '--interactive', action='store_true', help="Interactive mode: allows to zoom and view the plot interactively" ) parser.add_argument( '-win', '--time_window', nargs='+', metavar='interval', help= 'Date range to window data. Can be specified in yyyy/mm/dd, yyyy.doy or as a single ' 'integer value (N) which shall be interpreted as last epoch-N') parser.add_argument( '-q', '--query', nargs=2, metavar='{type} {date}', type=str, help= 'Dates to query the ETM. Specify "model" or "solution" to get the ETM value or the value ' 'of the daily solution (if exists). Output is in XYZ.') parser.add_argument( '-gamit', '--gamit', type=str, nargs=1, metavar='{stack}', help="Plot the GAMIT time series specifying which stack name to plot.") parser.add_argument( '-lang', '--language', type=str, help="Change the language of the plots. Default is English. " "Use ESP to select Spanish. To add more languages, " "include the ISO 639-1 code in pyETM.py", default='ENG') parser.add_argument('-hist', '--histogram', action='store_true', help="Plot histogram of residuals") parser.add_argument( '-file', '--filename', type=str, help= "Obtain data from an external source (filename). Format should be specified with -format." ) parser.add_argument( '-format', '--format', nargs='+', type=str, help= "To be used together with --filename. Specify order of the fields as found in the input " "file. Format strings are gpsWeek, gpsWeekDay, year, doy, fyear, month, day, mjd, " "x, y, z, na. Use 'na' to specify a field that should be ignored. If fields to be ignored " "are at the end of the line, then there is no need to specify those.") parser.add_argument('-outliers', '--plot_outliers', action='store_true', help="Plot an additional panel with the outliers") parser.add_argument('-vel', '--velocity', action='store_true', help="During query, output the velocity in XYZ.") parser.add_argument('-seasonal', '--seasonal_terms', action='store_true', help="During query, output the seasonal terms in NEU.") parser.add_argument('-quiet', '--suppress_messages', action='store_true', help="Quiet mode: suppress information messages") args = parser.parse_args() cnn = dbConnection.Cnn('gnss_data.cfg') stnlist = Utils.process_stnlist(cnn, args.stnlist) # define the language pyETM.LANG = args.language.lower() # set the logging level if not args.suppress_messages: pyETM.logger.setLevel(pyETM.INFO) ##################################### # date filter dates = None if args.time_window is not None: if len(args.time_window) == 1: try: dates = process_date(args.time_window, missing_input=None, allow_days=False) dates = (dates[0].fyear, ) except ValueError: # an integer value dates = float(args.time_window[0]) else: dates = process_date(args.time_window) dates = (dates[0].fyear, dates[1].fyear) if stnlist: # do the thing if args.directory: if not os.path.exists(args.directory): os.mkdir(args.directory) else: if not os.path.exists('production'): os.mkdir('production') args.directory = 'production' for stn in stnlist: try: if args.gamit is None and args.filename is None: etm = pyETM.PPPETM(cnn, stn['NetworkCode'], stn['StationCode'], False, args.no_model) elif args.filename is not None: etm = from_file(args, cnn, stn) else: polyhedrons = cnn.query_float( 'SELECT "X", "Y", "Z", "Year", "DOY" FROM stacks ' 'WHERE "name" = \'%s\' AND "NetworkCode" = \'%s\' AND ' '"StationCode" = \'%s\' ' 'ORDER BY "Year", "DOY", "NetworkCode", "StationCode"' % (args.gamit[0], stn['NetworkCode'], stn['StationCode'])) soln = pyETM.GamitSoln(cnn, polyhedrons, stn['NetworkCode'], stn['StationCode'], args.gamit[0]) etm = pyETM.GamitETM(cnn, stn['NetworkCode'], stn['StationCode'], False, args.no_model, gamit_soln=soln) # print ' > %5.2f %5.2f %5.2f %i %i' % \ # (etm.factor[0]*1000, etm.factor[1]*1000, etm.factor[2]*1000, etm.soln.t.shape[0], # etm.soln.t.shape[0] - # np.sum(np.logical_and(np.logical_and(etm.F[0], etm.F[1]), etm.F[2]))) # print two largest outliers if etm.A is not None: lres = np.sqrt(np.sum(np.square(etm.R), axis=0)) slres = lres[np.argsort(-lres)] print ' >> Two largest residuals:' for i in [0, 1]: print(' %s %6.3f %6.3f %6.3f' % (pyDate.Date(mjd=etm.soln.mjd[ lres == slres[i]]).yyyyddd(), etm.R[0, lres == slres[i]], etm.R[1, lres == slres[i]], etm.R[2, lres == slres[i]])) if args.interactive: xfile = None else: if args.gamit is None: if args.filename is None: xfile = os.path.join( args.directory, '%s.%s_ppp' % (etm.NetworkCode, etm.StationCode)) else: xfile = os.path.join( args.directory, '%s.%s_file' % (etm.NetworkCode, etm.StationCode)) else: xfile = os.path.join( args.directory, '%s.%s_gamit' % (etm.NetworkCode, etm.StationCode)) # leave pngfile empty to enter interactive mode (GUI) if not args.no_plots: etm.plot(xfile + '.png', t_win=dates, residuals=args.residuals, plot_missing=not args.no_missing_data, plot_outliers=args.plot_outliers) if args.histogram: etm.plot_hist(xfile + '_hist.png') if args.json is not None: with open(xfile + '.json', 'w') as f: if args.json == 1: json.dump(etm.todictionary(time_series=True), f, indent=4, sort_keys=False) elif args.json == 2: json.dump(etm.todictionary(time_series=True, model=True), f, indent=4, sort_keys=False) else: json.dump(etm.todictionary(False), f, indent=4, sort_keys=False) if args.query is not None: model = True if args.query[0] == 'model' else False q_date = pyDate.Date(fyear=float(args.query[1])) xyz, _, _, txt = etm.get_xyz_s(q_date.year, q_date.doy, force_model=model) strp = '' # if user requests velocity too, output it if args.velocity: if etm.A is not None: vxyz = etm.rotate_2xyz(etm.Linear.p.params[:, 1]) strp = '%8.5f %8.5f %8.5f ' \ % (vxyz[0, 0], vxyz[1, 0], vxyz[2, 0]) # also output seasonal terms, if requested if args.seasonal_terms: if etm.Periodic.frequency_count > 0: strp += ' '.join([ '%8.5f' % (x * 1000) for x in etm.Periodic.p.params.flatten().tolist() ]) print ' %s.%s %14.5f %14.5f %14.5f %8.3f %s -> %s' \ % (etm.NetworkCode, etm.StationCode, xyz[0], xyz[1], xyz[2], q_date.fyear, strp, txt) print 'Successfully plotted ' + stn['NetworkCode'] + '.' + stn[ 'StationCode'] except pyETM.pyETMException as e: print str(e) except Exception: print 'Error during processing of ' + stn[ 'NetworkCode'] + '.' + stn['StationCode'] print traceback.format_exc() pass
def rinex_task(NetworkCode, StationCode, date, ObservationFYear, metafile): from pyRunWithRetry import RunCommandWithRetryExeception etm_err = '' # local directory as destiny for the CRINEZ files pwd_rinex = '/media/leleiona/aws-files/' + date.yyyy() + '/' + date.ddd() stop_no_aprs = False Config = pyOptions.ReadOptions( "gnss_data.cfg") # type: pyOptions.ReadOptions cnn = dbConnection.Cnn('gnss_data.cfg') # create Archive object Archive = pyArchiveStruct.RinexStruct( cnn) # type: pyArchiveStruct.RinexStruct ArchiveFile = Archive.build_rinex_path(NetworkCode, StationCode, date.year, date.doy) ArchiveFile = os.path.join(Config.archive_path, ArchiveFile) # check for a station alias in the alias table alias = cnn.query( 'SELECT * FROM stationalias WHERE "NetworkCode" = \'%s\' AND "StationCode" = \'%s\'' % (NetworkCode, StationCode)) sa = alias.dictresult() if len(sa) > 0: StationAlias = sa[0]['StationAlias'] else: StationAlias = StationCode # create the crinez filename filename = StationAlias + date.ddd() + '0.' + date.yyyy()[2:4] + 'd.Z' try: # create the ETM object etm = pyETM.PPPETM(cnn, NetworkCode, StationCode) # get APRs and sigmas (only in NEU) Apr, sigmas, Window, source = etm.get_xyz_s(date.year, date.doy) del etm except pyETM.pyETMException as e: # no PPP solutions available! MUST have aprs in the last run, try that stop_no_aprs = True Window = None source = '' etm_err = str(e) except Exception: return (None, None, traceback.format_exc() + ' processing ' + NetworkCode + '.' + StationCode + ' using node ' + platform.node() + '\n', metafile) # find this station-day in the lastest global run APRs apr_tbl = cnn.query( 'SELECT * FROM apr_coords WHERE "NetworkCode" = \'%s\' AND "StationCode" = \'%s\' ' 'AND "Year" = %i AND "DOY" = %i' % (NetworkCode, StationCode, date.year, date.doy)) apr = apr_tbl.dictresult() if len(apr) > 0: # APRs exist for this station-day # replace PPP ETM with Mike's APRs Apr = numpy.array( ([float(apr[0]['x'])], [float(apr[0]['y'])], [float(apr[0]['z'])])) sigmas = numpy.array(([float(apr[0]['sn'])], [float(apr[0]['se'])], [float(apr[0]['su'])])) source = apr[0]['ReferenceFrame'] + ' APRs' elif len(apr) == 0 and stop_no_aprs: return ( None, None, '%s.%s has no PPP solutions and no APRs from last global run for %s! ' 'Specific error from pyETM.PPPETM (if available) was: %s' % (NetworkCode, StationCode, date.yyyyddd(), etm_err), metafile) # convert sigmas to XYZ stn = cnn.query( 'SELECT * FROM stations WHERE "NetworkCode" = \'%s\' AND "StationCode" = \'%s\'' % (NetworkCode, StationCode)) stn = stn.dictresult() sigmas_xyz = sigmas_neu2xyz(stn[0]['lat'], stn[0]['lon'], sigmas) # write the station.info # if no station info comes back for this date, program will print a message and continue with next try: # Use the argument 'ObservationFYear' to get the exact RINEX session fyear without opening the file rnx_date = pyDate.Date(fyear=float(ObservationFYear)) stninfo = pyStationInfo.StationInfo(cnn, NetworkCode, StationCode, rnx_date, h_tolerance=12) except pyStationInfo.pyStationInfoException: # if no metadata, warn user and continue return ( None, None, '%s.%s has no metadata available for this date, but a RINEX exists!' % (NetworkCode, StationCode), metafile) # check if RINEX file needs to be synced or not. aws_sync = cnn.query( 'SELECT * FROM aws_sync WHERE "NetworkCode" = \'%s\' AND "StationCode" = \'%s\' ' 'AND "Year" = %i AND "DOY" = %i' % (NetworkCode, StationCode, date.year, date.doy)).dictresult() if len(aws_sync) == 0: # only copy RINEX if not synced! # open the RINEX file in the Archive try: with pyRinex.ReadRinex(NetworkCode, StationCode, ArchiveFile, False) as Rinex: # type: pyRinex.ReadRinex Rnx = None if Rinex.multiday: # find the rinex that corresponds to the session being processed, if multiday for rinex in Rinex.multiday_rnx_list: if rinex.date == date: Rnx = rinex break if Rnx is None: return ( None, None, '%s.%s was a multiday file and date %8.3f could not be found!' % (NetworkCode, StationCode, date.fyear), metafile) else: # if Rinex is not multiday Rnx = Rinex Rnx.purge_comments() Rnx.normalize_header(stninfo) Rnx.rename(filename) if Window is not None: window_rinex(Rnx, Window) source += ' windowed from/to ' + Window.datetime( ).strftime('%Y-%M-%d %H:%M:%S') # before creating local copy, decimate file Rnx.decimate(30) Rnx.compress_local_copyto(pwd_rinex) except (pyRinex.pyRinexException, RunCommandWithRetryExeception): # new behavior: if error occurs while generating RINEX, then copy raw file from the archive try: shutil.copy(ArchiveFile, os.path.join(pwd_rinex, filename)) except Exception: return (None, None, traceback.format_exc() + ' processing ' + NetworkCode + '.' + StationCode + ' using node ' + platform.node() + '\n', metafile) except Exception: return (None, None, traceback.format_exc() + ' processing ' + NetworkCode + '.' + StationCode + ' using node ' + platform.node() + '\n', metafile) # everything ok, return information APR = '%s.%s %s %12.3f %12.3f %12.3f %5.3f %5.3f %5.3f %5.3f %5.3f %5.3f %s' % ( NetworkCode, StationCode, StationAlias, Apr[0, 0], Apr[1, 0], Apr[2, 0], sigmas_xyz[0, 0], sigmas_xyz[1, 0], sigmas_xyz[2, 0], sigmas[1, 0], sigmas[0, 0], sigmas[2, 0], source.replace(' ', '_')) return APR, stninfo.return_stninfo().replace( StationCode.upper(), StationAlias.upper()), None, metafile
def main(): parser = argparse.ArgumentParser( description= 'Program to perform weekly loosely-constrained solutions. Combination ' 'is performed using GLOBK. Result is output in SINEX format.') parser.add_argument( 'stnlist', type=str, nargs='+', metavar='all|net.stnm', help="List of networks/stations to include in the solution.") parser.add_argument( '-s', '--session_config', type=str, nargs=1, metavar='session.cfg', help="Filename with the session configuration to run Parallel.GAMIT") parser.add_argument('-w', '--gpsweek', nargs=1, help="GPS week to combine.") parser.add_argument( '-e', '--exclude', type=str, nargs='+', metavar='station', help="List of stations to exclude (e.g. -e igm1 lpgs vbca)") args = parser.parse_args() cnn = dbConnection.Cnn("gnss_data.cfg") # get the working dates date_s = pyDate.Date(gpsWeek=int(args.gpsweek[0]), gpsWeekDay=0) date_e = pyDate.Date(gpsWeek=int(args.gpsweek[0]), gpsWeekDay=6) print ' >> Working with GPS week ' + args.gpsweek[0] + ' (%s to %s)' % ( date_s.yyyyddd(), date_e.yyyyddd()) exclude = args.exclude if exclude is not None: print(' >> User selected list of stations to exclude:') Utils.print_columns(exclude) args.stnlist += ['-' + exc for exc in exclude] # get the station list stnlist = Utils.process_stnlist(cnn, args.stnlist) # check that the selected stations have all different station codes # otherwise, exit with error for i in range(len(stnlist) - 1): for j in range(i + 1, len(stnlist)): if stnlist[i]['StationCode'] == stnlist[j]['StationCode']: print 'During station selection, two identical station codes were found. Please remove one and ' \ 'try again.' exit() GamitConfig = pyGamitConfig.GamitConfiguration( args.session_config[0]) # type: pyGamitConfig.GamitConfiguration project = GamitConfig.NetworkConfig.network_id.lower() org = GamitConfig.gamitopt['org'] # start making sure that the solutions were already incorporated into the stack cc = cnn.query_float( 'SELECT count(*) FROM stacks WHERE ("Year", "DOY") BETWEEN (%i, %i) AND (%i, %i) ' 'AND "Project" = \'%s\'' % (date_s.year, date_s.doy, date_e.year, date_e.doy, project)) if cc[0] == 0: print ' >> No solutions could be found in the stack for the specified gps week. Did you run the stacker ' \ 'before attempting to combine the solutions?' exit() else: soln_pwd = GamitConfig.gamitopt['solutions_dir'] # create a globk directory in production if not os.path.exists('production/globk'): os.makedirs('production/globk') # check if week folder exists if os.path.exists('production/globk/' + args.gpsweek[0]): rmtree('production/globk/' + args.gpsweek[0]) # create the directory os.makedirs('production/globk/' + args.gpsweek[0]) globk_pwd = 'production/globk/' + args.gpsweek[0] glx_list = [] # make a list of the h files that need to be combined for day in range(0, 7): date = pyDate.Date(gpsWeek=int(args.gpsweek[0]), gpsWeekDay=day) soln_dir = os.path.join( soln_pwd, date.yyyy() + '/' + date.ddd() + '/' + project + '/glbf') if os.path.exists(soln_dir): glx = glob.glob(os.path.join(soln_dir, '*.GLX.*')) if len(glx) > 0: glx_list.append({'file': glx[0], 'gpsweek': date.wwwwd()}) else: glx = glob.glob(os.path.join(soln_dir, '*.glx')) glx_list.append({'file': glx[0], 'gpsweek': date.wwwwd()}) # create the earthquakes.txt file to remove outliers with open(globk_pwd + '/eq_rename.txt', 'w') as fd: for stn in stnlist: rm = cnn.query_float( 'SELECT * FROM gamit_soln_excl WHERE "Project" = \'%s\' AND "NetworkCode" = \'%s\'' ' AND "StationCode" = \'%s\' AND ("Year", "DOY") BETWEEN (%i, %i) AND (%i, %i)' % (project, stn['NetworkCode'], stn['StationCode'], date_s.year, date_s.doy, date_e.year, date_e.doy), as_dict=True) for r in rm: date = pyDate.Date(year=r['Year'], doy=r['DOY']) fd.write( ' rename %s_gps %s_xcl %-20s %s %02i %02i 0 0 %s %02i %02i 24 0\n' % (stn['StationCode'], stn['StationCode'], org + date.wwwwd() + '.GLX', date.yyyy()[2:], date.month, date.day, date.yyyy()[2:], date.month, date.day)) # check for renames that might not agree between days mv = cnn.query_float( 'SELECT * FROM gamit_subnets WHERE "Project" = \'%s\' AND ("Year", "DOY") ' 'BETWEEN (%i, %i) AND (%i, %i) AND \'%s.%s\' = ANY(stations)' % (project, date_s.year, date_s.doy, date_e.year, date_e.doy, stn['NetworkCode'], stn['StationCode']), as_dict=True) for m in mv: date = pyDate.Date(year=m['Year'], doy=m['DOY']) # check on each day to see if alias agrees with station code for i, s in enumerate(m['stations']): if s.split('.')[1] != m['alias'][i] and \ s == stn['NetworkCode'] + '.' + stn['StationCode']: print ' -- Alias for %s.%s = %s: renaming' \ % (stn['NetworkCode'], stn['StationCode'], m['alias'][i]) # change the name of the station to the original name fd.write( ' rename %s_gps %s_gps %-20s %s %02i %02i 0 0 %s %02i %02i 24 0\n' % (m['alias'], stn['StationCode'], org + date.wwwwd() + '.GLX', date.yyyy()[2:], date.month, date.day, date.yyyy()[2:], date.month, date.day)) elif s not in [ st['NetworkCode'] + '.' + st['StationCode'] for st in stnlist ]: print ' -- Removing %s: not selected' % s # just in case, remove any other occurrences of this station code fd.write( ' rename %s_gps %s_xcl %-20s %s %02i %02i 0 0 %s %02i %02i 24 0\n' % (m['alias'][i], id_generator(), org + date.wwwwd() + '.GLX', date.yyyy()[2:], date.month, date.day, date.yyyy()[2:], date.month, date.day)) # ready to pass list to globk object Globk(globk_pwd, org, glx_list, date_s.wwww(), ' '.join([stn['StationCode'].upper() for stn in stnlist])) process_sinex(cnn, project, [date_s, date_e], globk_pwd + '/' + org + date_s.wwww() + '7.snx')
def main(): parser = argparse.ArgumentParser( description='Archive operations Main Program') parser.add_argument( 'stnlist', type=str, nargs='+', metavar='all|net.stnm', help= "List of networks/stations to process given in [net].[stnm] format or just [stnm] " "(separated by spaces; if [stnm] is not unique in the database, all stations with that " "name will be processed). Use keyword 'all' to process all stations in the database. " "If [net].all is given, all stations from network [net] will be processed. " "Alternatevily, a file with the station list can be provided.") parser.add_argument( '-date', '--date_range', nargs='+', action=required_length(1, 2), metavar='date_start|date_end', help="Date range to check given as [date_start] or [date_start] " "and [date_end]. Allowed formats are yyyy.doy or yyyy/mm/dd..") parser.add_argument( '-win', '--window', nargs=1, metavar='days', type=int, help= "Download data from a given time window determined by today - {days}.") try: args = parser.parse_args() cnn = dbConnection.Cnn('gnss_data.cfg') Config = pyOptions.ReadOptions('gnss_data.cfg') stnlist = Utils.process_stnlist(cnn, args.stnlist) print ' >> Selected station list:' print_columns([ item['NetworkCode'] + '.' + item['StationCode'] for item in stnlist ]) dates = [] try: if args.window: # today - ndays d = pyDate.Date(year=datetime.now().year, month=datetime.now().month, day=datetime.now().day) dates = [d - int(args.window[0]), d] else: dates = process_date(args.date_range) except ValueError as e: parser.error(str(e)) if dates[0] < pyDate.Date(gpsWeek=650, gpsWeekDay=0): dates = [ pyDate.Date(gpsWeek=650, gpsWeekDay=0), pyDate.Date(year=datetime.now().year, month=datetime.now().month, day=datetime.now().day) ] # go through the dates drange = np.arange(dates[0].mjd, dates[1].mjd + 1, 1) download_data(cnn, Config, stnlist, drange) except argparse.ArgumentTypeError as e: parser.error(str(e))
raise pyStationCollectionException('Requested station code ' + item.netstn + ' could not be found') else: raise pyStationException( 'type: ' + str(type(item)) + ' invalid. Can only pass Station or String objects.') def __contains__(self, item): return self.ismember(item) if __name__ == '__main__': import dbConnection cnn = dbConnection.Cnn('gnss_data.cfg') import pyDate dr = [pyDate.Date(year=2010, doy=1), pyDate.Date(year=2010, doy=2)] s1 = Station(cnn, 'rms', 'igm1', dr) s2 = Station(cnn, 'rms', 'lpgs', dr) s3 = Station(cnn, 'rms', 'chac', dr) s4 = Station(cnn, 'cap', 'chac', dr) c = StationCollection() c.append(s1) c.append(s2) c.append(s3) print c c.append(s4) print c c.replace_alias([s1, s2], ['zzz1', 'zzz1']) print c
def main(): parser = argparse.ArgumentParser( description= 'Database integrity tools, metadata check and fixing tools program') parser.add_argument( 'stnlist', type=str, nargs='+', metavar='all|net.stnm', help= "List of networks/stations to process given in [net].[stnm] format or just [stnm] " "(separated by spaces; if [stnm] is not unique in the database, all stations with that " "name will be processed). Use keyword 'all' to process all stations in the database. " "If [net].all is given, all stations from network [net] will be processed. " "Alternatevily, a file with the station list can be provided.") parser.add_argument( '-d', '--date_filter', nargs='+', metavar='date', help='Date range filter for all operations. ' 'Can be specified in wwww-d, yyyy_ddd, yyyy/mm/dd or fyear format') parser.add_argument( '-rinex', '--check_rinex', choices=['fix', 'report'], type=str, nargs=1, help= 'Check the RINEX integrity of the archive-database by verifying that the RINEX files ' 'reported in the rinex table exist in the archive. If argument = "fix" and a RINEX file ' 'does not exist, remove the record. PPP records or gamit_soln are deleted. If argument = ' '"report" then just list the missing files.') parser.add_argument( '-rnx_count', '--rinex_count', action='store_true', help='Count the total number of RINEX files (unique station-days) ' 'per day for a given time interval.') parser.add_argument( '-stnr', '--station_info_rinex', action='store_true', help= 'Check that the receiver serial number in the rinex headers agrees with the station info ' 'receiver serial number.') parser.add_argument( '-stns', '--station_info_solutions', action='store_true', help='Check that the PPP hash values match the station info hash.') parser.add_argument( '-stnp', '--station_info_proposed', metavar='ignore_days', const=0, type=int, nargs='?', help= 'Output a proposed station.info using the RINEX metadata. Optional, specify [ignore_days] ' 'to ignore station.info records <= days.') parser.add_argument( '-stnc', '--station_info_check', action='store_true', help= 'Check the consistency of the station information records in the database. Date range ' 'does not apply. Also, check that the RINEX files fall within a valid station information ' 'record.') parser.add_argument( '-g', '--data_gaps', metavar='ignore_days', const=0, type=int, nargs='?', help= 'Check the RINEX files in the database and look for gaps (missing days). ' 'Optional, [ignore_days] with the smallest gap to display.') parser.add_argument('-gg', '--graphical_gaps', action='store_true', help='Visually output RINEX gaps for stations.') parser.add_argument( '-sc', '--spatial_coherence', choices=['exclude', 'delete', 'noop'], type=str, nargs=1, help= 'Check that the RINEX files correspond to the stations they are linked to using their ' 'PPP coordinate. If keyword [exclude] or [delete], add the PPP solution to the excluded ' 'table or delete the PPP solution. If [noop], then only report but do not ' 'exlude or delete.') parser.add_argument( '-print', '--print_stninfo', choices=['long', 'short'], type=str, nargs=1, help= 'Output the station info to stdout. [long] outputs the full line of the station info. ' '[short] outputs a short version (better for screen visualization).') parser.add_argument( '-r', '--rename', metavar='net.stnm', nargs=1, help= "Takes the data from the station list and renames (merges) it to net.stnm. " "It also changes the rinex filenames in the archive to match those of the new destiny " "station. Only a single station can be given as the origin and destiny. " "Limit the date range using the -d option.") parser.add_argument( '-es', '--exclude_solutions', metavar=('{start_date}', '{end_date}'), nargs=2, help= 'Exclude PPP solutions (by adding them to the excluded table) between {start_date} ' 'and {end_date}') parser.add_argument( '-del', '--delete_rinex', metavar=('{start_date}', '{end_date}', '{completion}'), nargs=3, help='Delete RINEX files (and associated solutions, PPP and GAMIT) ' 'from archive between {start_date} and {end_date} with completion <= {completion}. ' 'Completion ranges form 1.0 to 0.0. Use 1.0 to delete all data. ' 'Operation cannot be undone!') parser.add_argument('-np', '--noparallel', action='store_true', help="Execute command without parallelization.") args = parser.parse_args() cnn = dbConnection.Cnn("gnss_data.cfg") # type: dbConnection.Cnn # create the execution log cnn.insert('executions', script='pyIntegrityCheck.py') Config = pyOptions.ReadOptions( "gnss_data.cfg") # type: pyOptions.ReadOptions stnlist = Utils.process_stnlist(cnn, args.stnlist) JobServer = pyJobServer.JobServer( Config, run_parallel=not args.noparallel) # type: pyJobServer.JobServer ##################################### # date filter dates = [pyDate.Date(year=1980, doy=1), pyDate.Date(year=2100, doy=1)] try: dates = process_date(args.date_filter) except ValueError as e: parser.error(str(e)) ##################################### if args.check_rinex: CheckRinexIntegrity(cnn, Config, stnlist, dates[0], dates[1], args.check_rinex[0], JobServer) ##################################### if args.rinex_count: RinexCount(cnn, stnlist, dates[0], dates[1]) ##################################### if args.station_info_rinex: StnInfoRinexIntegrity(cnn, stnlist, dates[0], dates[1], JobServer) ##################################### if args.station_info_check: StnInfoCheck(cnn, stnlist, Config) ##################################### if args.data_gaps is not None: GetStnGaps(cnn, stnlist, args.data_gaps, dates[0], dates[1]) if args.graphical_gaps: VisualizeGaps(cnn, stnlist, dates[0], dates[1]) ##################################### if args.spatial_coherence is not None: CheckSpatialCoherence(cnn, stnlist, dates[0], dates[1]) ##################################### if args.exclude_solutions is not None: try: dates = process_date(args.exclude_solutions) except ValueError as e: parser.error(str(e)) ExcludeSolutions(cnn, stnlist, dates[0], dates[1]) ##################################### if args.print_stninfo is not None: if args.print_stninfo[0] == 'short': PrintStationInfo(cnn, stnlist, True) elif args.print_stninfo[0] == 'long': PrintStationInfo(cnn, stnlist, False) else: parser.error( 'Argument for print_stninfo has to be either long or short') ##################################### if args.station_info_proposed is not None: for stn in stnlist: stninfo = pyStationInfo.StationInfo(cnn, stn['NetworkCode'], stn['StationCode'], allow_empty=True) sys.stdout.write( stninfo.rinex_based_stninfo(args.station_info_proposed)) ##################################### if args.delete_rinex is not None: try: dates = process_date(args.delete_rinex[0:2]) except ValueError as e: parser.error(str(e)) DeleteRinex(cnn, stnlist, dates[0], dates[1], float(args.delete_rinex[2])) ##################################### if args.rename: if len(stnlist) > 1: parser.error( 'Only a single station should be given for the origin station') if '.' not in args.rename[0]: parser.error('Format for destiny station should be net.stnm') else: DestNetworkCode = args.rename[0].split('.')[0] DestStationCode = args.rename[0].split('.')[1] RenameStation(cnn, stnlist[0]['NetworkCode'], stnlist[0]['StationCode'], DestNetworkCode, DestStationCode, dates[0], dates[1], Config.archive_path) JobServer.close_cluster()