def read_survey_config_fn(self, survey_config_fn=None): """ read in survey configuration file and output into a useful dictionary """ if survey_config_fn is not None: self.surve_config_fn = survey_config_fn if not os.path.isfile(self.survey_config_fn): raise mtex.MTpyError_inputarguments( 'Could not find {0}, check path'.format(survey_config_fn)) # read in survey information self.survey_config_dict = mtcfg.read_survey_configfile( self.survey_config_fn)[self.station]
def main(): if len(sys.argv) < 3: sys.exit('\nNeed at least 2 arguments:\n <path to files> \n ' '<config file> \n ' '[optional:<output dir>] \n [optional:<station>] \n ' '[optional:<recursive flag -R>] \n ' '[optional:<re-orientation flag -O]\n\n') outdir = None stationname = None recursive = False orientation = False if len(sys.argv) > 3: optionals = sys.argv[3:] for o in optionals: o = o.strip() if o[0] == '-': if o[1].lower() == 'r': recursive = True elif o[1].lower() == 'o': orientation = True continue elif outdir is None: outdir = o continue elif stationname is None: stationname = o.upper() continue pathname_raw = sys.argv[1] pathname = op.abspath(op.realpath(pathname_raw)) if not op.isdir(pathname): sys.exit('Data file(s) path not existing: {0}'.format(pathname)) configfilename_raw = sys.argv[2] configfile = op.abspath(op.realpath(op.join(os.curdir, configfilename_raw))) if not op.isfile(configfile): sys.exit('Config file not found: {0}'.format(configfile)) if recursive is True: lo_dirs = [pathname] for i, j, k in os.walk(pathname): lof = [op.abspath(op.join(i, f)) for f in j] lo_dirs.extend(lof) pathname = list(set(lo_dirs)) else: pathname = [pathname] #config_dict = MTcf.read_survey_configfile(configfile) try: config_dict = MTcf.read_survey_configfile(configfile) # done internally already # MTcf.validate_dict(config_dict) except: sys.exit( 'Config file invalid or cannot be read: {0}'.format(configfile)) #------------------------------------------------------------------------- # select files by header entries: components = ['ex', 'ey', 'bx', 'by', 'bz'] lo_allfiles = [] lo_allheaders = [] lo_allstations = [] for folder in pathname: wd = op.abspath(op.realpath(folder)) if not op.isdir(wd): # print 'Directory not existing: %s' % (wd) lo_foldernames.remove(wd) continue dirfiles = [op.abspath(op.join(wd, i)) for i in os.listdir(wd)] for tmpfile in dirfiles: try: header = MTfh.read_ts_header(tmpfile) if header['channel'].lower() in components: if stationname is not None: if stationname.upper() != header['station'].upper(): continue lo_allstations.append(header['station'].upper()) lo_allfiles.append(op.abspath(op.join(wd, tmpfile))) lo_allheaders.append(header) except: continue lo_allstations = list(set(lo_allstations)) # check, if list of files is empty if len(lo_allfiles) == 0: sys.exit('Directory(ies) do(es) not contain files to calibrate:' ' {0}'.format(pathname)) #------------------------------------------------------- # set up the directory structure for the output: # 1. generic calibration output directory cal_outdir = op.abspath(op.join(pathname[0], 'calibrated')) if outdir is not None: try: cal_outdir = op.abspath(op.join(os.curdir, outdir)) if not op.isdir(cal_outdir): os.makedirs(cal_outdir) print 'generated ', cal_outdir except: print 'Output directory cannot be generated: '\ '{0} - using generic location'.format(cal_outdir) cal_outdir = op.abspath(op.join(pathname[0], 'calibrated')) try: if not op.isdir(cal_outdir): os.makedirs(cal_outdir) except: # this only comes up, if the generic location cannot be generated sys.exit( 'Generic directory cannot be generated: {0}'.format(cal_outdir)) print '\t Output directory ok: {0}\n'.format(cal_outdir) # if re-orientation is required, do it first: if orientation is True: print '\n\t....re-orient data first...\n' ori_outdir = op.abspath(op.join(cal_outdir, '../reoriented_tmp')) try: if not op.isdir(ori_outdir): os.makedirs(ori_outdir) except: # this only comes up, if the generic location cannot be generated sys.exit('Re-orientation directory cannot be generated:' ' {0}'.format(ori_outdir)) MTfh.reorient_files(lo_allfiles, configfile, lo_stations=lo_allstations, outdir=ori_outdir) # change to calibration setup : outdir = cal_outdir new_inputdir = ori_outdir # file structure has changed, so the re-oriented files have to be read # again: components = ['ex', 'ey', 'bx', 'by', 'bz'] lo_allfiles = [] lo_allheaders = [] lo_allstations = [] dirfiles = [ op.abspath(op.join(new_inputdir, i)) for i in os.listdir(new_inputdir) ] for tmpfile in dirfiles: header = MTfh.read_ts_header(tmpfile) lo_allstations.append(header['station'].upper()) lo_allfiles.append(tmpfile) lo_allheaders.append(header) lo_allstations = list(set(lo_allstations)) # check, if list of files is empty if len(lo_allfiles) == 0: sys.exit('Directory(ies) do(es) not contain files to calibrate:' ' {0}'.format(ori_outdir)) #------------------------------------------------- # calibration lo_calibrated_files = [] lo_calibrated_stations = [] for file_idx, filename in enumerate(lo_allfiles): curr_station = lo_allheaders[file_idx]['station'].upper() if stationname is not None: if stationname.upper() != curr_station.upper(): continue print 'reading file {0}...'.format(filename) channel = lo_allheaders[file_idx]['channel'] lo_calibrated_stations.append(curr_station) # get configuration dictionary for this station try: stationdict = config_dict[curr_station] except: print 'no entry for station {0} found in configuration file'\ ' {1} skipping file'.format(curr_station, configfile) continue latitude = float(stationdict['latitude']) longitude = float(stationdict['longitude']) elevation = float(stationdict['elevation']) field = channel[0] direction = channel[1] station_type = stationdict['station_type'] if field == 'e': if station_type == 'b': continue # check North-South axis orientation if direction == 'x': #angle = float(stationdict['e_xaxis_azimuth']) dipolelength = float(stationdict['e_xaxis_length']) # check East-West axis orientation if direction == 'y': #angle = float(stationdict['e_yaxis_azimuth']) dipolelength = float(stationdict['e_yaxis_length']) logger = stationdict['e_logger_type'] gain = float(stationdict['e_logger_gain']) instrument = stationdict.get('e_instrument_type', 'electrodes') instrument_amplification = float( stationdict['e_instrument_amplification']) elif field == 'b': if station_type == 'e': continue dipolelength = 1. logger = stationdict['b_logger_type'] gain = float(stationdict['b_logger_gain']) instrument = stationdict.get('b_instrument_type', 'coils') instrument_amplification = float( stationdict['b_instrument_amplification']) MTcb.calibrate_file(filename, outdir, instrument, instrument_amplification, logger, gain, dipolelength, curr_station, channel, latitude, longitude, elevation, offset=0) # print 'calibrated file {0},{1}'.format(outdir, filename) lo_calibrated_files.append(filename) lo_calibrated_stations = list(set(lo_calibrated_stations)) if len(lo_calibrated_files) == 0: if stationname is not None: print 'No files found for station {0}'.format(stationname) return else: print 'No files found for stations {0}'.format(lo_allstations) print '{0} files calibrated for stations'\ ' {1}'.format(len(lo_calibrated_files), lo_calibrated_stations)
def reorient_files(lo_files, configfile, lo_stations=None, outdir=None): #read config file try: config_dict = MTcf.read_survey_configfile(configfile) except: raise MTex.MTpyError_config_file('Config file cannot be read:' ' {0}'.format(configfile)) if lo_stations is not None: try: if type(lo_stations) == str: raise #check, if it's iterable: dummy = [i for i in lo_stations] except: raise MTex.MTpyError_inputarguments('ERROR - "lo_stations"' ' argument must be iterable!') print '\t re-orienting data for collection of stations:\n{0}'.format( lo_stations) #Do not require list of headers as input, as this function can be called directly rather than from a 'calibratefiles.py'-like script - so the list not necessarily exists in beforehand - #collect header lines of files in list lo_headers = [] lo_stationnames = [] for file_idx, filename in enumerate(lo_files): header = read_ts_header(filename) station = header['station'] if station.upper() not in [i.upper() for i in lo_stations]: #TODO: check, if this causes problems with the indices for the current loop: lo_files.remove(filename) continue lo_headers.append(header) lo_stationnames.append(station.upper()) if len(lo_headers) == 0: if lo_stations is not None: print 'ERROR - No files with header lines found for station(s)'\ ' {0}'.format(lo_stations) else: print 'ERROR - No files with header lines found' return 1 lo_stationnames = list(set(lo_stationnames)) # set up output directory ori_outdir = op.abspath(op.join(os.curdir, 'reoriented')) if outdir is not None: try: ori_outdir = op.abspath(op.join(os.curdir, outdir)) if not op.isdir(ori_outdir): os.makedirs(ori_outdir) except: print 'Output directory cannot be generated: {0} - using generic'\ ' location'.format(ori_outdir) ori_outdir = op.abspath(op.join(os.curdir, 'reoriented')) try: if not op.isdir(ori_outdir): os.makedirs(ori_outdir) except: #this only comes up, if the generic location cannot be generated raise MTex.MTpyError_inputarguments( 'Generic directory cannot be' ' generated: {0}'.format(ori_outdir)) #---------------------- #start re-orientation #present: list of all files, list of all headers, list of all stations for sta_idx, sta in enumerate(lo_stationnames): #print sta try: stationconfig = config_dict[sta] except: print 'Warning - No config file entry for station {0} -'\ ' no processing possible'.format(sta) continue declination = float(stationconfig.get('declination', 0.)) for sensor in ['e', 'b']: #TODO: # reduce this function to the re-orientation of files that have the same length for X and Y. #Do the puzzlling for varying lengths later!! for idx_h_x, header_x in enumerate(lo_headers): #looking for one specific station if not header_x['station'].upper() == sta.upper(): continue #looking for the specific sensor type if not header_x['channel'].lower()[0] == sensor: continue #looking for the X channel (the to-be-North) if not header_x['channel'].lower()[1] == 'x': continue x_file = lo_files[idx_h_x] x_header_string = get_ts_header_string(header_x) t0 = float(header_x['t_min']) #print t0 #now look for the respective y-file and possible z-file - unfortunately by another loop over all headers: y_file = None z_file = None for idx_h_y, header_y in enumerate(lo_headers): if (header_y['station'].upper() == sta.upper()) and \ (header_y['channel'].lower()[0] == sensor) and \ (float(header_y['t_min']) == float(header_x['t_min'] ) ): if (header_y['channel'].lower()[1] == 'y'): y_file = lo_files[idx_h_y] y_header_string = get_ts_header_string(header_y) elif (header_y['channel'].lower()[1] == 'z'): z_file = lo_files[idx_h_y] else: continue if y_file == None: continue x_outfn = op.abspath(op.join(ori_outdir, op.basename(x_file))) y_outfn = op.abspath(op.join(ori_outdir, op.basename(y_file))) if z_file is not None: z_outfn = op.abspath( op.join(ori_outdir, op.basename(z_file))) xdata = np.loadtxt(x_file) ydata = np.loadtxt(y_file) #declination is positive, if magnetic North is east of true North. # the measured angles are w.r.t. magnetic North, so the given # azimuths do not include the declination #-> thus the declination value is added to azimuths if sensor == 'e': xangle = float(stationconfig.get('e_xaxis_azimuth', 0.)) + declination yangle = float(stationconfig.get('e_yaxis_azimuth', 90.)) + declination else: xangle = float(stationconfig.get('b_xaxis_azimuth', 0.)) + declination yangle = float(stationconfig.get('b_yaxis_azimuth', 90.)) + declination newx, newy = MTcc.reorient_data2D(xdata, ydata, x_sensor_angle=xangle, y_sensor_angle=yangle) #print xdata.shape, ydata.shape, newx.shape, newy.shape #continue outFx = open(x_outfn, 'w') outFx.write(x_header_string) np.savetxt(outFx, newx) outFx.close() outFy = open(y_outfn, 'w') outFy.write(y_header_string) np.savetxt(outFy, newy) outFy.close() written_files = [x_outfn, y_outfn] if z_file is not None: shutil.copyfile(z_file, z_outfn) written_files.append(z_outfn) print '\tSuccessfullly written files {0}'.format(written_files) return 0
def reorient_files(lo_files, configfile, lo_stations = None, outdir = None): #read config file try: config_dict = MTcf.read_survey_configfile(configfile) except: raise MTex.MTpyError_config_file( 'Config file cannot be read:' ' {0}'.format(configfile) ) if lo_stations is not None: try: if type(lo_stations) == str: raise #check, if it's iterable: dummy = [i for i in lo_stations] except: raise MTex.MTpyError_inputarguments('ERROR - "lo_stations"' ' argument must be iterable!') print '\t re-orienting data for collection of stations:\n{0}'.format(lo_stations) #Do not require list of headers as input, as this function can be called directly rather than from a 'calibratefiles.py'-like script - so the list not necessarily exists in beforehand - #collect header lines of files in list lo_headers = [] lo_stationnames = [] for file_idx, filename in enumerate(lo_files): header = read_ts_header(filename) station = header['station'] if station.upper() not in [i.upper() for i in lo_stations]: #TODO: check, if this causes problems with the indices for the current loop: lo_files.remove(filename) continue lo_headers.append(header) lo_stationnames.append(station.upper()) if len(lo_headers) == 0 : if lo_stations is not None: print 'ERROR - No files with header lines found for station(s)'\ ' {0}'.format(lo_stations) else: print 'ERROR - No files with header lines found' return 1 lo_stationnames = list(set(lo_stationnames)) # set up output directory ori_outdir = op.abspath(op.join(os.curdir,'reoriented')) if outdir is not None: try: ori_outdir = op.abspath(op.join(os.curdir,outdir)) if not op.isdir(ori_outdir): os.makedirs(ori_outdir) except: print 'Output directory cannot be generated: {0} - using generic'\ ' location'.format(ori_outdir) ori_outdir = op.abspath(op.join(os.curdir,'reoriented')) try: if not op.isdir(ori_outdir): os.makedirs(ori_outdir) except: #this only comes up, if the generic location cannot be generated raise MTex.MTpyError_inputarguments('Generic directory cannot be' ' generated: {0}'.format(ori_outdir)) #---------------------- #start re-orientation #present: list of all files, list of all headers, list of all stations for sta_idx, sta in enumerate(lo_stationnames): #print sta try: stationconfig = config_dict[sta] except: print 'Warning - No config file entry for station {0} -'\ ' no processing possible'.format(sta) continue declination = float(stationconfig.get('declination',0.)) for sensor in ['e','b']: #TODO: # reduce this function to the re-orientation of files that have the same length for X and Y. #Do the puzzlling for varying lengths later!! for idx_h_x, header_x in enumerate(lo_headers): #looking for one specific station if not header_x['station'].upper() == sta.upper(): continue #looking for the specific sensor type if not header_x['channel'].lower()[0] == sensor: continue #looking for the X channel (the to-be-North) if not header_x['channel'].lower()[1] == 'x': continue x_file = lo_files[idx_h_x] x_header_string = get_ts_header_string(header_x) t0 = float(header_x['t_min']) #print t0 #now look for the respective y-file and possible z-file - unfortunately by another loop over all headers: y_file = None z_file = None for idx_h_y, header_y in enumerate(lo_headers): if (header_y['station'].upper() == sta.upper()) and \ (header_y['channel'].lower()[0] == sensor) and \ (float(header_y['t_min']) == float(header_x['t_min'] ) ): if (header_y['channel'].lower()[1] == 'y') : y_file = lo_files[idx_h_y] y_header_string = get_ts_header_string(header_y) elif (header_y['channel'].lower()[1] == 'z') : z_file = lo_files[idx_h_y] else: continue if y_file == None: continue x_outfn = op.abspath(op.join(ori_outdir,op.basename(x_file))) y_outfn = op.abspath(op.join(ori_outdir,op.basename(y_file))) if z_file is not None: z_outfn = op.abspath(op.join(ori_outdir,op.basename(z_file))) xdata = np.loadtxt(x_file) ydata = np.loadtxt(y_file) #declination is positive, if magnetic North is east of true North. # the measured angles are w.r.t. magnetic North, so the given # azimuths do not include the declination #-> thus the declination value is added to azimuths if sensor == 'e': xangle = float(stationconfig.get( 'e_xaxis_azimuth', 0.)) + declination yangle = float(stationconfig.get( 'e_yaxis_azimuth',90.)) + declination else: xangle = float(stationconfig.get( 'b_xaxis_azimuth', 0.)) + declination yangle = float(stationconfig.get( 'b_yaxis_azimuth',90.)) + declination newx, newy = MTcc.reorient_data2D(xdata, ydata, x_sensor_angle = xangle , y_sensor_angle = yangle) #print xdata.shape, ydata.shape, newx.shape, newy.shape #continue outFx = open(x_outfn,'w') outFx.write(x_header_string) np.savetxt(outFx,newx) outFx.close() outFy = open(y_outfn,'w') outFy.write(y_header_string) np.savetxt(outFy,newy) outFy.close() written_files = [x_outfn,y_outfn] if z_file is not None: shutil.copyfile(z_file, z_outfn) written_files.append(z_outfn) print '\tSuccessfullly written files {0}'.format(written_files) return 0
def main(): if len(sys.argv) < 3: sys.exit('\nNeed at least 2 arguments:\n <path to files> \n ' '<config file> \n ' '[optional:<output dir>] \n [optional:<station>] \n ' '[optional:<recursive flag -R>] \n ' '[optional:<re-orientation flag -O]\n\n') outdir = None stationname = None recursive = False orientation = False if len(sys.argv) > 3: optionals = sys.argv[3:] for o in optionals: o = o.strip() if o[0] == '-': if o[1].lower() == 'r': recursive = True elif o[1].lower() == 'o': orientation = True continue elif outdir is None: outdir = o continue elif stationname is None: stationname = o.upper() continue pathname_raw = sys.argv[1] pathname = op.abspath(op.realpath(pathname_raw)) if not op.isdir(pathname): sys.exit('Data file(s) path not existing: {0}'.format(pathname)) configfilename_raw = sys.argv[2] configfile = op.abspath(op.realpath(op.join(os.curdir,configfilename_raw))) if not op.isfile(configfile): sys.exit('Config file not found: {0}'.format(configfile)) if recursive is True: lo_dirs = [pathname] for i,j,k in os.walk(pathname): lof = [op.abspath(op.join(i,f)) for f in j] lo_dirs.extend(lof) pathname = list(set(lo_dirs)) else: pathname = [pathname] #config_dict = MTcf.read_survey_configfile(configfile) try: config_dict = MTcf.read_survey_configfile(configfile) #done internally already #MTcf.validate_dict(config_dict) except: sys.exit( 'Config file invalid or cannot be read: {0}'.format(configfile) ) #---------------------------------------------------------------------------- #select files by header entries: components = ['ex', 'ey', 'bx', 'by', 'bz'] lo_allfiles = [] lo_allheaders = [] lo_allstations = [] for folder in pathname: wd = op.abspath(op.realpath(folder)) if not op.isdir(wd): #print 'Directory not existing: %s' % (wd) lo_foldernames.remove(wd) continue dirfiles = [op.abspath(op.join(wd,i)) for i in os.listdir(wd)] for tmpfile in dirfiles: try: header = MTfh.read_ts_header(tmpfile) if header['channel'].lower() in components: if stationname is not None: if stationname.upper() != header['station'].upper(): continue lo_allstations.append(header['station'].upper()) lo_allfiles.append(op.abspath(op.join(wd,tmpfile))) lo_allheaders.append(header) except: continue lo_allstations = list(set(lo_allstations)) #check, if list of files is empty if len(lo_allfiles) == 0: sys.exit('Directory(ies) do(es) not contain files to calibrate:' ' {0}'.format(pathname)) #------------------------------------------------------- # set up the directory structure for the output: #1. generic calibration output directory cal_outdir = op.abspath(op.join(pathname[0],'calibrated')) if outdir is not None: try: cal_outdir = op.abspath(op.join(os.curdir,outdir)) if not op.isdir(cal_outdir): os.makedirs(cal_outdir) print('generated ', cal_outdir) except: print('Output directory cannot be generated: '\ '{0} - using generic location'.format(cal_outdir)) cal_outdir = op.abspath(op.join(pathname[0],'calibrated')) try: if not op.isdir(cal_outdir): os.makedirs(cal_outdir) except: #this only comes up, if the generic location cannot be generated sys.exit('Generic directory cannot be generated: {0}'.format(cal_outdir)) print('\t Output directory ok: {0}\n'.format(cal_outdir)) #if re-orientation is required, do it first: if orientation is True: print('\n\t....re-orient data first...\n') ori_outdir = op.abspath(op.join(cal_outdir,'../reoriented_tmp')) try: if not op.isdir(ori_outdir): os.makedirs(ori_outdir) except: #this only comes up, if the generic location cannot be generated sys.exit('Re-orientation directory cannot be generated:' ' {0}'.format(ori_outdir)) MTfh.reorient_files(lo_allfiles, configfile, lo_stations = lo_allstations, outdir = ori_outdir) #change to calibration setup : outdir = cal_outdir new_inputdir = ori_outdir #file structure has changed, so the re-oriented files have to be read again: components = ['ex', 'ey', 'bx', 'by', 'bz'] lo_allfiles = [] lo_allheaders = [] lo_allstations = [] dirfiles = [op.abspath(op.join(new_inputdir,i)) for i in os.listdir(new_inputdir) ] for tmpfile in dirfiles: header = MTfh.read_ts_header(tmpfile) lo_allstations.append(header['station'].upper()) lo_allfiles.append(tmpfile) lo_allheaders.append(header) lo_allstations = list(set(lo_allstations)) #check, if list of files is empty if len(lo_allfiles) == 0: sys.exit('Directory(ies) do(es) not contain files to calibrate:' ' {0}'.format(ori_outdir)) #------------------------------------------------- #calibration lo_calibrated_files = [] lo_calibrated_stations = [] for file_idx, filename in enumerate(lo_allfiles): curr_station = lo_allheaders[file_idx]['station'].upper() if stationname is not None: if stationname.upper() != curr_station.upper(): continue print('reading file {0}...'.format(filename)) channel = lo_allheaders[file_idx]['channel'] lo_calibrated_stations.append(curr_station) #get configuration dictionary for this station try: stationdict = config_dict[curr_station] except: print('no entry for station {0} found in configuration file'\ ' {1} skipping file'.format(curr_station, configfile )) continue latitude = float(stationdict['latitude']) longitude = float(stationdict['longitude']) elevation = float(stationdict['elevation']) field = channel[0] direction = channel[1] station_type = stationdict['station_type'] if field == 'e': if station_type == 'b': continue #check North-South axis orientation if direction == 'x': #angle = float(stationdict['e_xaxis_azimuth']) dipolelength = float(stationdict['e_xaxis_length']) #check East-West axis orientation if direction == 'y': #angle = float(stationdict['e_yaxis_azimuth']) dipolelength = float(stationdict['e_yaxis_length']) logger = stationdict['e_logger_type'] gain = float(stationdict['e_logger_gain']) instrument = stationdict.get('e_instrument_type','electrodes') instrument_amplification = float(stationdict['e_instrument_amplification']) elif field == 'b': if station_type == 'e': continue dipolelength = 1. logger = stationdict['b_logger_type'] gain = float(stationdict['b_logger_gain']) instrument = stationdict.get('b_instrument_type','coils') instrument_amplification = float(stationdict['b_instrument_amplification']) MTcb.calibrate_file(filename, outdir, instrument, instrument_amplification, logger, gain, dipolelength, curr_station, channel, latitude, longitude, elevation, offset = 0 ) #print 'calibrated file {0},{1}'.format(outdir, filename) lo_calibrated_files.append(filename) lo_calibrated_stations = list(set(lo_calibrated_stations)) if len(lo_calibrated_files) == 0: if stationname is not None: print('No files found for station {0}'.format(stationname)) return else: print('No files found for stations {0}'.format(lo_allstations)) print('{0} files calibrated for stations'\ ' {1}'.format(len(lo_calibrated_files),lo_calibrated_stations))