def convert_dms_tuple2degrees(latlon_list): """ Convert a triple (list, tuple, array) of degrees, minuts, seconds into degrees. Validity of the triple is assumed and has to be asserted in advance. """ sign = 1. try: latlon_list = [float(i) for i in latlon_list] if str(latlon_list[0]).startswith('-'): sign = -1. except: # if triple is given as string: latlon_raw = latlon_list # allow separation by :,. or space for (deg min sec) format try: latlon_list = re.split('[ :,]', latlon_raw) except: raise MTex.MTpyError_config_file( 'Config file error: lat/lon is in invalid format') try: latlon_list = [float(i) for i in latlon_list] except: raise MTex.MTpyError_config_file( 'Config file error: lat/lon is in invalid format') if str(latlon_list[0])[0] == '-': sign = -1. deg = latlon_list[0] minutes = latlon_list[1] seconds = latlon_list[2] if not (-180 <= deg <= 360 and 0 <= minutes < 60 and 0 <= seconds < 60): print 'Value for degrees, minutes, or seconds invalid' raise MTex.MTpyError_inputarguments() # take out sign for easier conversion into degrees if deg < 0: deg *= -1 degrees = deg + 1 / 60. * minutes + 1 / 3600. * seconds return degrees * sign
def _validate_dictionary(dict2validate, referencedict): """Check, if there are lists of allowed entries for all keys of the current dictionary. If yes, test, if the current value is among the allowed ones. """ for key, value in dict2validate.items(): #make everything to strings - easier to compare #in case of numbers, make to float first try: allowed_vals = referencedict[key] except: try: key = key.lower() allowed_vals = referencedict[key] except: #no reference entry found - skip key continue tmp = [] #allowed values must be given as a list (iterable)!! for i in allowed_vals: try: tmp.append(str(float(i))) except: tmp.append(str(i)) tmp = [i.lower() for i in tmp] allowed_vals = tmp #compare case-insensitive value = value.lower() if not value in allowed_vals: raise MTex.MTpyError_config_file( 'Config file error --' ' key {0}, value {1} not valid'.format(key, value))
def reorient_files(lo_files, configfile, lo_stations=None, outdir=None): #read config file try: config_dict = MTcf.read_survey_configfile(configfile) except: raise MTex.MTpyError_config_file('Config file cannot be read:' ' {0}'.format(configfile)) if lo_stations is not None: try: if type(lo_stations) == str: raise #check, if it's iterable: dummy = [i for i in lo_stations] except: raise MTex.MTpyError_inputarguments('ERROR - "lo_stations"' ' argument must be iterable!') print '\t re-orienting data for collection of stations:\n{0}'.format( lo_stations) #Do not require list of headers as input, as this function can be called directly rather than from a 'calibratefiles.py'-like script - so the list not necessarily exists in beforehand - #collect header lines of files in list lo_headers = [] lo_stationnames = [] for file_idx, filename in enumerate(lo_files): header = read_ts_header(filename) station = header['station'] if station.upper() not in [i.upper() for i in lo_stations]: #TODO: check, if this causes problems with the indices for the current loop: lo_files.remove(filename) continue lo_headers.append(header) lo_stationnames.append(station.upper()) if len(lo_headers) == 0: if lo_stations is not None: print 'ERROR - No files with header lines found for station(s)'\ ' {0}'.format(lo_stations) else: print 'ERROR - No files with header lines found' return 1 lo_stationnames = list(set(lo_stationnames)) # set up output directory ori_outdir = op.abspath(op.join(os.curdir, 'reoriented')) if outdir is not None: try: ori_outdir = op.abspath(op.join(os.curdir, outdir)) if not op.isdir(ori_outdir): os.makedirs(ori_outdir) except: print 'Output directory cannot be generated: {0} - using generic'\ ' location'.format(ori_outdir) ori_outdir = op.abspath(op.join(os.curdir, 'reoriented')) try: if not op.isdir(ori_outdir): os.makedirs(ori_outdir) except: #this only comes up, if the generic location cannot be generated raise MTex.MTpyError_inputarguments( 'Generic directory cannot be' ' generated: {0}'.format(ori_outdir)) #---------------------- #start re-orientation #present: list of all files, list of all headers, list of all stations for sta_idx, sta in enumerate(lo_stationnames): #print sta try: stationconfig = config_dict[sta] except: print 'Warning - No config file entry for station {0} -'\ ' no processing possible'.format(sta) continue declination = float(stationconfig.get('declination', 0.)) for sensor in ['e', 'b']: #TODO: # reduce this function to the re-orientation of files that have the same length for X and Y. #Do the puzzlling for varying lengths later!! for idx_h_x, header_x in enumerate(lo_headers): #looking for one specific station if not header_x['station'].upper() == sta.upper(): continue #looking for the specific sensor type if not header_x['channel'].lower()[0] == sensor: continue #looking for the X channel (the to-be-North) if not header_x['channel'].lower()[1] == 'x': continue x_file = lo_files[idx_h_x] x_header_string = get_ts_header_string(header_x) t0 = float(header_x['t_min']) #print t0 #now look for the respective y-file and possible z-file - unfortunately by another loop over all headers: y_file = None z_file = None for idx_h_y, header_y in enumerate(lo_headers): if (header_y['station'].upper() == sta.upper()) and \ (header_y['channel'].lower()[0] == sensor) and \ (float(header_y['t_min']) == float(header_x['t_min'] ) ): if (header_y['channel'].lower()[1] == 'y'): y_file = lo_files[idx_h_y] y_header_string = get_ts_header_string(header_y) elif (header_y['channel'].lower()[1] == 'z'): z_file = lo_files[idx_h_y] else: continue if y_file == None: continue x_outfn = op.abspath(op.join(ori_outdir, op.basename(x_file))) y_outfn = op.abspath(op.join(ori_outdir, op.basename(y_file))) if z_file is not None: z_outfn = op.abspath( op.join(ori_outdir, op.basename(z_file))) xdata = np.loadtxt(x_file) ydata = np.loadtxt(y_file) #declination is positive, if magnetic North is east of true North. # the measured angles are w.r.t. magnetic North, so the given # azimuths do not include the declination #-> thus the declination value is added to azimuths if sensor == 'e': xangle = float(stationconfig.get('e_xaxis_azimuth', 0.)) + declination yangle = float(stationconfig.get('e_yaxis_azimuth', 90.)) + declination else: xangle = float(stationconfig.get('b_xaxis_azimuth', 0.)) + declination yangle = float(stationconfig.get('b_yaxis_azimuth', 90.)) + declination newx, newy = MTcc.reorient_data2D(xdata, ydata, x_sensor_angle=xangle, y_sensor_angle=yangle) #print xdata.shape, ydata.shape, newx.shape, newy.shape #continue outFx = open(x_outfn, 'w') outFx.write(x_header_string) np.savetxt(outFx, newx) outFx.close() outFy = open(y_outfn, 'w') outFy.write(y_header_string) np.savetxt(outFy, newy) outFy.close() written_files = [x_outfn, y_outfn] if z_file is not None: shutil.copyfile(z_file, z_outfn) written_files.append(z_outfn) print '\tSuccessfullly written files {0}'.format(written_files) return 0
def read_survey_configfile(filename): """ Read in a survey configuration file and return a dictionary. Input config file must contain station names as section headers! The output dictionary keys are station names (capitalised), the values are (sub-)dictionaries. The configuration file must contain sections for all stations, each containing all mandatory keywords: - latitude (deg) - longitude (deg) - elevation (in meters) - sampling_interval (in seconds) - station_type (MT, E, B) Not mandatory, but recommended - declination (in degrees, positive to East) - this is set to '0.0', if omitted Depending on the type of station the following entries are required. E-field recorded: - E_logger_type ('edl'/'elogger'/'qel') - E_logger_gain (factor/gain-level) - E_instrument_type ('electrodes'/'dipole') - E_instrument_amplification (applied amplification factor) - E_Xaxis_azimuth (degrees) - E_Xaxis_length (in meters) - E_Yaxis_azimuth (degrees) - E_Yaxis_length (in meters) B-field recorded: - B_logger_type ('edl'/'qel_blogger') - B_logger_gain (factor/gain level) - B_instrument_type ('coil(s)', 'fluxgate') - B_instrument_amplification (applied amplification factor) - B_Xaxis_azimuth (degrees) - B_Yaxis_azimuth (degrees) """ error_counter = 0 #generate config parser instance configobject = ConfigParser.ConfigParser() #check, if file is present if not op.isfile(filename): raise MTex.MTpyError_inputarguments('File does not' ' exist: {0}'.format(filename)) # try to parse file - exit, if not a config file try: configobject.read(filename) except: raise MTex.MTpyError_inputarguments( 'File is not a ' 'proper configuration file: {0}'.format(filename)) #obtain dict of dicts containing the input file's sections (station names) #excludes DEFAULT section and key-value pairs without section header configobject_dict = configobject._sections #initialise the output dictionary config_dict = {} #loop over the sections (stations) of the config file for station in configobject_dict: #read in the sub-dictionary for the current station - bringing all keys #to lowercase! temp_dict_in = dict( (k.lower(), v) for k, v in configobject_dict[station].items()) #initialise output sub-directory for current station stationdict = temp_dict_in #stationnames are uppercase in MTpy stationname = station.upper() stationdict['station'] = stationname #check for presence of all mandatory keywords for the current station #case insensitive - allow for short forms 'lat', 'lon', and 'ele' try: for req_keyword in list_of_required_keywords: if req_keyword.lower() in temp_dict_in.keys(): stationdict[req_keyword.lower()] = \ temp_dict_in[req_keyword.lower()].lower() elif req_keyword in ['latitude', 'longitude', 'elevation']: if req_keyword[:3] in temp_dict_in.keys(): stationdict[req_keyword] = temp_dict_in[ req_keyword[:3]] else: print 'Station {0} - keyword {1} missing'.format( stationname, req_keyword) error_counter += 1 continue #check format of lat/lon - convert to degrees, if given in #(deg,min,sec)-triple for coordinate in ['latitude', 'longitude', 'elevation']: value = stationdict[coordinate] try: new_value = MTft._assert_position_format(coordinate, value) except: raise MTex.MTpyError_config_file( 'Error - wrong ' 'coordinate format for station {0}'.format( stationname)) stationdict[coordinate] = new_value if not stationdict['station_type'] in list_of_station_types: raise MTex.MTpyError_config_file('Station type not valid') except: print 'Missing information on station {0} in config file - skipping'.format( station) continue if stationdict['station_type'] in ['mt', 'e']: #check for required electric field parameters for req_keyword in list_of_efield_keywords: if req_keyword.lower() in temp_dict_in.keys(): stationdict[req_keyword.lower()] = \ temp_dict_in[req_keyword.lower()].lower() else: print 'Station {0} - keyword {1} missing'.format( stationname, req_keyword) error_counter += 1 continue _validate_dictionary(stationdict, dict_of_allowed_values_efield) if stationdict['station_type'] in ['mt', 'b']: #check for required magnetic field parameters for req_keyword in list_of_bfield_keywords: if req_keyword.lower() in temp_dict_in.keys(): stationdict[req_keyword.lower()] = \ temp_dict_in[req_keyword.lower()].lower() else: print 'Station {0} - keyword {1} missing'.format( stationname, req_keyword) error_counter += 1 continue _validate_dictionary(stationdict, dict_of_allowed_values_bfield) #add the station's sub-dictionary to the config dictionary config_dict[stationname] = stationdict #re-loop for setting up correct remote reference station information : #if rem.ref. station key is present, its information must be contained #in the same config file! for station in config_dict.iterkeys(): stationdict = config_dict[station] if not stationdict.has_key('rr_station'): continue #stationdict['rr_station'] = None stationdict['rr_station_latitude'] = None stationdict['rr_station_longitude'] = None stationdict['rr_station_elevation'] = None rem_station = stationdict['rr_station'] try: #check, if values are contained in dict float(stationdict['rr_station_latitude']) float(stationdict['rr_station_longitude']) float(stationdict['rr_station_elevation']) except: try: #check for shortened form stationdict['rr_station_latitude'] = float( stationdict['rr_station_lat']) stationdict['rr_station_longitude'] = float( stationdict['rr_station_lon']) stationdict['rr_station_elevation'] = float( stationdict['rr_station_ele']) except: try: #read from other config dict entry stationdict['rr_station_latitude'] = \ config_dict[rem_station]['latitude'] stationdict['rr_station_longitude'] = \ config_dict[rem_station]['longitude'] stationdict['rr_station_elevation'] = \ config_dict[rem_station]['elevation'] except: #if finally failed to read rr_station info,\ #set rr_station back to None stationdict['rr_station'] = None stationdict['rr_station_latitude'] = None stationdict['rr_station_longitude'] = None stationdict['rr_station_elevation'] = None #check consistency of coordinates, if rr_station is present if stationdict['rr_station'] != None: try: stationdict['rr_station_latitude'] = \ MTft._assert_position_format( 'latitude',stationdict['rr_station_latitude']) stationdict['rr_station_longitude'] = \ MTft._assert_position_format( 'longitude',stationdict['rr_station_longitude']) stationdict['rr_station_elevation'] = \ MTft._assert_position_format( 'elevation',stationdict['rr_station_elevation']) except: print 'Problem with remote reference station ({0}) -' ' remote reference ({1}) coordinates invalid -' ' remote reference set to None'.format( station, stationdict['rr_station']) stationdict['rr_station'] = None stationdict['rr_station_latitude'] = None stationdict['rr_station_longitude'] = None stationdict['rr_station_elevation'] = None if error_counter != 0: print 'Could not read all mandatory sections and options'\ ' in config file - found {0} errors - check configuration'\ ' file before continue!'.format(error_counter) answer = 5 while not answer in ['y', 'n']: answer = raw_input('\n\tDo you want to continue anyway? (y/n)') try: answer = answer.strip().lower()[0] except: continue if answer == 'n': print sys.exit() print return config_dict
def read_survey_configfile(filename): """ Read in a survey configuration file and return a dictionary. Input config file must contain station names as section headers! The output dictionary keys are station names (capitalised), the values are (sub-)dictionaries. The configuration file must contain sections for all stations, each containing all mandatory keywords: - latitude (deg) - longitude (deg) - elevation (in meters) - sampling_interval (in seconds) - station_type (MT, (Q)E, (Q)B) Not mandatory, but recommended - declination (in degrees, positive to East) - this is set to '0.0', if omitted Depending on the type of station the following entries are required. E-field recorded: - E_logger_type ('edl'/'elogger'/'qel') - E_logger_gain (factor/gain-level) - E_instrument_type ('electrodes'/'dipole') - E_instrument_amplification (applied amplification factor) - E_Xaxis_azimuth (degrees) - E_Xaxis_length (in meters) - E_Yaxis_azimuth (degrees) - E_Yaxis_length (in meters) B-field recorded: - B_logger_type ('edl'/'qel_blogger') - B_logger_gain (factor/gain level) - B_instrument_type ('coil(s)', 'fluxgate') - B_instrument_amplification (applied amplification factor) - B_Xaxis_azimuth (degrees) - B_Yaxis_azimuth (degrees) A global section can be used to include parameters for all stations. The name of the section must be one of: global/main/default/general """ error_counter = 0 #generate config parser instance configobject = configparser.ConfigParser() #check, if file is present if not op.isfile(filename): raise MTex.MTpyError_inputarguments( 'File does not' ' exist: {0}'.format(filename) ) # try to parse file - exit, if not a config file try: configobject.read(filename) except: raise MTex.MTpyError_inputarguments( 'File is not a ' 'proper configuration file: {0}'.format(filename) ) #obtain dict of dicts containing the input file's sections (station names) #excludes DEFAULT section and key-value pairs without section header configobject_dict = configobject._sections #initialise the output dictionary config_dict = {} #loop over the sections (stations) of the config file for station in configobject_dict: #read in the sub-dictionary for the current station - bringing all keys #to lowercase! temp_dict_in = dict((k.lower(), v) for k, v in list(configobject_dict[station].items())) #initialise output sub-directory for current station stationdict = temp_dict_in #stationnames are uppercase in MTpy stationname = station if stationname in ['GLOBAL','MAIN','DEFAULT','GENERAL']: stationname = 'GLOBAL' stationdict['station'] = stationname #add the station's sub-dictionary to the config dictionary config_dict[stationname] = stationdict # Check if a global section is present if 'GLOBAL' in config_dict: globaldict = config_dict['GLOBAL'] else: #set defaults for location globaldict={} # for i in ['latitude', 'longitude', 'elevation']: # #skip if values are present # if i in globaldict.keys() or i[:3] in globaldict.keys(): # continue # #otherwise set defaults # globaldict[i] = 0 #remove other general sections to avoid redundancy for i in ['MAIN','DEFAULT','GENERAL']: if i in config_dict: dummy = config_dict.pop(i) # RE-loop to check for each station if required keywords are present, # if not if they can be pulled from the global section #============================================================ # local function definition def fromglobals(key,stationdict,globaldict): """ Check if stationdict contains key. If not search for key in global dict and add it to station dict. Return if global dict is not defined. Return True if key was present in either dictionary, False if not. """ if key in list(stationdict.keys()): return True, stationdict.get(key) if globaldict is None or len(globaldict) == 0: return False, None if key in globaldict: stationdict[key] = globaldict[key] return True,globaldict.get(key) return False, None #============================================================ for station in sorted(config_dict): #do not alter the global section if station == 'GLOBAL': continue stationdict = config_dict[station] #check for presence of all mandatory keywords for the current station #case insensitive - allow for short forms 'sampling', 'lat', 'lon', and 'elev' for idx,req_keyword in enumerate(list_of_required_keywords): shortform = list_of_required_keywords_short[idx] try: found = False #import ipdb #ipdb.set_trace() if fromglobals(req_keyword,stationdict,globaldict)[0] is False: #try short form instead found,value = fromglobals(shortform,stationdict,globaldict) #print shortform,value if found is True: stationdict[req_keyword] = value else: found = True if found is False: print('Station {0} - keyword {1} missing'.format(stationname, req_keyword)) error_counter += 1 raise Exception if req_keyword in ['elevation','latitude', 'longitude']: #check format of lat/lon - convert to degrees, if given in #(deg,min,sec)-triple#assert correct format value = stationdict[req_keyword] try: if req_keyword in 'latitude': new_value = gis_tools.assert_lat_value(value) elif req_keyword in 'longitude': new_value = gis_tools.assert_lon_value(value) elif req_keyword in 'elevation': new_value = gis_tools.assert_elevation_value(value) except: raise MTex.MTpyError_config_file('Error - wrong ' 'coordinate format for station {0}'.format(stationname)) stationdict[req_keyword] = new_value except: raise print('Missing information on station {0} in config file'\ ' - setting default (dummy) value'.format(station)) stationdict[req_keyword] = list_of_keyword_defaults_general[idx] #to avoid duplicates remove the now obsolete short form from #the station dictionary dummy = stationdict.pop(shortform,None) if not stationdict['station_type'] in list_of_station_types: raise MTex.MTpyError_config_file( 'Station type not valid' ) if stationdict['station_type'] in ['mt','e']: #check for required electric field parameters - not done for QEL loggers yet for req_keyword in list_of_efield_keywords: if req_keyword.lower() in list(temp_dict_in.keys()): stationdict[req_keyword.lower()] = \ temp_dict_in[req_keyword.lower()].lower() else: print('Station {0} - keyword {1} missing'.format(stationname, req_keyword)) error_counter += 1 continue _validate_dictionary(stationdict,dict_of_allowed_values_efield) if stationdict['station_type'] in ['mt','b']: #check for required magnetic field parameters for req_keyword in list_of_bfield_keywords: if req_keyword.lower() in list(temp_dict_in.keys()): stationdict[req_keyword.lower()] = \ temp_dict_in[req_keyword.lower()].lower() else: print('Station {0} - keyword {1} missing'.format(stationname, req_keyword)) error_counter += 1 continue _validate_dictionary(stationdict,dict_of_allowed_values_bfield) #re-loop for setting up correct remote reference station information : #if rem.ref. station key is present, its information must be contained #in the same config file! for station in config_dict.keys(): stationdict = config_dict[station] if 'rr_station' not in stationdict: continue #stationdict['rr_station'] = None stationdict['rr_latitude'] = None stationdict['rr_longitude'] = None stationdict['rr_elevation'] = None rem_station = stationdict['rr_station'] try: #check, if values are contained in dict float(stationdict['rr_latitude'] ) float(stationdict['rr_longitude']) float(stationdict['rr_elevation']) except: try: #check for shortened form stationdict['rr_latitude'] = float(stationdict['rr_lat'] ) stationdict['rr_longitude'] = float(stationdict['rr_lon'] ) stationdict['rr_elevation'] = float(stationdict['rr_elev'] ) except: try: #read from other config dict entry stationdict['rr_latitude'] = \ config_dict[rem_station]['latitude'] stationdict['rr_longitude'] = \ config_dict[rem_station]['longitude'] stationdict['rr_elevation'] = \ config_dict[rem_station]['elevation'] except: #if finally failed to read rr_station info,\ #set rr_station back to None stationdict['rr_station'] = None stationdict['rr_latitude'] = None stationdict['rr_longitude'] = None stationdict['rr_elevation'] = None #check consistency of coordinates, if rr_station is present if stationdict['rr_station'] != None: try: stationdict['rr_latitude'] = \ gis_tools.assert_lat_value(stationdict['rr_latitude']) stationdict['rr_longitude'] = \ gis_tools.assert_lon_value(stationdict['rr_longitude']) stationdict['rr_elevation'] = \ gis_tools.assert_elevation_value(stationdict['rr_elevation']) except: print('Problem with remote reference station ({0}) -') ' remote reference ({1}) coordinates invalid -' ' remote reference set to None'.format(station, stationdict['rr_station']) stationdict['rr_station'] = None stationdict['rr_latitude'] = None stationdict['rr_longitude'] = None stationdict['rr_elevation'] = None if error_counter != 0: print('Could not read all mandatory sections and options'\ ' in config file - found {0} errors - check configuration'\ ' file before continue!'.format(error_counter)) answer = 5 while not answer in ['y','n']: answer = input('\n\tDo you want to continue anyway? (y/n)') try: answer = answer.strip().lower()[0] except: continue if answer == 'n': sys.exit() return config_dict