def convert_dms_tuple2degrees(latlon_list):
"""
Convert a triple (list, tuple, array) of degrees, minuts, seconds into degrees.
Validity of the triple is assumed and has to be asserted in advance.
"""
sign = 1.
try:
latlon_list = [float(i) for i in latlon_list]
if str(latlon_list[0]).startswith('-'):
sign = -1.
except:
# if triple is given as string:
latlon_raw = latlon_list
# allow separation by :,. or space for (deg min sec) format
try:
latlon_list = re.split('[ :,]', latlon_raw)
except:
raise MTex.MTpyError_config_file(
'Config file error: lat/lon is in invalid format')
try:
latlon_list = [float(i) for i in latlon_list]
except:
raise MTex.MTpyError_config_file(
'Config file error: lat/lon is in invalid format')
if str(latlon_list[0])[0] == '-':
sign = -1.
deg = latlon_list[0]
minutes = latlon_list[1]
seconds = latlon_list[2]
if not (-180 <= deg <= 360 and 0 <= minutes < 60 and 0 <= seconds < 60):
print 'Value for degrees, minutes, or seconds invalid'
raise MTex.MTpyError_inputarguments()
# take out sign for easier conversion into degrees
if deg < 0:
deg *= -1
degrees = deg + 1 / 60. * minutes + 1 / 3600. * seconds
return degrees * sign
def _validate_dictionary(dict2validate, referencedict):
"""Check, if there are lists of allowed entries for all
keys of the current dictionary. If yes, test, if the current
value is among the allowed ones.
"""
for key, value in dict2validate.items():
#make everything to strings - easier to compare
#in case of numbers, make to float first
try:
allowed_vals = referencedict[key]
except:
try:
key = key.lower()
allowed_vals = referencedict[key]
except:
#no reference entry found - skip key
continue
tmp = []
#allowed values must be given as a list (iterable)!!
for i in allowed_vals:
try:
tmp.append(str(float(i)))
except:
tmp.append(str(i))
tmp = [i.lower() for i in tmp]
allowed_vals = tmp
#compare case-insensitive
value = value.lower()
if not value in allowed_vals:
raise MTex.MTpyError_config_file(
'Config file error --'
' key {0}, value {1} not valid'.format(key, value))
def reorient_files(lo_files, configfile, lo_stations=None, outdir=None):
#read config file
try:
config_dict = MTcf.read_survey_configfile(configfile)
except:
raise MTex.MTpyError_config_file('Config file cannot be read:'
' {0}'.format(configfile))
if lo_stations is not None:
try:
if type(lo_stations) == str:
raise
#check, if it's iterable:
dummy = [i for i in lo_stations]
except:
raise MTex.MTpyError_inputarguments('ERROR - "lo_stations"'
' argument must be iterable!')
print '\t re-orienting data for collection of stations:\n{0}'.format(
lo_stations)
#Do not require list of headers as input, as this function can be called directly rather than from a 'calibratefiles.py'-like script - so the list not necessarily exists in beforehand -
#collect header lines of files in list
lo_headers = []
lo_stationnames = []
for file_idx, filename in enumerate(lo_files):
header = read_ts_header(filename)
station = header['station']
if station.upper() not in [i.upper() for i in lo_stations]:
#TODO: check, if this causes problems with the indices for the current loop:
lo_files.remove(filename)
continue
lo_headers.append(header)
lo_stationnames.append(station.upper())
if len(lo_headers) == 0:
if lo_stations is not None:
print 'ERROR - No files with header lines found for station(s)'\
' {0}'.format(lo_stations)
else:
print 'ERROR - No files with header lines found'
return 1
lo_stationnames = list(set(lo_stationnames))
# set up output directory
ori_outdir = op.abspath(op.join(os.curdir, 'reoriented'))
if outdir is not None:
try:
ori_outdir = op.abspath(op.join(os.curdir, outdir))
if not op.isdir(ori_outdir):
os.makedirs(ori_outdir)
except:
print 'Output directory cannot be generated: {0} - using generic'\
' location'.format(ori_outdir)
ori_outdir = op.abspath(op.join(os.curdir, 'reoriented'))
try:
if not op.isdir(ori_outdir):
os.makedirs(ori_outdir)
except:
#this only comes up, if the generic location cannot be generated
raise MTex.MTpyError_inputarguments(
'Generic directory cannot be'
' generated: {0}'.format(ori_outdir))
#----------------------
#start re-orientation
#present: list of all files, list of all headers, list of all stations
for sta_idx, sta in enumerate(lo_stationnames):
#print sta
try:
stationconfig = config_dict[sta]
except:
print 'Warning - No config file entry for station {0} -'\
' no processing possible'.format(sta)
continue
declination = float(stationconfig.get('declination', 0.))
for sensor in ['e', 'b']:
#TODO:
# reduce this function to the re-orientation of files that have the same length for X and Y.
#Do the puzzlling for varying lengths later!!
for idx_h_x, header_x in enumerate(lo_headers):
#looking for one specific station
if not header_x['station'].upper() == sta.upper():
continue
#looking for the specific sensor type
if not header_x['channel'].lower()[0] == sensor:
continue
#looking for the X channel (the to-be-North)
if not header_x['channel'].lower()[1] == 'x':
continue
x_file = lo_files[idx_h_x]
x_header_string = get_ts_header_string(header_x)
t0 = float(header_x['t_min'])
#print t0
#now look for the respective y-file and possible z-file - unfortunately by another loop over all headers:
y_file = None
z_file = None
for idx_h_y, header_y in enumerate(lo_headers):
if (header_y['station'].upper() == sta.upper()) and \
(header_y['channel'].lower()[0] == sensor) and \
(float(header_y['t_min']) == float(header_x['t_min'] ) ):
if (header_y['channel'].lower()[1] == 'y'):
y_file = lo_files[idx_h_y]
y_header_string = get_ts_header_string(header_y)
elif (header_y['channel'].lower()[1] == 'z'):
z_file = lo_files[idx_h_y]
else:
continue
if y_file == None:
continue
x_outfn = op.abspath(op.join(ori_outdir, op.basename(x_file)))
y_outfn = op.abspath(op.join(ori_outdir, op.basename(y_file)))
if z_file is not None:
z_outfn = op.abspath(
op.join(ori_outdir, op.basename(z_file)))
xdata = np.loadtxt(x_file)
ydata = np.loadtxt(y_file)
#declination is positive, if magnetic North is east of true North.
# the measured angles are w.r.t. magnetic North, so the given
# azimuths do not include the declination
#-> thus the declination value is added to azimuths
if sensor == 'e':
xangle = float(stationconfig.get('e_xaxis_azimuth',
0.)) + declination
yangle = float(stationconfig.get('e_yaxis_azimuth',
90.)) + declination
else:
xangle = float(stationconfig.get('b_xaxis_azimuth',
0.)) + declination
yangle = float(stationconfig.get('b_yaxis_azimuth',
90.)) + declination
newx, newy = MTcc.reorient_data2D(xdata,
ydata,
x_sensor_angle=xangle,
y_sensor_angle=yangle)
#print xdata.shape, ydata.shape, newx.shape, newy.shape
#continue
outFx = open(x_outfn, 'w')
outFx.write(x_header_string)
np.savetxt(outFx, newx)
outFx.close()
outFy = open(y_outfn, 'w')
outFy.write(y_header_string)
np.savetxt(outFy, newy)
outFy.close()
written_files = [x_outfn, y_outfn]
if z_file is not None:
shutil.copyfile(z_file, z_outfn)
written_files.append(z_outfn)
print '\tSuccessfullly written files {0}'.format(written_files)
return 0
def read_survey_configfile(filename):
"""
Read in a survey configuration file and return a dictionary.
Input config file must contain station names as section headers!
The output dictionary keys are station names (capitalised),
the values are (sub-)dictionaries. The configuration file must contain
sections for all stations, each containing all mandatory keywords:
- latitude (deg)
- longitude (deg)
- elevation (in meters)
- sampling_interval (in seconds)
- station_type (MT, E, B)
Not mandatory, but recommended
- declination (in degrees, positive to East) - this is set to '0.0', if omitted
Depending on the type of station the following entries are required.
E-field recorded:
- E_logger_type ('edl'/'elogger'/'qel')
- E_logger_gain (factor/gain-level)
- E_instrument_type ('electrodes'/'dipole')
- E_instrument_amplification (applied amplification factor)
- E_Xaxis_azimuth (degrees)
- E_Xaxis_length (in meters)
- E_Yaxis_azimuth (degrees)
- E_Yaxis_length (in meters)
B-field recorded:
- B_logger_type ('edl'/'qel_blogger')
- B_logger_gain (factor/gain level)
- B_instrument_type ('coil(s)', 'fluxgate')
- B_instrument_amplification (applied amplification factor)
- B_Xaxis_azimuth (degrees)
- B_Yaxis_azimuth (degrees)
"""
error_counter = 0
#generate config parser instance
configobject = ConfigParser.ConfigParser()
#check, if file is present
if not op.isfile(filename):
raise MTex.MTpyError_inputarguments('File does not'
' exist: {0}'.format(filename))
# try to parse file - exit, if not a config file
try:
configobject.read(filename)
except:
raise MTex.MTpyError_inputarguments(
'File is not a '
'proper configuration file: {0}'.format(filename))
#obtain dict of dicts containing the input file's sections (station names)
#excludes DEFAULT section and key-value pairs without section header
configobject_dict = configobject._sections
#initialise the output dictionary
config_dict = {}
#loop over the sections (stations) of the config file
for station in configobject_dict:
#read in the sub-dictionary for the current station - bringing all keys
#to lowercase!
temp_dict_in = dict(
(k.lower(), v) for k, v in configobject_dict[station].items())
#initialise output sub-directory for current station
stationdict = temp_dict_in
#stationnames are uppercase in MTpy
stationname = station.upper()
stationdict['station'] = stationname
#check for presence of all mandatory keywords for the current station
#case insensitive - allow for short forms 'lat', 'lon', and 'ele'
try:
for req_keyword in list_of_required_keywords:
if req_keyword.lower() in temp_dict_in.keys():
stationdict[req_keyword.lower()] = \
temp_dict_in[req_keyword.lower()].lower()
elif req_keyword in ['latitude', 'longitude', 'elevation']:
if req_keyword[:3] in temp_dict_in.keys():
stationdict[req_keyword] = temp_dict_in[
req_keyword[:3]]
else:
print 'Station {0} - keyword {1} missing'.format(
stationname, req_keyword)
error_counter += 1
continue
#check format of lat/lon - convert to degrees, if given in
#(deg,min,sec)-triple
for coordinate in ['latitude', 'longitude', 'elevation']:
value = stationdict[coordinate]
try:
new_value = MTft._assert_position_format(coordinate, value)
except:
raise MTex.MTpyError_config_file(
'Error - wrong '
'coordinate format for station {0}'.format(
stationname))
stationdict[coordinate] = new_value
if not stationdict['station_type'] in list_of_station_types:
raise MTex.MTpyError_config_file('Station type not valid')
except:
print 'Missing information on station {0} in config file - skipping'.format(
station)
continue
if stationdict['station_type'] in ['mt', 'e']:
#check for required electric field parameters
for req_keyword in list_of_efield_keywords:
if req_keyword.lower() in temp_dict_in.keys():
stationdict[req_keyword.lower()] = \
temp_dict_in[req_keyword.lower()].lower()
else:
print 'Station {0} - keyword {1} missing'.format(
stationname, req_keyword)
error_counter += 1
continue
_validate_dictionary(stationdict, dict_of_allowed_values_efield)
if stationdict['station_type'] in ['mt', 'b']:
#check for required magnetic field parameters
for req_keyword in list_of_bfield_keywords:
if req_keyword.lower() in temp_dict_in.keys():
stationdict[req_keyword.lower()] = \
temp_dict_in[req_keyword.lower()].lower()
else:
print 'Station {0} - keyword {1} missing'.format(
stationname, req_keyword)
error_counter += 1
continue
_validate_dictionary(stationdict, dict_of_allowed_values_bfield)
#add the station's sub-dictionary to the config dictionary
config_dict[stationname] = stationdict
#re-loop for setting up correct remote reference station information :
#if rem.ref. station key is present, its information must be contained
#in the same config file!
for station in config_dict.iterkeys():
stationdict = config_dict[station]
if not stationdict.has_key('rr_station'):
continue
#stationdict['rr_station'] = None
stationdict['rr_station_latitude'] = None
stationdict['rr_station_longitude'] = None
stationdict['rr_station_elevation'] = None
rem_station = stationdict['rr_station']
try:
#check, if values are contained in dict
float(stationdict['rr_station_latitude'])
float(stationdict['rr_station_longitude'])
float(stationdict['rr_station_elevation'])
except:
try:
#check for shortened form
stationdict['rr_station_latitude'] = float(
stationdict['rr_station_lat'])
stationdict['rr_station_longitude'] = float(
stationdict['rr_station_lon'])
stationdict['rr_station_elevation'] = float(
stationdict['rr_station_ele'])
except:
try:
#read from other config dict entry
stationdict['rr_station_latitude'] = \
config_dict[rem_station]['latitude']
stationdict['rr_station_longitude'] = \
config_dict[rem_station]['longitude']
stationdict['rr_station_elevation'] = \
config_dict[rem_station]['elevation']
except:
#if finally failed to read rr_station info,\
#set rr_station back to None
stationdict['rr_station'] = None
stationdict['rr_station_latitude'] = None
stationdict['rr_station_longitude'] = None
stationdict['rr_station_elevation'] = None
#check consistency of coordinates, if rr_station is present
if stationdict['rr_station'] != None:
try:
stationdict['rr_station_latitude'] = \
MTft._assert_position_format(
'latitude',stationdict['rr_station_latitude'])
stationdict['rr_station_longitude'] = \
MTft._assert_position_format(
'longitude',stationdict['rr_station_longitude'])
stationdict['rr_station_elevation'] = \
MTft._assert_position_format(
'elevation',stationdict['rr_station_elevation'])
except:
print 'Problem with remote reference station ({0}) -'
' remote reference ({1}) coordinates invalid -'
' remote reference set to None'.format(
station, stationdict['rr_station'])
stationdict['rr_station'] = None
stationdict['rr_station_latitude'] = None
stationdict['rr_station_longitude'] = None
stationdict['rr_station_elevation'] = None
if error_counter != 0:
print 'Could not read all mandatory sections and options'\
' in config file - found {0} errors - check configuration'\
' file before continue!'.format(error_counter)
answer = 5
while not answer in ['y', 'n']:
answer = raw_input('\n\tDo you want to continue anyway? (y/n)')
try:
answer = answer.strip().lower()[0]
except:
continue
if answer == 'n':
print
sys.exit()
print
return config_dict
def read_survey_configfile(filename):
"""
Read in a survey configuration file and return a dictionary.
Input config file must contain station names as section headers!
The output dictionary keys are station names (capitalised),
the values are (sub-)dictionaries. The configuration file must contain
sections for all stations, each containing all mandatory keywords:
- latitude (deg)
- longitude (deg)
- elevation (in meters)
- sampling_interval (in seconds)
- station_type (MT, (Q)E, (Q)B)
Not mandatory, but recommended
- declination (in degrees, positive to East) - this is set to '0.0', if omitted
Depending on the type of station the following entries are required.
E-field recorded:
- E_logger_type ('edl'/'elogger'/'qel')
- E_logger_gain (factor/gain-level)
- E_instrument_type ('electrodes'/'dipole')
- E_instrument_amplification (applied amplification factor)
- E_Xaxis_azimuth (degrees)
- E_Xaxis_length (in meters)
- E_Yaxis_azimuth (degrees)
- E_Yaxis_length (in meters)
B-field recorded:
- B_logger_type ('edl'/'qel_blogger')
- B_logger_gain (factor/gain level)
- B_instrument_type ('coil(s)', 'fluxgate')
- B_instrument_amplification (applied amplification factor)
- B_Xaxis_azimuth (degrees)
- B_Yaxis_azimuth (degrees)
A global section can be used to include parameters for all stations.
The name of the section must be one of:
global/main/default/general
"""
error_counter = 0
#generate config parser instance
configobject = configparser.ConfigParser()
#check, if file is present
if not op.isfile(filename):
raise MTex.MTpyError_inputarguments( 'File does not'
' exist: {0}'.format(filename) )
# try to parse file - exit, if not a config file
try:
configobject.read(filename)
except:
raise MTex.MTpyError_inputarguments( 'File is not a '
'proper configuration file: {0}'.format(filename) )
#obtain dict of dicts containing the input file's sections (station names)
#excludes DEFAULT section and key-value pairs without section header
configobject_dict = configobject._sections
#initialise the output dictionary
config_dict = {}
#loop over the sections (stations) of the config file
for station in configobject_dict:
#read in the sub-dictionary for the current station - bringing all keys
#to lowercase!
temp_dict_in = dict((k.lower(), v)
for k, v in list(configobject_dict[station].items()))
#initialise output sub-directory for current station
stationdict = temp_dict_in
#stationnames are uppercase in MTpy
stationname = station
if stationname in ['GLOBAL','MAIN','DEFAULT','GENERAL']:
stationname = 'GLOBAL'
stationdict['station'] = stationname
#add the station's sub-dictionary to the config dictionary
config_dict[stationname] = stationdict
# Check if a global section is present
if 'GLOBAL' in config_dict:
globaldict = config_dict['GLOBAL']
else:
#set defaults for location
globaldict={}
# for i in ['latitude', 'longitude', 'elevation']:
# #skip if values are present
# if i in globaldict.keys() or i[:3] in globaldict.keys():
# continue
# #otherwise set defaults
# globaldict[i] = 0
#remove other general sections to avoid redundancy
for i in ['MAIN','DEFAULT','GENERAL']:
if i in config_dict:
dummy = config_dict.pop(i)
# RE-loop to check for each station if required keywords are present,
# if not if they can be pulled from the global section
#============================================================
# local function definition
def fromglobals(key,stationdict,globaldict):
"""
Check if stationdict contains key.
If not search for key in global dict and add it to station dict.
Return if global dict is not defined.
Return True if key was present in either dictionary, False if not.
"""
if key in list(stationdict.keys()):
return True, stationdict.get(key)
if globaldict is None or len(globaldict) == 0:
return False, None
if key in globaldict:
stationdict[key] = globaldict[key]
return True,globaldict.get(key)
return False, None
#============================================================
for station in sorted(config_dict):
#do not alter the global section
if station == 'GLOBAL':
continue
stationdict = config_dict[station]
#check for presence of all mandatory keywords for the current station
#case insensitive - allow for short forms 'sampling', 'lat', 'lon', and 'elev'
for idx,req_keyword in enumerate(list_of_required_keywords):
shortform = list_of_required_keywords_short[idx]
try:
found = False
#import ipdb
#ipdb.set_trace()
if fromglobals(req_keyword,stationdict,globaldict)[0] is False:
#try short form instead
found,value = fromglobals(shortform,stationdict,globaldict)
#print shortform,value
if found is True:
stationdict[req_keyword] = value
else:
found = True
if found is False:
print('Station {0} - keyword {1} missing'.format(stationname,
req_keyword))
error_counter += 1
raise Exception
if req_keyword in ['elevation','latitude', 'longitude']:
#check format of lat/lon - convert to degrees, if given in
#(deg,min,sec)-triple#assert correct format
value = stationdict[req_keyword]
try:
if req_keyword in 'latitude':
new_value = gis_tools.assert_lat_value(value)
elif req_keyword in 'longitude':
new_value = gis_tools.assert_lon_value(value)
elif req_keyword in 'elevation':
new_value = gis_tools.assert_elevation_value(value)
except:
raise MTex.MTpyError_config_file('Error - wrong '
'coordinate format for station {0}'.format(stationname))
stationdict[req_keyword] = new_value
except:
raise
print('Missing information on station {0} in config file'\
' - setting default (dummy) value'.format(station))
stationdict[req_keyword] = list_of_keyword_defaults_general[idx]
#to avoid duplicates remove the now obsolete short form from
#the station dictionary
dummy = stationdict.pop(shortform,None)
if not stationdict['station_type'] in list_of_station_types:
raise MTex.MTpyError_config_file( 'Station type not valid' )
if stationdict['station_type'] in ['mt','e']:
#check for required electric field parameters - not done for QEL loggers yet
for req_keyword in list_of_efield_keywords:
if req_keyword.lower() in list(temp_dict_in.keys()):
stationdict[req_keyword.lower()] = \
temp_dict_in[req_keyword.lower()].lower()
else:
print('Station {0} - keyword {1} missing'.format(stationname,
req_keyword))
error_counter += 1
continue
_validate_dictionary(stationdict,dict_of_allowed_values_efield)
if stationdict['station_type'] in ['mt','b']:
#check for required magnetic field parameters
for req_keyword in list_of_bfield_keywords:
if req_keyword.lower() in list(temp_dict_in.keys()):
stationdict[req_keyword.lower()] = \
temp_dict_in[req_keyword.lower()].lower()
else:
print('Station {0} - keyword {1} missing'.format(stationname,
req_keyword))
error_counter += 1
continue
_validate_dictionary(stationdict,dict_of_allowed_values_bfield)
#re-loop for setting up correct remote reference station information :
#if rem.ref. station key is present, its information must be contained
#in the same config file!
for station in config_dict.keys():
stationdict = config_dict[station]
if 'rr_station' not in stationdict:
continue
#stationdict['rr_station'] = None
stationdict['rr_latitude'] = None
stationdict['rr_longitude'] = None
stationdict['rr_elevation'] = None
rem_station = stationdict['rr_station']
try:
#check, if values are contained in dict
float(stationdict['rr_latitude'] )
float(stationdict['rr_longitude'])
float(stationdict['rr_elevation'])
except:
try:
#check for shortened form
stationdict['rr_latitude'] = float(stationdict['rr_lat'] )
stationdict['rr_longitude'] = float(stationdict['rr_lon'] )
stationdict['rr_elevation'] = float(stationdict['rr_elev'] )
except:
try:
#read from other config dict entry
stationdict['rr_latitude'] = \
config_dict[rem_station]['latitude']
stationdict['rr_longitude'] = \
config_dict[rem_station]['longitude']
stationdict['rr_elevation'] = \
config_dict[rem_station]['elevation']
except:
#if finally failed to read rr_station info,\
#set rr_station back to None
stationdict['rr_station'] = None
stationdict['rr_latitude'] = None
stationdict['rr_longitude'] = None
stationdict['rr_elevation'] = None
#check consistency of coordinates, if rr_station is present
if stationdict['rr_station'] != None:
try:
stationdict['rr_latitude'] = \
gis_tools.assert_lat_value(stationdict['rr_latitude'])
stationdict['rr_longitude'] = \
gis_tools.assert_lon_value(stationdict['rr_longitude'])
stationdict['rr_elevation'] = \
gis_tools.assert_elevation_value(stationdict['rr_elevation'])
except:
print('Problem with remote reference station ({0}) -')
' remote reference ({1}) coordinates invalid -'
' remote reference set to None'.format(station,
stationdict['rr_station'])
stationdict['rr_station'] = None
stationdict['rr_latitude'] = None
stationdict['rr_longitude'] = None
stationdict['rr_elevation'] = None
if error_counter != 0:
print('Could not read all mandatory sections and options'\
' in config file - found {0} errors - check configuration'\
' file before continue!'.format(error_counter))
answer = 5
while not answer in ['y','n']:
answer = input('\n\tDo you want to continue anyway? (y/n)')
try:
answer = answer.strip().lower()[0]
except:
continue
if answer == 'n':
sys.exit()
return config_dict