def test_output():
tmpdir = tempfile.mkdtemp()
# Full path to save Occam2D data file
o2d_path = os.path.join(tmpdir, 'o2d_data.dat')
rot_o2d_path = os.path.join(tmpdir, 'rot_o2d_data.dat')
# Full path to save Mare2D data file
m2d_path = os.path.join(tmpdir, 'mare2dem_test.txt')
# Generate an Occam2D data object from EDI data
o2d_data = o2d.Data(edi_path=EDI_DATA_DIR2,
model_mode='1',
optimize_line=True,
interpolate_freq=False,
res_te_err=20.,
phase_te_err=10.,
res_tm_err=10.,
phase_tm_err=5.)
# Save the data file
# We need a data file with the non-rotated profile and the rotated
# profile. This is because the elevation will be interpolated over
# the non-projected profile and stations.
rot_o2d_data = deepcopy(
o2d_data) # Make a copy because 'write_data_file' will populate data
o2d_data._rotate_to_strike = False
o2d_data.save_path = o2d_path
rot_o2d_data.save_path = rot_o2d_path
o2d_data.write_data_file(data_fn=o2d_path)
rot_o2d_data.write_data_file(data_fn=rot_o2d_path)
gstrike = o2d_data.geoelectric_strike
# Convert the Occam2D profile to Mare2D
mare_origin, utm_zone, site_locations, site_elevations, site_names, m2d_profile, profile_elevation = \
m2d.occam2d_to_mare2dem(o2d_data, rot_o2d_data, AUS_TOPO_FILE, elevation_sample_n=300)
m2d.write_mare2dem_data(o2d_path,
site_locations,
site_elevations,
site_names,
mare_origin,
utm_zone,
gstrike,
solve_statics=False,
savepath=m2d_path)
yield m2d_path
if os.path.exists(tmpdir):
shutil.rmtree(tmpdir)
def build_mesh(self):
"""
create a mesh using occam2d
"""
# create an occam2d setup object
so = o2d.Setup(wd=self.working_directory,
edi_directory=self.edi_directory,
edifiles=self.edifiles,
configfile=self.occam_configfile,
strike=self.parameters_data['strike'],
**self.parameters_model)
so.read_edifiles(edi_dir=self.edi_directory)
# create an occam2d data object
so.Data = o2d.Data(edilist=self.edifiles,
wd=so.wd, **so.parameters_data)
# set up meshlocations
so.setup_mesh_and_model()
self.stationlocations = np.array(so.Data.stationlocations) / 1000.
# set occam mesh attributes to pek2d object
for attribute in ['meshlocations_x', 'meshlocations_z',
'meshblockwidths_x', 'meshblockthicknesses_z',
'profile_easts', 'profile_norths', 'Data']:
if 'mesh' in attribute:
attvalue = np.array(getattr(so, attribute)) / 1000.
else:
attvalue = getattr(so, attribute)
setattr(self, attribute, attvalue)
for attribute in ['firstlayer_thickness', 'model_depth',
'no_sideblockelements', 'no_bottomlayerelements',
'no_layers', 'max_blockwidth']:
self.parameters_model[attribute] = so.parameters_inmodel[attribute]
self.meshlocations_z = np.array([0.] + list(self.meshlocations_z))
# get block centres
self.blockcentres_x = [np.mean(self.meshlocations_x[i:i + 2]) for i in
range(len(self.meshlocations_x) - 1)]
self.blockcentres_z = [np.mean(self.meshlocations_z[k:k + 2]) for k in
range(len(self.meshlocations_z) - 1)]
def build_inputfiles(self):
"""
Set up collection of required input files files for OCCAM.
- data
- startup
- model
- mesh
"""
returnvalue = 0
#1. Build OCCAM data file, or use existing one (if field is checked in GUI)
D = self.parameters
#print D['check_usedatafile']
#print D['olddatafile']
olddatafile = D['olddatafile']
if olddatafile is not None:
datafile = op.abspath(op.join(D['wd'], olddatafile))
messagetext = ''
returnvalue = 0
try:
data_object = MTo2.Data()
data_object.readfile(datafile)
D['strike'] = data_object.strike
D['azimuth'] = data_object.azimuth
D['stationlocations'] = data_object.stationlocations
messagetext += "<P><FONT COLOR='#000000'>Working directory: "\
"{0} </FONT></P> \n".format(data_object.wd)
messagetext += "<P><b><FONT COLOR='#008080'>Read old data file:</FONT></b></P><br>{0}".format(
datafile)
except:
messagetext += "<P><b><FONT COLOR='#800000'>Error: Cannot read old data file: {0} </FONT></b></P> ".format(
datafile)
returnvalue = 1
QtGui.QMessageBox.about(self, "Data file generation", messagetext)
if returnvalue == 1:
return
else:
outfilename = D['datafile']
edidirectory = D['edi_dir']
def make_stationlist(listfilename):
"""
Read in stations from file.
"""
FH = file(listfilename, 'r')
raw_string = FH.read()
FH.close()
raw_list1 = raw_string.strip().split()
raw_list2 = []
for i in raw_list1:
if len(i.split(',')) == 1:
raw_list2.append(i)
else:
for j in i.split(','):
raw_list2.append(j)
return raw_list2
#define internal station list
stationlist = None
if D['use_stationfile']:
stationlist = make_stationlist(D['stationlistfile'])
D['stationlist'] = stationlist
#make data file -------------------------------------------
returnvalue = 0
messagetext = ''
try:
setup_object = MTo2.Setup(**D)
except:
messagetext += "<P><b><FONT COLOR='#800000'>Error: Could not "\
"generate setup object - check input parameters! </FONT></b></P> \n"
QtGui.QMessageBox.about(self, "Input files generation",
messagetext)
return 1
try:
edi_dir = D['edi_dir']
setup_object.read_edifiles(edi_dir)
datafile = D['datafile']
setup_object.datafile = datafile
try:
setup_object.write_datafile()
except:
raise
datafilename = setup_object.datafile
self.parameters[
'stationlocations'] = setup_object.Data.stationlocations
messagetext += "<P><FONT COLOR='#000000'>Working directory: "\
"{0} </FONT></P> \n".format(setup_object.wd)
messagetext += "<P><b><FONT COLOR='#008080'>"\
"Data file: {0} </FONT></b></P> \n".format(op.split(setup_object.datafile)[1])
except:
messagetext += "<P><b><FONT COLOR='#800000'>Error: Could not "\
"write data file: {0} </FONT></b></P> \n".format(setup_object.datafile)
returnvalue = 1
QtGui.QMessageBox.about(self, "Data file generation", messagetext)
#---------------
#2. other input files:
D = self.parameters
#datafile = D['datafile']
messagetext = ''
# try:
# #1. make startup file
# self._setup_startupfile()
# except:
# messagetext += "<P><b><FONT COLOR='#800000'>Error: Could not generate startup file! </FONT></b></P> \n"
if olddatafile is not None:
try:
setup_object = MTo2.Setup(**D)
except:
messagetext += "<P><b><FONT COLOR='#800000'>Error: Could not "\
"generate setup object - check input parameters! </FONT></b></P> \n"
QtGui.QMessageBox.about(self, "Input files generation",
messagetext)
return 1
# edi_dir = D['edi_dir']
# if not D['check_usedatafile']:
# try:
# setup_object.read_edifiles(edi_dir)
# setup_object.datafile = D['datafilename']
# setup_object.write_datafile()
# messagetext += "<P><b><FONT COLOR='#008080'>Wrote "\
# "data file: {0} </FONT></b></P> \n".format(setup_object.datafile)
# except:
# messagetext += "<P><b><FONT COLOR='#800000'>Error: Could not "\
# "write data file: {0} </FONT></b></P> \n".format(setup_object.datafile)
# QtGui.QMessageBox.about(self, "Data file generation", messagetext )
try:
setup_object.setup_mesh_and_model()
except:
messagetext += "<P><b><FONT COLOR='#800000'>Error: Could not "\
"set up mesh and model! </FONT></b></P> \n"
returnvalue = 1
messagetext += "<P><FONT COLOR='#000000'>Working directory: "\
"{0} </FONT></P> \n".format(setup_object.wd)
try:
setup_object.write_meshfile()
messagetext += "<P><b><FONT COLOR='#008080'>"\
"Mesh file: {0} </FONT></b></P> \n".format(setup_object.meshfile)
except:
messagetext += "<P><b><FONT COLOR='#800000'>Error: Could not "\
"write mesh file: {0} </FONT></b></P> \n".format(setup_object.meshfile)
returnvalue = 1
try:
setup_object.write_inmodelfile()
messagetext += "<P><b><FONT COLOR='#008080'>"\
"Inmodel file: {0} </FONT></b></P> \n".format(setup_object.inmodelfile)
except:
messagetext += "<P><b><FONT COLOR='#800000'>Error: Could not "\
"write inmodel file: {0} </FONT></b></P> \n".format(setup_object.inmodelfile)
returnvalue = 1
if D['check_useiterationfile']:
try:
setup_object.startupfile = D['iterationfile']
base, short_fn = op.split(setup_object.startupfile)
if base != setup_object.wd:
new_startupfile = op.abspath(
op.join(setup_object.wd, short_fn))
shutil.copy(setup_object.startupfile, new_startupfile)
setup_object.startupfile = short_fn
messagetext += "<P><b><FONT COLOR='#008080'>Using old "\
"iteration file for startup: {0} </FONT></b></P> \n".format(setup_object.startupfile)
D['startupfile'] = D['iterationfile']
except:
messagetext += "<P><b><FONT COLOR='#800000'>Error: Could not "\
"find old iteration file: {0} </FONT></b></P> \nUsing default 'startup' instead ".format(D['iterationfile'])
D['startupfile'] = 'startup'
returnvalue = 1
else:
try:
setup_object.write_startupfile()
messagetext += "<P><b><FONT COLOR='#008080'>"\
"Startup file: {0} </FONT></b></P> \n".format(setup_object.startupfile)
D['startupfile'] = setup_object.startupfile
except:
messagetext += "<P><b><FONT COLOR='#800000'>Error: Could not "\
"write startup file: {0} </FONT></b></P> \n".format(setup_object.startupfile)
D['startupfile'] = 'startup'
returnvalue = 1
try:
setup_object.write_configfile()
messagetext += "<P><b><FONT COLOR='#008080'>"\
"Configuration file: {0} </FONT></b></P> \n".format(op.split(setup_object.configfile)[1])
except:
messagetext += "<P><b><FONT COLOR='#800000'>Error: Could not "\
"write configuration file: {0} </FONT></b></P> \n".format(op.split(setup_object.configfile)[1])
returnvalue = 1
QtGui.QMessageBox.about(self, "Input files generation", messagetext)
return returnvalue
# Full path to save Mare2D data file
m2d_path = os.path.join(savepath, 'MARE2Ddata.dat')
# Whether to solve statics for all stations
solve_statics = False
# Alternatively, pass a list of station names
# Specified stations will have solve_statics==True, all others False
# solve_statics = ['Synth10', 'Synth11', 'Synth12']
# ASCII grid topo file for interpoalting elevation across the profile
surface_file = os.path.join(mtpy_dir, 'examples', 'data',
'AussieContinent_etopo1.asc')
# Generate an Occam2D data object from EDI data
o2d_data = o2d.Data(edi_path=edi_dir, model_mode='1', optimize_line=True,
interpolate_freq=False, res_te_err=20.,
phase_te_err=10., res_tm_err=10., phase_tm_err=5.)
# We need a data file with the non-rotated profile and the rotated
# profile. This is because the elevation will be interpolated over
# the non-projected profile and stations.
rot_o2d_data = deepcopy(o2d_data) # Make a copy because 'write_data_file' will populate data
o2d_data._rotate_to_strike = False
o2d_data.save_path = o2d_path
rot_o2d_data.save_path = rot_o2d_path
o2d_data.write_data_file(data_fn=o2d_path)
rot_o2d_data.write_data_file(data_fn=rot_o2d_path)
# Convert the Occam2D profile to Mare2D
mare_origin, utm_zone, site_locations, site_elevations, site_names, m2d_profile, profile_elevation = \
m2d.occam2d_to_mare2dem(o2d_data, rot_o2d_data, surface_file, elevation_sample_n=300)
def _main_func(self, edipath):
"""
test function should be successful with a edipath
:return:
"""
# path to save to
savepath = self._output_dir
# list of stations
slst = [
edi[0:-4] for edi in os.listdir(edipath) if edi.find('.edi') > 0
]
# create an occam data object
ocd = occam2d.Data(
edi_path=edipath,
station_list=slst,
# interpolate_freq=True,
# freq=np.logspace(-3,1,30)
)
ocd.save_path = savepath
# choose frequency range to invert
# ocd.freq_num = 50
ocd.freq_min = 1
ocd.freq_max = 10000
# ocd.freq_num = 50 # number of frequencies to invert for
###########make data file
# error floors
ocd.res_te_err = 10
ocd.res_tm_err = 10
ocd.phase_te_err = 5
ocd.phase_tm_err = 5
# ocd.model_mode= 4
ocd.write_data_file()
# make model and mesh files
ocr = occam2d.Regularization(ocd.station_locations)
# number of layers
ocr.n_layers = 60
# cell width to aim for, note this is the mesh size (2 mesh blocks per model block)
ocr.cell_width = 200
# controls number and size of padding
ocr.num_x_pad_cells = 9
ocr.x_pad_multiplier = 1.9
# controls aspect ratio of blocks
ocr.trigger = 0.25
# z1 layer and target depth in metres
ocr.z1_layer = 20
ocr.z_target_depth = 10000
ocr.num_z_pad_cells = 10
ocr.z_bottom = 100000
ocr.save_path = ocd.save_path
ocr.build_mesh()
ocr.build_regularization()
ocr.write_mesh_file()
ocr.write_regularization_file()
ocr.plot_mesh()
plt_wait(1)
plt_close()
# make startup file
ocs = occam2d.Startup()
ocs.iterations_to_run = 40
ocs.data_fn = os.path.join(ocd.save_path, 'OccamDataFile.dat')
ocs.resistivity_start = 2.0
ocr.get_num_free_params()
ocs.param_count = ocr.num_free_param
ocs.save_path = ocd.save_path
ocs.model_fn = ocr.reg_fn
ocs.write_startup_file()
return savepath
def write_mare2dem_data(o2d_filepath, site_locations, site_elevations, site_names,
mare_origin, utm_zone, gstrike, solve_statics=False, savepath=None):
"""
Uses an Occam2D data file and site locations + elevations to
generate a MARE2DEM data file.
Parameters
----------
o2d_filepath : bytes or str
Full path to the Occam2D data file created from EDI data.
site_locations : np.ndarray
Array of shape (nstations). According to the original comments
in the `EDI2Mare2DEM_withOccam2D_new` script, this is the
site elevation but in Mare2D coordinates, however it gets set
as the 'y' component of receiver locations in the Mare2DEM
data file.
site_elevations : np.ndarray
Array of shape (nstations). I think this is the site elevation
in UTM coordinates. Gets set as the 'z' component of receiver
locations in the Mare2DEM data file.
site_names : np.ndarray
Array of site_names.
mare_origin : tuple
Tuple of float (x, y, utm_zone). The Mare2D origin in UTM
coordinates. Note according to the original script, Mare2D origin
is the middle of the profile line. utm_zone is the UTM string,
e.g. '54S'.
gstrike : int
The 2D strike, same as
`geoelectric_strike` in Occam2D model. This information is used
for the UTM origin line in the data file.
solve_statics : bool or list, optional
If boolean, sets whether to solve statics for all stations.
A list of site names can be passed. Sites in the list will
have solve_statics set to True, any sites not included
are set to False. This writes a '1' or a '0' in
the Receiver section of the Mare2D data file in the SovleStatics
column.
savepath : bytes or str, optional
Full path of where to save the Mare2D data file. If not
provided, will be saved as 'Mare2D_data.txt' in working
directory.
"""
# Prepare O2D data for data block
o2d_sites = []
o2d_freqs = []
o2d_types = []
o2d_datums = []
o2d_errors = []
with open(o2d_filepath, 'r') as f:
read_data = f.readlines()
reading_data = False
for line in read_data:
if line.startswith('SITE '):
reading_data = True
continue
elif reading_data:
parts = line.split()
o2d_sites.append(parts[0])
o2d_freqs.append(parts[1])
o2d_types.append(parts[2])
o2d_datums.append(parts[3])
o2d_errors.append(parts[4])
sites = np.array(o2d_sites, dtype=np.int8)
freqs = np.array(o2d_freqs, dtype=np.int8)
types = np.array(o2d_types, dtype=np.int8)
datums = np.array(o2d_datums, dtype=np.float64)
errors = np.array(o2d_errors, dtype=np.float64)
# Convert occam2d types to mare2d types
# The below is: for each element in types array, return corresponding element in conversion
# dict, if not found in dict return original element
type_conversion = {1: 123, 2: 104, 3: 133, 4: 134, 5: 125, 6: 106, 9: 103, 10: 105}
types = np.vectorize(lambda x: type_conversion.get(x, x))(types)
# Put into dataframe for easier stringifying
# Note: TX# == RX# == site ID for MT stations
data_df = pd.DataFrame((types, freqs, sites, sites, datums, errors), dtype=np.object).T
# Bit of a hack: add the '!' to the data frame header because the 'type' integer is small
# enough that the 'Type' header will have no left whitespace padding, so we can't prepend
# it with '!' without throwing off the alignment.
data_df.columns = ['! Type', 'Freq #', 'Tx #', 'Rx #', 'Data', 'StdErr']
data_str = data_df.to_string(index=False, float_format=lambda x: '%.4f' % x)
# Prepare data for the Reciever block
# Zeros of shape (n_sites) for X (as float), Theta, Alpha, Beta and Length (ints) columns
x_col = np.zeros(site_locations.shape, dtype=np.float64)
zero_ints = np.zeros(site_locations.shape, dtype=np.int8)
t_col, a_col, b_col, l_col = zero_ints, zero_ints, zero_ints, zero_ints
# add 0.1 m (shift the sites 10 cm beneath subsurface as recommended)
site_elevations += 0.1
# According to original script, need to reread the Occam2D file to get stations in the correct
# order
if isinstance(solve_statics, bool):
statics = np.ones(site_locations.shape, dtype=np.int8) if solve_statics else zero_ints
else:
statics = np.zeros(site_locations.shape)
for sn in solve_statics:
statics[np.where(site_names == sn)] = 1
# Put into dataframe for easier stringifying
recv_df = pd.DataFrame((x_col, site_locations, site_elevations, t_col, a_col, b_col, l_col,
statics, site_names)).T
recv_df.columns = ['X', 'Y', 'Z', 'Theta', 'Alpha', 'Beta', 'Length', 'SolveStatic', 'Name']
recv_str = list(recv_df.to_string(index=False, float_format=lambda x: '%.6f' % x))
# Replace the first char of header with Mare2DEM comment symbol '!'
# This way the header is correct but Pandas handles the alignment and spacing
recv_str[0] = '!'
recv_str = "".join(recv_str)
o2d_data = o2d.Data()
o2d_data.read_data_file(o2d_filepath)
if savepath is None:
savepath = os.path.join(os.getcwd(), 'Mare2D_data.txt')
with open(savepath, 'w') as output:
# 1. header
fstring = 'Format: EMData_2.2\n'
fstring += 'UTM of x,y origin (UTM zone, N, E, 2D strike):'
gstrike = float(gstrike)
fstring += ' {:s}{:>13.1f}{:>13.1f}\t{:f}\n'.format(
utm_zone, mare_origin[0], mare_origin[1], gstrike)
# 2. frequencies
fstring += '# MT Frequencies: {}\n'.format(len(o2d_data.freq))
fstring += '\n'.join([str(round(f, 8)) for f in o2d_data.freq])
# 3. receiver info
fstring += '\n# MT Receivers: {}\n'.format(len(site_names))
fstring += recv_str
fstring += '\n'
# 4. data
fstring += '# Data: {}\n'.format(len(datums))
fstring += data_str
output.write(fstring)
# -*- coding: utf-8 -*-
"""
Created on Wed Sep 23 13:51:54 2015
@author: jpeacock
"""
import mtpy.modeling.occam2d as occam
s_edi_path = r"c:\Users\jpeacock\Documents\MountainPass\EDI_Files_birrp\Edited\geographic_north\final_edi"
s_list = ["mp1{0:02}".format(ii) for ii in range(1, 20)]
ocd = occam.Data(edi_path=s_edi_path, station_list=s_list)
ocd.model_mode = "log_tm"
ocd.phase_tm_err = 1.4
ocd.res_tm_err = 30
ocd.save_path = r"c:\MinGW32-xy\Peacock\occam\MountainPass\inv06_tm_rot"
# ocd.geoelectric_strike = 70.0
ocd._rotate_to_strike = True
ocd.write_data_file()
ocm = occam.Regularization(station_locations=ocd.station_locations)
ocm.cell_width = 150
ocm.n_layers = 80
ocm.z1_layer = 10
ocm.z_target_depth = 40000
ocm.x_pad_multiplier = 1.5
ocm.num_x_pad_cells = 9
ocm.num_x_pad_small_cells = 3
ocm.build_mesh()
ocm.save_path = ocd.save_path
# path to save to
savepath = r'/home/workshop/MT/Occam2d/model_input_output'
if not op.exists(savepath):
os.mkdir(savepath)
# geoelectric strike for rotation
strike = 0
# list of stations
slst = [edi[0:-4] for edi in os.listdir(edipath) if edi.find('.edi') > 0]
# create an occam data object
ocd = occam2d.Data(
edi_path=edipath,
station_list=slst,
)
ocd.save_path = savepath
# choose frequency range to invert
ocd.freq_min = 1
ocd.freq_max = 10000
###########make data file
ocd.geoelectric_strike = strike
ocd._rotate_to_strike = False
# error floors
ocd.res_te_err = 10
wd = r'C:\Users\roberk20\Dropbox\Lithospheric_Architecture_Shared_Folder\Software\MARE2DEM_COMPILED\Inversions\Musgraves\mare2d'
epath = wd #specify path to EDIs here if different to the working directory above
solve_statics = False #whether you want the code to solve for static shift (at the moment only coded to have all sites the same, could set independently though)
csv_fn = r'C:\Users\roberk20\Dropbox\Lithospheric_Architecture_Shared_Folder\Software\MARE2DEM_COMPILED\DataFile\etopo1_bedrock_smaller_UTM__127_131_-32_-23.csv'
# =============================================================================
elst = [op.join(epath, ef) for ef in os.listdir(epath)
if ef.endswith('.edi')] #make a list of EDIs
slst = [edi[0:-4] for edi in os.listdir(epath)
if edi.find('.edi') > 0] #make a list of MT stations
gstrike = -72
data = o2d.Data(edi_path=epath,
model_mode='1',
station_list=slst,
interpolate_freq=False,
geoelectric_strike=gstrike,
res_te_err=20.,
phase_te_err=10.,
res_tm_err=10.,
phase_tm_err=5.)
data._fill_data()
data.save_path = wd
data.write_data_file()
data_fn = op.join(data.save_path, 'OccamDataFile.dat')
#data_slst=data.station_list
#data_sloc=data.station_locations
#read in the Occam data file that was just created, to get the station list in the right order (different to the order of data.station_list)
newData = o2d.Data()
newData.read_data_file(data.data_fn)
newData_slst = newData.station_list
newData_sloc = newData.station_locations
# path to save to
savepath = r"C:\Git\mtpy\examples\model_files\Occam2d"
if not op.exists(savepath):
os.mkdir(savepath)
# list of stations
slst=[edi[0:-4] for edi in os.listdir(edipath) if edi.find('.edi')>0]
# create an occam data object
ocd = occam2d.Data(edi_path=edipath,
station_list=slst,
# interpolate_freq=True,
# freq=np.logspace(-3,1,30)
)
ocd.save_path = savepath
# choose frequency range to invert
#ocd.freq_num = 50
ocd.freq_min = 1
ocd.freq_max = 10000
#ocd.freq_num = 50 # number of frequencies to invert for
###########make data file
# error floors
