def process_edi_files( input ):
print ("Directory for edi files or single file ---------> {}".format(input))
edis =[]
if not ( " " in input ):
if not os.path.isdir(input):
print ("Invalid Ditectory Input")
sys.exit()
if os.path.isdir(input):
edis = glob.glob(input + '/*.edi')
print(edis)
if len(edis) == 0 :
print ("Directory edi files {} empty".format(input))
sys.exit()
obj = EdiCollection(edilist=edis)
else:
edis = input.split(' ')
for fl in edis:
if not os.path.isfile(fl):
print ("Invalid Input File {}".format(fl))
sys.exit()
obj = EdiCollection(edilist=edis)
# Compute distances
min_dist, max_dist = obj.get_min_max_distance(obj)
# mt_distances = obj.get_stations_distances_stats()
# min_dist = mt_distances.get("MIN_DIST")
# max_dist = mt_distances.get("MAX_DIST")
print("Min Distance = {}".format(min_dist))
print("Max Distance = {}".format(max_dist))
def write_readme(self, fname):
with open(fname, 'w') as outf:
outf.write("{} Data File Generated by MTPy2\n".format(
"ModEM" if self._data_kwargs['format'] == '1' else "WinMT (wrong name)")
)
outf.write("====================\n")
outf.write("Created on: {}\n".format(str(datetime.datetime.now())))
outf.write("\n")
outf.write("**NOTE** All paths are modified to hide identity. \"[*]\" indicates the hidden path.\n")
outf.write("## EDI Files:\n")
# hide path that may review user's identity
outf.write("\n".join([
os.path.normpath(
os.path.join("[*]/" + os.path.basename(os.path.dirname(edi_file)), os.path.basename(edi_file))
) for edi_file in self._edi_list
]))
outf.write("\n\n")
outf.write("## Parameters Used:\n\n")
outf.write("### Period Included\n")
pprint.pprint(self._select_period_kwargs, stream=outf)
outf.write("\n")
outf.write("\nSelected Period List:\n")
pprint.pprint(EdiCollection(self._edi_list).select_periods( **self._select_period_kwargs), stream=outf)
outf.write("\n")
outf.write("### Data:\n")
kwargs = self._data_kwargs.copy()
# hide path that may review user's identity
kwargs["save_path"] = os.path.normpath("[*]/" + os.path.basename(kwargs["save_path"]))
pprint.pprint(kwargs, stream=outf)
outf.write("\n")
outf.write("### Mesh Model:\n")
kwargs = self._mesh_kwagrs.copy()
# hide path that may review user's identity
kwargs["save_path"] = os.path.normpath("[*]/" + os.path.basename(kwargs["save_path"]))
pprint.pprint(kwargs, stream=outf)
outf.write("\n")
outf.write("### Topography:\n")
kwargs = self._topo_args.copy()
# hide path that may review user's identity
kwargs["topography_file"] = os.path.normpath("[*]/" + os.path.basename(kwargs["topography_file"]))
pprint.pprint(kwargs, stream=outf)
outf.write("\n")
outf.write("### Covariance:\n")
kwargs = self._covariance_kwargs.copy()
# hide path that may review user's identity
kwargs["save_path"] = os.path.normpath("[*]/" + os.path.basename(kwargs["save_path"]))
pprint.pprint(kwargs, stream=outf)
outf.write("\n")
outf.write("## Appendix I - Data Parameter Definition\n")
outf.write(Data.__doc__)
outf.write("\n")
outf.write("## Appendix II - Mesh Model Parameter Definition\n")
outf.write(Model.__doc__)
outf.write("\n")
outf.write("## Appendix III - Covariance Parameter Definition\n")
outf.write(Covariance.__doc__) # todo update Covariance class document
outf.write("\n")
def _export_shape_file(self, *args, **kwargs):
# show files
msg = QMessageBox()
msg.setIcon(QMessageBox.Information)
msg.setText("You are about to create shape files.")
msg.setInformativeText(
"Please select an output directory after click \"OK\"\n"
"For the list of .edi files (stations) included in the creation, please click \"Show Details\""
)
msg.setWindowTitle("Note")
msg.setDetailedText("\n".join([
"{station} ({fn})".format(
station=station, fn=self._file_handler.station2ref(station))
for station in self._station_viewer.selected_stations
]))
msg.setStandardButtons(QMessageBox.Ok | QMessageBox.Cancel)
if msg.exec_() == QMessageBox.Ok:
dialog = QFileDialog(self)
dir_name = None
dialog.setWindowTitle("Selecting Output Directory ...")
dialog.setFileMode(QFileDialog.DirectoryOnly)
while dir_name is None:
if dialog.exec_() == QDialog.Accepted:
dir_name = dialog.selectedFiles()[0]
dir_name = str(dir_name)
if not os.path.isdir(dir_name):
QMessageBox.information(
self, "NOTE",
"Please select a directory to save the created shape files."
)
dir_name = None # will read again
else:
break
if dir_name is not None:
collect = EdiCollection(mt_objs=[
self._file_handler.get_MT_obj(
self._file_handler.station2ref(station))
for station in self._station_viewer.selected_stations
])
collect.create_mt_station_gdf(dir_name)
QMessageBox.information(self, "Creation Completed",
"Output written to %s" % dir_name)
webbrowser.open(dir_name)
def plot_many_periods(edidir, n_periods=5):
from mtpy.core.edi_collection import EdiCollection
edilist = glob.glob(os.path.join(edidir, '*.edi'))
ediset = EdiCollection(edilist)
for period_sec in ediset.all_unique_periods[:n_periods]:
try:
# This will enable the loop continue even though for some freq,
# cannot interpolate due to not enough data points
plot_latlon_depth_profile(edidir, period_sec, zcomponent='det', showfig=False)
except Exception as exwhy:
print(str(exwhy))
def _test_func(self):
if not os.path.isdir(edi_path):
# input file does not exist, skip test after remove the output dir
os.rmdir(self._output_dir)
self.skipTest("edi path does not exist: {}".format(edi_path))
# generate data
edi_list = glob.glob(edi_path + '/*.edi')
period_list = EdiCollection(edi_list).select_periods()
datob = Data(edi_list=edi_list,
inv_mode='1',
period_list=period_list,
epsg=epsg_code,
error_type_tipper='abs',
error_type_z='egbert',
comp_error_type=None,
error_floor=10)
datob.write_data_file(save_path=self._output_dir)
# create mesh grid model object
model = Model(
stations_object=datob.station_locations,
Data=datob,
epsg=epsg_code,
cell_size_east=10000,
cell_size_north=10000, # GA_VIC
pad_north=
8, # number of padding cells in each of the north and south directions
pad_east=8, # number of east and west padding cells
pad_z=8, # number of vertical padding cells
pad_stretch_v=
1.5, # factor to increase by in padding cells (vertical)
pad_stretch_h=
1.5, # factor to increase by in padding cells (horizontal)
n_air_layers=
0, # number of air layers 0, 10, 20, depend on topo elev height
res_model=
100, # halfspace resistivity value for initial reference model
n_layers=50, # total number of z layers, including air and pad_z
z1_layer=50, # first layer thickness metres, depend
z_target_depth=500000)
model.make_mesh(
) # the data file will be re-write in this method. No topo elev file used yet
model.plot_mesh()
model.plot_mesh_xy()
model.plot_mesh_xz()
# write a model file and initialise a resistivity model
model.write_model_file(save_path=self._output_dir)
def test_func(self):
if not os.path.isdir(edi_path):
# input file does not exist, skip test after remove the output dir
os.rmdir(self._output_dir)
self.skipTest("edi path does not exist: {}".format(edi_path))
edi_list = glob.glob(edi_path + '/*.edi')
period_list = EdiCollection(edi_list).select_periods()
datob = Data(edi_list=edi_list,
inv_mode='1',
period_list=period_list,
epsg=epsg_code,
error_type_tipper=error_type_tipper,
error_type_z=error_type_z,
comp_error_type=comp_error_type,
error_floor=10)
datob.write_data_file(save_path=self._output_dir)
# check the output
if self._expected_output_dir:
output_data_file = os.path.normpath(os.path.join(self._output_dir, "ModEM_Data.dat"))
self.assertTrue(os.path.isfile(output_data_file), "output data file does not exist")
expected_data_file = os.path.normpath(os.path.join(self._expected_output_dir,
"ModEM_Data.dat"))
self.assertTrue(
os.path.isfile(expected_data_file),
"expected output data file does not exist, nothing to compare"
)
print("\ncomparing", output_data_file, "and", expected_data_file)
with open(output_data_file, 'r') as output:
with open(expected_data_file, 'r') as expected:
diff = difflib.unified_diff(
expected.readlines(),
output.readlines(),
fromfile='expected',
tofile='output'
)
count = 0
for line in diff:
sys.stdout.write(line)
count += 1
self.assertTrue(count == 0, "output different!")
else:
print("no expected output exist, nothing to compare")
import glob
import numpy as np
from mtpy.core.edi_collection import EdiCollection
# directory format for linux users
#edi_path = "../data/edi_files"
#savepath '/tmp'
# directory format for windows users
edi_path = r'C:\mtpywin\mtpy\examples\data\edi_files'
savepath = r'C:\tmp'
edi_files = glob.glob(
os.path.normpath(os.path.abspath(os.path.join(edi_path, "*.edi"))))
edi_collection = EdiCollection(edi_files)
# period_list to interpolate onto
period_list = np.logspace(-4, 3, 43)
edi_collection.export_edi_files(
savepath,
period_list=period_list, # if not provided, will search edi files and find
# periods present in at least 10% of edis
period_buffer=
2 # factor to stretch interpolation by. For example: if period_buffer=2
# then interpolated data points will only be included if they are
# within a factor of 2 of a true data point.
)
def setUp(self):
_BaseTest.setUp(self)
mt_objs = [MT(edi_file) for edi_file in self.edi_files]
self.edi_collection = EdiCollection(mt_objs=mt_objs)
def setUp(self):
_BaseTest.setUp(self)
self.edi_collection = EdiCollection(self.edi_files)
import os
import glob
import numpy as np
from mtpy.core.edi_collection import EdiCollection
# directory format for linux users
#edi_path = "../data/edi_files"
#savepath '/tmp'
# directory format for windows users
edi_path = r'C:\mtpywin\mtpy\examples\data\edi_files'
savepath = r'C:\tmp'
edi_files = glob.glob(os.path.normpath(os.path.abspath(os.path.join(edi_path, "*.edi"))))
edi_collection = EdiCollection(edi_files)
# period_list to interpolate onto
period_list = np.logspace(-4, 3, 43)
edi_collection.create_phase_tensor_csv(savepath,
period_list=period_list # list of periods, if provided it
# interpolates onto those frequencies,
# if blank it exports all periods in file.
)
temp_dir = tempfile.gettempdir()
print('Using temporary directory ' + temp_dir)
savepath = temp_dir
# savepath = r'C:/tmp'
if not os.path.isdir(edidir):
print("please provide the path to edi folder")
sys.exit(1)
edifiles = glob.glob(os.path.join(edidir, "*.edi"))
print(
"***************** Stats analysis on the EDI files [periods list] *******************************"
)
edis_obj = EdiCollection(edilist=edifiles)
edis_obj.select_periods(percentage=5.0)
print(
"***************** the periods with high occurance in EDI files **************************"
)
per_freq = edis_obj.get_periods_by_stats(
percentage=99.0) # occur in 99% of EDI files
print(per_freq)
for aper in per_freq:
print(aper, edis_obj.get_period_occurance(aper))
print(
"***************** For each uniqu period, show it's occurance in the EDI files ************"
)
for aper in edis_obj.all_unique_periods:
# change the variable below according to your edi files folder
edidir = r'c:\Users\jpeacock\OneDrive - DOI\Geysers\EDI_Files_birrp\Edited'
savepath = r'c:\Users\jpeacock\OneDrive - DOI\Geysers\jvgr\figures'
if not os.path.isdir(edidir):
print("please provide the path to edi folder")
sys.exit(1)
edifiles = glob.glob(os.path.join(edidir, "*.edi"))
# Create plot for a period index number 1,2,3 ..., 10 for determinant. This may not make sense sometimes.
# change to your preferred file resolution
#pen3d.plot_latlon_depth_profile(edidir, 4, 'det', showfig=True, savefig=True,
# savepath=savepath, fig_dpi=400)
# The recommended way is to use a float value for the period.
#############################################################
# More Stats analysis on the EDI files periods
edis_obj = EdiCollection(edilist=edifiles)
edis_obj.select_periods(percentage=5.0)
per_freq = edis_obj.get_periods_by_stats(percentage=80.0)
for aper in range(-15, 0, -1):
# pen3d.plot_bar3d_depth(edidir, aper)
pen3d.plot_latlon_depth_profile(edidir, aper, showfig=True, savefig=True,
savepath=savepath,
fig_dpi=600)
if len(sys.argv) > 2:
path2out = sys.argv[2]
else:
path2out = None
# filter the edi files here if desired, to get a subset:
# edifiles2 = edifiles[0:-1:2]
shp_maker = ShapefilesCreator(edifiles, path2out)
# ptdic = shp_maker.create_csv_files_deprecated() # dest_dir=path2out) # create csv files E:/temp1
ptdic = shp_maker.create_phase_tensor_csv(path2out) # compare csv in E:/temp2
# print ptdic
# print ptdic[ptdic.keys()[0]]
# edisobj = mtpy.core.edi_collection.EdiCollection(edifiles)
edisobj = EdiCollection(edifiles)
bbox_dict = edisobj.bound_box_dict
print(bbox_dict)
bbox_dict2 = shp_maker.bound_box_dict
print(bbox_dict2)
if bbox_dict != bbox_dict2:
raise Exception("parent-child's attribute bbo_dic not equal!!!")
# create shapefiles and plots
# epsg projection 4283 - gda94
# process_csv_folder(path2out, bbox_dict)
# Further testing epsg codes:
# epsg projection 28354 - gda94 / mga zone 54
# epsg projection 32754 - wgs84 / utm zone 54s
epsg_code = 3112
edi_list = glob.glob(edipath + '/*.edi')
if edi_list is None or (edi_list) < 1:
print("Error: No edi files found in the dir %s" % edipath)
sys.exit(2)
# period list (can take periods from one of the edi files, or just specify
# periods directly using the logspace function (commented out))
# eo = mtedi.Edi(edi_list[0]) # this may miss some periods?
# period_list = 1. / eo.Z.freq # period_list = np.logspace(-3,3)
print("edi_list = {}".format(edi_list))
period_list = EdiCollection(edi_list).select_periods()
datob = Data(edi_list=edi_list,
inv_mode='1',
period_list=period_list,
epsg=epsg_code,
error_type='floor',
error_floor=10)
# period_buffer=0.000001)
datob.write_data_file(save_path=outputdir)
# create mesh grid model object
# model = Model(Data=datob,
model = Model(
station_object=datob.station_locations,
def run(self):
# monkey patch plt.show() so the plot is not displayed in worker thread
true_plt_show = plt.show
plt.show = _fake_plt_show
self._figures = []
try:
if not os.path.exists(self.output_dir):
os.mkdir(self.output_dir)
# get period_list list
self.status_updated.emit("Selecting Periods...")
period_list = EdiCollection(self._edi_list).select_periods(**self._select_period_kwargs)
# save period plot for reference
figure = plt.gcf()
figure.savefig(os.path.join(self.output_dir, self._period_image_name))
if self.show:
self.figure_updated.emit('Period Distribution', figure)
# data object
self.status_updated.emit("Creating ModEM Data Object...")
self._data_kwargs['period_list'] = period_list
data = Data(edi_list=self._edi_list, **self._data_kwargs)
# write data file
self.status_updated.emit("Writing Initial ModEM Data File...")
data.write_data_file()
# create model
self.status_updated.emit("Creating Mesh Model...")
model = Model(data_object=data, **self._mesh_kwagrs)
model.make_mesh()
# plot mesh
model.plot_mesh(fig_num=plt.gcf().number + 1)
figure = plt.gcf()
figure.savefig(os.path.join(self.output_dir, self._mesh_image_name))
if self.show:
self.figure_updated.emit("Mesh", figure)
# model.plot_mesh_xy()
# self.figure_updated.emit("Mesh XY", plt.gcf())
# model.plot_mesh_xz()
# self.figure_updated.emit("Mesh XZ", plt.gcf())
model.write_model_file()
# add topography
self.status_updated.emit("Adding Topography...")
model.add_topography_to_mesh(**self._topo_args)
if self.show:
model.plot_topograph() # this is too slow so only plot and save image when asked
figure = plt.gcf()
figure.savefig(os.path.join(self.output_dir, self._topo_image_name))
self.figure_updated.emit("Topography", figure)
self.status_updated.emit("Updating Mesh Model...")
model.write_model_file()
# covariance
self.status_updated.emit("Creating Covariance File...")
self._covariance_kwargs['mask_arr'] = model.covariance_mask
cov = Covariance(**self._covariance_kwargs)
self.status_updated.emit("Writing Covariance File...")
cov.write_covariance_file(model_fn=model.model_fn, sea_water=self._topo_args['sea_resistivity'],
air=self._topo_args['air_resistivity'])
self.status_updated.emit("Creating README File...")
self.write_readme(os.path.join(self.output_dir, self._readme_name))
# done
self.status_updated.emit("Finishing...")
except Exception as e:
frm = inspect.trace()[-1]
mod = inspect.getmodule(frm[0])
self.export_error.emit("{}: {}".format(mod.__name__, e.message))
# restore plt.show()
plt.show = true_plt_show