def loadData(self, filename, **kwargs):
"""Import spectral data.
Import Data and try to assume the file format.
"""
verbose = kwargs.pop('verbose', self._verbose)
with codecs.open(filename,
'r',
encoding='iso-8859-15',
errors='replace') as f:
firstLine = f.readline()
f.close()
fnLow = filename.lower()
if 'SIP Fuchs III' in firstLine:
if verbose:
pg.info("Reading SIP Fuchs III file")
self.f, self.amp, self.phi, header = readFuchs3File(
filename, verbose=verbose, **kwargs)
self.phi *= -np.pi / 180.
# print(header) # not used?
elif 'SIP-Fuchs Software rev.: 070903' in firstLine:
if verbose:
pg.info("Reading SIP Fuchs file")
self.f, self.amp, self.phi, drhoa, dphi = readRadicSIPFuchs(
filename, verbose=verbose, **kwargs)
self.phi *= -np.pi / 180.
elif fnLow.endswith('.txt') or fnLow.endswith('.csv'):
self.basename = filename[:-4]
self.f, self.amp, self.phi = readTXTSpectrum(filename)
self.amp *= self.k
return self.f, self.amp, self.phi
def saveResult(self, folder=None, size=(16, 10), **kwargs):
"""Save all results in the specified folder.
Saved items are:
Inverted profile
Resistivity vector
Coverage vector
Standardized coverage vector
Mesh (bms and vtk with results)
"""
subfolder = self.__class__.__name__
path = getSavePath(folder, subfolder)
pg.info('Saving resistivity data to: {}'.format(path))
np.savetxt(path + '/resistivity.vector', self.model)
np.savetxt(path + '/resistivity-cov.vector', self.coverage())
np.savetxt(path + '/resistivity-scov.vector',
self.standardizedCoverage())
m = pg.Mesh(self.paraDomain)
m['Resistivity'] = self.paraModel(self.model)
m['Resistivity (log10)'] = np.log10(m['Resistivity'])
m['Coverage'] = self.coverage()
m['S_Coverage'] = self.standardizedCoverage()
m.exportVTK(os.path.join(path, 'resistivity'))
m.saveBinaryV2(os.path.join(path, 'resistivity-pd'))
self.fop.mesh().save(os.path.join(path, 'resistivity-mesh'))
if self.paraDomain.dim() == 2:
fig, ax = plt.subplots(figsize=size)
self.showResult(ax=ax, coverage=self.coverage(), **kwargs)
fig.savefig(path + '/resistivity.pdf', bbox_inches="tight")
return path, fig, ax
return path
def createGeometricFactors(scheme, numerical=None, mesh=None, verbose=False):
"""Create geometric factors for a data scheme.
Create geometric factors for a data scheme with and without topography.
Calculation will be done analytical (only for half space geometry)
or numerical.
This function caches the result depending on scheme, mesh and pg.version()
Parameters
----------
scheme: :gimliapi:`GIMLI::DataContainerERT`
Datacontainer of the scheme.
numerical: bool | None [False]
If numerical is None, False is assumed, we try to guess topography
and warn if we think we found them.
If set to True or False, numerical calculation will used respectively.
mesh: :gimliapi:`GIMLI::Mesh` | str
Mesh for numerical calculation. If not given, analytical geometric
factors for halfspace earth are guessed or a default mesh will be
created. The mesh will be h and p refined. If given topo is set to
True. If the numerical effort is to high or the accuracy to low
you should consider to calculate the factors manual.
verbose: bool
Give some output.
"""
if numerical is None:
numerical = False
if (min(pg.z(scheme)) != max(pg.z(scheme))):
verbose = True
pg.warn('Sensor z-coordinates not equal. Is there topography?')
if numerical is False and mesh is None:
if verbose:
pg.info('Calculate analytical flat earth geometric factors.')
return pg.core.geometricFactors(scheme, forceFlatEarth=True)
if mesh is None:
mesh = createInversionMesh(scheme)
if verbose:
pg.info('mesh', mesh)
m = mesh.createH2()
if verbose:
pg.info('mesh-h2', m)
m = m.createP2()
if verbose:
pg.info('mesh-p2', m)
pg.info('Calculate numerical geometric factors.')
d = simulate(m,
res=1.0,
scheme=scheme,
sr=False,
useBert=True,
calcOnly=True,
verbose=True)
return 1. / d['u']
def __init__(self, A, verbose=False):
"""Constructor saving matrix and vector.
Parameters
----------
A : ndarray
numpy type (full) matrix
"""
super().__init__(verbose) # only in Python 3
self._mul = None
if isinstance(A, str):
self.load(A)
else:
from scipy.linalg import eigh # , get_blas_funcs
if A.shape[0] != A.shape[1]: # rows/cols for pgcore matrix
raise Exception("Matrix must by square (and symmetric)!")
if verbose:
t = pg.tic(key='init cm05')
self.ew, self.EV = eigh(A)
if verbose:
pg.info(
'(C) Time for eigenvalue decomposition: {:.1f}s'.format(
pg.dur(key='init cm05')))
def restore(self):
"""Read data from json infos"""
if os.path.exists(self._name + '.json'):
# Fricking mpl kills locale setting to system default .. this went
# horrible wrong for german 'decimal_point': ','
pg.checkAndFixLocaleDecimal_point(verbose=False)
try:
with open(self._name + '.json') as file:
self.info = json.load(file)
# if len(self.info['type']) != 1:
# pg.error('only single return caches supported for now.')
#pg._y(pg.pf(self.info))
if self.info['type'] == 'DataContainerERT':
self._value = pg.DataContainerERT(self.info['file'],
removeInvalid=False)
# print(self._value)
elif self.info['type'] == 'RVector':
self._value = pg.Vector()
self._value.load(self.info['file'], format=pg.core.Binary)
elif self.info['type'] == 'Mesh':
pg.tic()
self._value = pg.Mesh()
self._value.loadBinaryV2(self.info['file'] + '.bms')
pg.debug("Restoring cache took:", pg.dur(), "s")
elif self.info['type'] == 'ndarray':
self._value = np.load(self.info['file'] + '.npy',
allow_pickle=True)
elif self.info['type'] == 'Cm05Matrix':
self._value = pg.matrix.Cm05Matrix(self.info['file'])
elif self.info['type'] == 'GeostatisticConstraintsMatrix':
self._value = pg.matrix.GeostatisticConstraintsMatrix(
self.info['file'])
else:
self._value = np.load(self.info['file'] + '.npy',
allow_pickle=True)
if self.value is not None:
self.info['restored'] = self.info['restored'] + 1
self.updateCacheInfo()
pg.info('Cache {3} restored ({1}s x {0}): {2}'.\
format(self.info['restored'],
round(self.info['dur'], 1),
self._name, self.info['codeinfo']))
else:
# default try numpy
pg.warn('Could not restore cache of type {0}.'.format(self.info['type']))
pg.debug("Restoring cache took:", pg.dur(), "s")
except Exception as e:
import traceback
traceback.print_exc(file=sys.stdout)
print(self.info)
pg.error('Cache restoring failed.')
def waitOnExit():
backend = matplotlib.get_backend()
if not 'inline' in backend:
if 'Qt' in backend or 'Wx' in backend:
if len(plt.get_fignums()) > 0:
pg.info('Showing pending widgets on exit. '
'Close all figures or Ctrl-C to quit the programm')
pg.wait()
def saveResult(self, folder=None, size=(16, 10), verbose=False, **kwargs):
"""Save the results in the specified folder.
Saved items are:
* Resulting inversion model
* Velocity vector
* Coverage vector
* Standardized coverage vector
* Mesh (bms and vtk with results)
Args
----
path: str[None]
Path to save into. If not set the name is automatic created
size: (float, float) (16,10)
Figure size.
Keyword Args
------------
Will be forwardet to showResults
Returns
-------
str:
Name of the result path.
"""
# TODO: How to extract the chi2 etc. from each iteration???
subfolder = self.__class__.__name__
path = getSavePath(folder, subfolder)
if verbose:
pg.info('Saving refraction data to: {}'.format(path))
np.savetxt(os.path.join(path, 'velocity.vector'),
self.velocity)
np.savetxt(os.path.join(path, 'velocity-cov.vector'),
self.rayCoverage())
np.savetxt(os.path.join(path, 'velocity-scov.vector'),
self.standardizedCoverage())
m = pg.Mesh(self.paraDomain)
m['Velocity'] = self.paraModel(self.velocity)
m['Coverage'] = self.rayCoverage()
m['S_Coverage'] = self.standardizedCoverage()
m.exportVTK(os.path.join(path, 'velocity'))
m.saveBinaryV2(os.path.join(path, 'velocity-pd'))
self.fop.mesh().save(os.path.join(path, 'velocity-mesh'))
np.savetxt(os.path.join(path, 'chi.txt'), self.inv.chi2History)
fig, ax = plt.subplots()
self.showResult(ax=ax, cov=self.standardizedCoverage(), **kwargs)
fig.set_size_inches(size)
fig.savefig(os.path.join(path, 'velocity.pdf'), bbox_inches='tight')
pg.plt.close(fig)
return path
def setDefaultBackground(self):
"""
"""
regionIds = self.regionManager().regionIdxs()
pg.info("Found {} regions.".format(len(regionIds)))
if len(regionIds) > 1:
bk = pg.sort(regionIds)[0]
pg.info("Region with smallest marker set to background (marker={0})".format(bk))
self.setRegionProperties(bk, background=True)
def createRefinedFwdMesh(self, mesh):
"""Refine the current mesh for higher accuracy.
This is called automatic when accesing self.mesh() so it ensures any
effect of changing region properties (background, single).
"""
pg.info("Creating refined mesh (secnodes: {0}) to "
"solve forward task.".format(self._refineSecNodes))
m = mesh.createMeshWithSecondaryNodes(self._refineSecNodes)
pg.verbose(m)
return m
def simulate(mesh, res, scheme, verbose=False, **kwargs):
"""Forward calculation vor given mesh, data and resistivity."""
fop = ERTModelling(verbose=verbose)
# fop = ERTManager.createFOP(verbose=verbose)
fop.setData(scheme)
fop.setMesh(mesh, ignoreRegionManager=True)
if not scheme.allNonZero('k'):
if min(pg.y(scheme)) != max(pg.y(scheme)) or min(
pg.z(scheme)) != max(pg.z(scheme)):
pg.info(
"Non flat earth topography found. "
"We will set geometric factors to -1 to emulate "
"electrical impedance tomography (EIT). If you want to "
"use ERT will full topography support. "
"Please consider the use of pyBERT.")
scheme.set('k', pg.RVector(scheme.size(), -1))
else:
scheme.set('k', fop.calcGeometricFactors(scheme))
rhoa = None
isArrayData = None
if hasattr(res[0], '__iter__'):
isArrayData = True
rhoa = np.zeros((len(res), scheme.size()))
for i, r in enumerate(res):
rhoa[i] = fop.response(r)
else:
rhoa = fop.response(res)
pg.renameKwarg('noisify', 'noiseLevel', kwargs)
noiseLevel = kwargs.pop('noiseLevel', 0.0)
if noiseLevel > 0:
noiseAbs = kwargs.pop('noiseAbs', 1e-4)
err = noiseLevel + noiseAbs / rhoa
scheme.set('err', err)
if verbose:
pg.info(
"Set noise (" + str(noiseLevel * 100) + "% + " +
str(noiseAbs) + " V) min:", min(err), "max:", max(err))
rhoa *= 1. + pg.randn(scheme.size()) * err
if isArrayData is None:
scheme.set('rhoa', rhoa)
if kwargs.pop('returnArray', False):
return rhoa
return scheme
def setDefaultBackground(self):
"""Set the default background behaviour."""
if self.complex():
self.regionManager().addRegion(3, self._baseMesh, 2)
regionIds = self.regionManager().regionIdxs()
pg.info("Found {} regions.".format(len(regionIds)))
if len(regionIds) > 1:
bk = pg.sort(regionIds)[0]
pg.info("Region with smallest marker ({0}) "
"set to background".format(bk))
self.setRegionProperties(bk, background=True)
def checkData(self, data=None):
"""Return data from container.
THINKABOUT: Data will be changed, or should the manager keep a copy?
"""
data = data or pg.DataContainerERT(self.data)
if isinstance(data, pg.DataContainer):
if not data.allNonZero('k'):
pg.warn("Data file contains no geometric factors (token='k').")
data['k'] = createGeometricFactors(data, verbose=True)
if self.fop.complex():
if not data.haveData('rhoa'):
pg.critical('Datacontainer have no "rhoa" values.')
if not data.haveData('ip'):
pg.critical('Datacontainer have no "ip" values.')
# pg.warn('check sign of phases')
rhoa = data['rhoa']
phia = -data['ip'] / 1000 # 'ip' is defined for neg mrad.
# we should think about some 'phia' in rad
return pg.utils.squeezeComplex(pg.utils.toComplex(rhoa, phia))
else:
if not data.haveData('rhoa'):
if data.allNonZero('r'):
pg.info("Creating apparent resistivies from "
"impedences rhoa = r * k")
data['rhoa'] = data['r'] * data['k']
elif data.allNonZero('u') and data.allNonZero('i'):
pg.info("Creating apparent resistivies from "
"voltage and currrent rhoa = u/i * k")
data['rhoa'] = data['u'] / data['i'] * data['k']
else:
pg.critical("Datacontainer have neither: "
"apparent resistivies 'rhoa', "
"or impedances 'r', "
"or voltage 'u' along with current 'i'.")
if any(data['rhoa'] < 0) and \
isinstance(self.inv.dataTrans, pg.core.TransLog):
print(pg.find(data['rhoa'] < 0))
print(data['rhoa'][data['rhoa'] < 0])
pg.critical("Found negative apparent resistivities. "
"These can't be processed with logarithmic "
"data transformation. You should consider to "
"filter them out using "
"data.remove(data['rhoa'] < 0).")
return data['rhoa']
return data
def simulate(mesh, res, scheme, verbose=False, **kwargs):
"""Forward calculation vor given mesh, data and resistivity."""
fop = ERTModelling(verbose=verbose)
# fop = ERTManager.createFOP(verbose=verbose)
fop.setData(scheme)
fop.setMesh(mesh, ignoreRegionManager=True)
if not scheme.allNonZero('k'):
if min(pg.y(scheme)) != max(pg.y(scheme)) or min(pg.z(scheme)) != max(pg.z(scheme)):
pg.info("Non flat earth topography found. "
"We will set geometric factors to -1 to emulate "
"electrical impedance tomography (EIT). If you want to "
"use ERT will full topography support. "
"Please consider the use of pyBERT.")
scheme.set('k', pg.RVector(scheme.size(), -1))
else:
scheme.set('k', fop.calcGeometricFactors(scheme))
rhoa = None
isArrayData = None
if hasattr(res[0], '__iter__'):
isArrayData = True
rhoa = np.zeros((len(res), scheme.size()))
for i, r in enumerate(res):
rhoa[i] = fop.response(r)
else:
rhoa = fop.response(res)
pg.renameKwarg('noisify', 'noiseLevel', kwargs)
noiseLevel = kwargs.pop('noiseLevel', 0.0)
if noiseLevel > 0:
noiseAbs = kwargs.pop('noiseAbs', 1e-4)
err = noiseLevel + noiseAbs / rhoa
scheme.set('err', err)
if verbose:
pg.info("Set noise (" + str(noiseLevel*100) + "% + " + str(noiseAbs) + " V) min:",
min(err), "max:", max(err))
rhoa *= 1. + pg.randn(scheme.size()) * err
if isArrayData is None:
scheme.set('rhoa', rhoa)
if kwargs.pop('returnArray', False):
return rhoa
return scheme
def createStartModel(self, dataVals=None):
"""Create the default startmodel as the median of the data values.
Overwriting might be a good idea.
Its used by inversion to create a valid startmodel if there are
no starting values from the regions.
"""
if dataVals is not None:
mv = pg.math.median(dataVals)
pg.info("Set default startmodel to median(data values)={0}".format(mv))
sm = pg.Vector(self.regionManager().parameterCount(), mv)
else:
sm = self.regionManager().createStartModel()
return sm
def value(self, v):
self.info['type'] = str(type(v).__name__)
# if len(self.info['type']) != 1:
# pg.error('only single return caches supported for now.')
# return
self.info['file'] = self._name
self.updateCacheInfo()
if self.info['type'] == 'Mesh':
pg.info('Save Mesh binary v2')
v.saveBinaryV2(self._name)
elif self.info['type'] == 'RVector':
pg.info('Save RVector binary')
v.save(self._name, format=pg.core.Binary)
elif self.info['type'] == 'ndarray':
pg.info('Save ndarray')
np.save(self._name, v, allow_pickle=True)
else:
np.save(self._name, v, allow_pickle=True)
# pg.warn('ascii save of type', self.info['type'], 'might by dangerous')
# v.save(self._name)
self._value = v
pg.info('Cache stored:', self._name)
def createRefinedFwdMesh(self, mesh):
"""Refine the current mesh for higher accuracy.
This is called automatic when accessing self.mesh() so it ensures any
effect of changing region properties (background, single).
"""
if self._refineP2 == True:
pg.info("Creating refined mesh (P2) to solve forward task.")
m = mesh.createP2()
else:
pg.info("Creating refined mesh (H2) to solve forward task.")
m = mesh.createH2()
pg.verbose(m)
return m
def createConstraints(self):
# First order smoothness matrix
self._Ctmp = pg.RSparseMapMatrix()
if self.corr_l is None:
pg.info("Using smoothing with zWeight = %.2f." % self.zWeight)
rm = self.RST.fop.regionManager()
rm.fillConstraints(self._Ctmp)
# Set zWeight
rm.setZWeight(self.zWeight)
self.cWeight = pg.RVector()
rm.fillConstraintsWeight(self.cWeight)
self._CW = pg.LMultRMatrix(self._Ctmp, self.cWeight)
else:
pg.info("Using geostatistical constraints with " +
str(self.corr_l))
# Geostatistical constraints by Jordi et al., GJI, 2018
CM = pg.utils.geostatistics.covarianceMatrix(self.mesh,
I=self.corr_l)
self._Ctmp = pg.matrix.Cm05Matrix(CM)
self._CW = self._Ctmp
# Putting together in block matrix
self._C = pg.RBlockMatrix()
cid = self._C.addMatrix(self._CW)
self._C.addMatrixEntry(cid, 0, 0)
self._C.addMatrixEntry(cid, self._Ctmp.rows(), self.cellCount)
self._C.addMatrixEntry(cid, self._Ctmp.rows() * 2, self.cellCount * 2)
self._C.addMatrixEntry(cid, self._Ctmp.rows() * 3, self.cellCount * 3)
self.setConstraints(self._C)
# Identity matrix for interparameter regularization
self._I = pg.IdentityMatrix(self.cellCount)
self._G = pg.RBlockMatrix()
iid = self._G.addMatrix(self._I)
self._G.addMatrixEntry(iid, 0, 0)
self._G.addMatrixEntry(iid, 0, self.cellCount)
self._G.addMatrixEntry(iid, 0, self.cellCount * 2)
self._G.addMatrixEntry(iid, 0, self.cellCount * 3)
self.fix_val_matrices = {}
# Optionally fix phases to starting model globally or in selected cells
phases = ["water", "ice", "air", "rock matrix"]
for i, phase in enumerate(
[self.fix_water, self.fix_ice, self.fix_air, self.fix_poro]):
name = phases[i]
vec = pg.RVector(self.cellCount)
if phase is True:
pg.info("Fixing %s content globally." % name)
vec += 1.0
elif hasattr(phase, "__len__"):
pg.info("Fixing %s content at selected cells." % name)
phase = np.asarray(phase, dtype="int")
vec[phase] = 1.0
self.fix_val_matrices[name] = pg.matrix.DiagonalMatrix(vec)
self._G.addMatrix(self.fix_val_matrices[name], self._G.rows(),
self.cellCount * i)
def load(fileName, verbose=False, **kwargs):
"""Shortcut to load ERT data.
Import Data and try to assume the file format.
Additionally to unified data format we support the wide-spread res2dinv
format as well as ASCII column files generated by the processing software
of various instruments (ABEM LS, Syscal Pro, Resecs, ?)
If this fails, install pybert and use its auto importer pybert.importData.
Parameters
----------
fileName: str
Returns
-------
data: pg.DataContainer
"""
data = pg.load(fileName)
if isinstance(data, pg.DataContainerERT):
return data
try:
pg.info("could not read unified data format for ERT ... try res2dinv")
data = importRes2dInv(fileName)
return data
except:
pg.info("could not read res2dinv ... try Ascii columns")
try:
data = importAsciiColumns(fileName)
return data
except Exception as e:
pg.info("Failed importing Ascii column file. Consider using pybert.")
pg.info(e)
if verbose:
pg.info("Try to import using pybert .. if available")
pb = pg.optImport('pybert')
data = pb.loadData(fileName)
if isinstance(data, pg.DataContainerERT):
return data
pg.critical("Can't import ERT data file.", fileName)
def createMeshPatches(ax, mesh, verbose=True):
"""Utility function to create 2d mesh patches within a given ax."""
if not mesh:
pg.error("drawMeshBoundaries(ax, mesh): invalid mesh:", mesh)
return
if mesh.nodeCount() < 2:
pg.error("drawMeshBoundaries(ax, mesh): to few nodes:", mesh)
return
pg.tic()
polys = [_createCellPolygon(c) for c in mesh.cells()]
patches = mpl.collections.PolyCollection(polys, picker=True)
if verbose:
pg.info("Creation of mesh patches took = ", pg.toc())
return patches
def startModel(self, model):
"""
model: [float] | float
Model used as starting model.
Float value is used as constant model.
"""
if model is None:
self._startModel = None
elif isinstance(model, float) or isinstance(model, int):
self._startModel = np.ones(self.parameterCount) * float(model)
pg.info("Startmodel set from given value.", float(model))
elif hasattr(model, '__iter__'):
if len(model) == self.parameterCount:
pg.info("Startmodel set from given array.", model)
self._startModel = model
else:
pg.error("Startmodel size invalid {0} != {0}.".format(
len(model), self.parameterCount))
def checkData(self, data):
"""Return data from container.
THINKABOUT: Data will be changed, or should the manager keeps an own copy?
"""
if isinstance(data, pg.DataContainer):
if not data.allNonZero('k'):
pg.warn("Data file contains no geometric factors (token='k').")
data['k'] = createGeometricFactors(data, verbose=True)
if self.fop.complex():
if not data.haveData('rhoa'):
pg.critical('Datacontainer have no "rhoa" values.')
if not data.haveData('ip'):
pg.critical('Datacontainer have no "ip" values.')
#pg.warn('check sign of phases')
rhoa = data['rhoa']
phia = -data['ip'] / 1000 # 'ip' is defined for neg mrad.
# we should think about some 'phia' in rad
return pg.utils.squeezeComplex(pg.utils.toComplex(rhoa, phia))
else:
if not data.haveData('rhoa'):
if data.allNonZero('r'):
pg.info("Creating apparent resistivies from "
"impedences rhoa = r * k")
data['rhoa'] = data['r'] * data['k']
elif data.allNonZero('u') and data.allNonZero('i'):
pg.info("Creating apparent resistivies from "
"voltage and currrent rhoa = u/i * k")
data['rhoa'] = data['u'] / data['i'] * data['k']
else:
pg.critical(
"Datacontainer have neither: "
"apparent resistivies 'rhoa', "
"or impedances 'r', "
"or voltage 'u' together with current 'i' values.")
return data['rhoa']
return data
def createStartModel(self, dataVals):
"""Create a starting model from data values (gradient or constant)."""
sm = None
if self._useGradient is not None:
[vTop, vBot] = self._useGradient # something strange here!!!
pg.info('Create gradient starting model. {0}: {1}'.format(
vTop, vBot))
sm = createGradientModel2D(self.data, self.paraDomain, vTop, vBot)
else:
dists = shotReceiverDistances(self.data, full=True)
aSlow = 1. / (dists / dataVals)
# pg._r(self.regionManager().parameterCount())
sm = pg.Vector(self.regionManager().parameterCount(),
pg.math.median(aSlow))
pg.info('Create constant starting model:', sm[0])
return sm
def createMeshPatches(ax, mesh, verbose=True, rasterized=False):
"""Utility function to create 2d mesh patches within a given ax."""
if not mesh:
pg.error("drawMeshBoundaries(ax, mesh): invalid mesh:", mesh)
return
if mesh.nodeCount() < 2:
pg.error("drawMeshBoundaries(ax, mesh): to few nodes:", mesh)
return
pg.tic()
polys = [_createCellPolygon(c) for c in mesh.cells()]
patches = mpl.collections.PolyCollection(polys, picker=True,
rasterized=rasterized)
if verbose:
pg.info("Creation of mesh patches took = ", pg.toc())
return patches
def createStartModel(self, dataVals):
"""Create Starting model for ERT inversion."""
if self.complex():
dataC = pg.utils.toComplex(dataVals)
nModel = self.regionManager().parameterCount() // 2
smRe = np.ones(nModel) * np.median(np.median(dataC.real))
smIm = np.ones(nModel) * np.median(np.median(dataC.imag))
if min(smIm) < 0:
# we want positive phase model
sm = smRe - 1j * smIm
pg.info("Model imaginary part being flipped to positive.")
self._conjImag = True
else:
sm = smRe + 1j * smIm
return pg.utils.squeezeComplex(sm) # complex impedance
else:
return super(ERTModelling, self).createStartModel(dataVals)
def startModel(self):
""" Gives the current default starting model.
Returns the current default starting model or
call fop.createStartmodel() if non is defined.
"""
if self._startModel is None:
sm = self.fop.regionManager().createStartModel()
if len(sm) > 0 and max(abs(np.atleast_1d(sm))) > 0.0:
self._startModel = sm
pg.info("Created startmodel from region infos:", sm)
else:
pg.verbose("No region infos for startmodel")
if self._startModel is None:
sm = self.fop.createStartModel(self.dataVals)
pg.info("Created startmodel from forward operator:", sm)
self._startModel = sm
return self._startModel
def createFwdMesh_(self):
""""""
pg.info("Creating forward mesh from region infos.")
m = pg.Mesh(self.regionManager().mesh())
regionIds = self.regionManager().regionIdxs()
for iId in regionIds:
pg.verbose("\tRegion: {0}, Parameter: {1}, PD: {2},"
" Single: {3}, Background: {4}, Fixed: {5}"
.format(iId,
self.regionManager().region(iId).parameterCount(),
self.regionManager().region(iId).isInParaDomain(),
self.regionManager().region(iId).isSingle(),
self.regionManager().region(iId).isBackground(),
self.regionManager().region(iId).fixValue(),
))
m = self.createRefinedFwdMesh(m)
self.setMeshPost(m)
self._regionChanged = False
super(Modelling, self).setMesh(m, ignoreRegionManager=True)
def convertStartModel(self, model):
"""Convert scalar or array into startmodel with valid range or
self.fop.parameterCount, if possible.
Attributes
----------
model: float|int|array|None
"""
if model is None:
return None
elif isinstance(model, float) or isinstance(model, int):
pg.info("Homogeneous starting model set to:", float(model))
return np.full(self.fop.parameterCount, float(model))
elif hasattr(model, '__iter__'):
if len(model) == self.fop.parameterCount:
pg.info("Starting model set from given array.", model)
return model
else:
pg.error("Starting model size invalid {0} != {1}.".
format(len(model), self.fop.parameterCount))
return None
def createConstraints(self):
""""""
# just ensure there is valid mesh
self.mesh()
foundGeoStat = False
for reg, props in self.regionProperties().items():
if props['correlationLengths'] is not None or \
props['dip'] is not None or \
props['strike'] is not None:
I = props.get('correlationLengths') or 5
dip = props.get('dip') or 0
strike = props.get('strike') or 0
pg.info('Creating GeostatisticConstraintsMatrix for region' +
f' {reg} with: I={I}, dip={dip}, strike={strike}')
if foundGeoStat == True:
pg.critical(
'Only one global GeostatisticConstraintsMatrix possible at the moment.'
)
### we need to keep a copy of C until refcounting in the core works
self._C = pg.matrix.GeostatisticConstraintsMatrix(
mesh=self.paraDomain,
I=I,
dip=dip,
strike=strike,
)
foundGeoStat = True
self.setConstraints(self._C)
if foundGeoStat == False:
super().createConstraints()
return self.constraints()
def invert_pygimli_ERT(inputfileERT, sensors, mesh, date):
#%% INVERT ERT data
invpath = main + '/invdir/pygimli/' + date + '/'
if not os.path.exists(invpath):
os.makedirs(invpath)
dataERT = pb.load(main + './raw_data/' + inputfileERT)
dataERT.setSensorPositions(sensors)
ert = pg.physics.ERTManager(
dataERT) # sr=False, verbose=True, debug=False)
k = createGeometricFactors(dataERT)
dataERT.set("k", k)
dataERT.set('r', dataERT('u') / dataERT('i'))
dataERT.set('rhoa', dataERT('r') * dataERT('k'))
dataERT['err'] = ert.estimateError(dataERT,
absoluteError=0.001,
relativeError=0.1)
dataERT.markInvalid(dataERT("rhoa") < 0)
dataERT.removeInvalid()
dataERT.save(invpath + 'dataERT.data')
# ert.setMesh(mesh3d) # important to do it explicitly as it avoids rebuilding region manager etc.
# C = pg.matrix.GeostatisticConstraintsMatrix(mesh=ert.fop.paraDomain, I=[25, 5], dip=-25)
# ert.fop.setConstraints(C)
# Set refernce nodes in corners (necessary for closed geometries)
lower_left_node = mesh.findNearestNode([mesh.xmin(), mesh.ymin(), 0.0])
mesh.node(lower_left_node).setMarker(-1000) #MARKER_NODE_CALIBRATION
lower_right_node = mesh.findNearestNode([mesh.xmax(), mesh.ymin(), 0.0])
mesh.node(lower_right_node).setMarker(
-999) #MARKER_NODE_REFERENCEELECTRODE
model = ert.invert(mesh=mesh, lam=20, verbose=True)
pg.info('Inversion stopped with chi² = {0:.3}'.format(ert.fw.chi2()))
return model, dataERT
def getExampleFile(path, load=False, verbose=False):
"""Download and return a filename to the example repository.
TODO:
checksum or hash test for the content.
Parameters
----------
path: str
Path to the remote repo
load: bool [False]
Try to load the file and return the relating object.
Returns
-------
filename: str
Filename to the data content
data: obj
content of the path if load is True
"""
url = exampleDataRepository + path
fileName = os.path.join(tempfile.gettempdir(), gimliExampleDataPath, path)
if not os.path.exists(fileName):
if verbose:
pg.info("Getting:", fileName)
os.makedirs(os.path.dirname(fileName), exist_ok=True)
tmp = urlretrieve(url, fileName)
else:
if verbose:
pg.info("File already exists:", fileName)
if load:
print(fileName)
d = pg.load(fileName)
return pg.load(fileName)
return fileName
def showDataContainerAsMatrix(data, x=None, y=None, v=None, **kwargs):
"""Plot data container as matrix (cross-plot).
for each x, y and v token strings or vectors should be given
"""
mul = kwargs.pop('mul', 10**int(np.ceil(np.log10(data.sensorCount()))))
plus = kwargs.pop('plus', 1) # add 1 to count
verbose = kwargs.pop('verbose', False)
if hasattr(x, '__iter__') and isinstance(x[0], str):
num = np.zeros(data.size())
for token in x:
num *= mul
num += data(token) + plus
x = num.copy()
# kwargs.setdefault('xmap', {n: i for i, n in enumerate(np.unique(x))})
# xmap = {}
# for i, n in enumerate(np.unique(x)):
# st = ''
# while n > 0:
# st = str(n % mul) + '-' + st
# n = n // mul
# xmap[]
elif isinstance(x, str):
x = data(x)
if hasattr(y, '__iter__') and isinstance(y[0], str):
num = np.zeros(data.size())
for token in y:
num *= mul
num += data(token) + plus
y = num.copy()
# kwargs.setdefault('ymap', {n: i for i, n in enumerate(np.unique(y))})
elif isinstance(y, str):
y = data(y)
if isinstance(v, str):
v = data(v)
if verbose:
pg.info("x vector length: {:d}".format(len(x)))
pg.info("y vector length: {:d}".format(len(y)))
pg.info("v vector length: {:d}".format(len(v)))
if x is None or y is None or v is None:
raise Exception("Vectors or strings must be given")
if len(x) != len(y) or len(x) != len(v):
raise Exception("lengths x/y/v not matching: {:d}!={:d}!={:d}".format(
len(x), len(y), len(v)))
return showVecMatrix(x, y, v, **kwargs)
def load(fileName, verbose=False, **kwargs):
"""Shortcut to load SIP spectral data.
Import Data and try to assume the file format.
Parameters
----------
fileName: str
Returns
-------
freqs, amp, phi : np.array
Frequencies, amplitudes and phases phi in neg. radiant
"""
firstLine = None
with codecs.open(fileName, 'r', encoding='iso-8859-15',
errors='replace') as fi:
firstLine = fi.readline()
f, amp, phi = None, None, None
fnLow = fileName.lower()
if 'SIP Fuchs III' in firstLine:
if verbose:
pg.info("Reading SIP Fuchs III file")
f, amp, phi, header = readFuchs3File(fileName,
verbose=verbose,
**kwargs)
phi *= -np.pi / 180.
# print(header) # not used?
elif 'SIP-Quad' in firstLine:
if verbose:
pg.info("Reading SIP Quad file")
f, amp, phi, header = readFuchs3File(fileName,
verbose=verbose,
**kwargs)
phi *= -np.pi / 180.
elif 'SIP-Fuchs' in firstLine:
if verbose:
pg.info("Reading SIP Fuchs file")
f, amp, phi, drhoa, dphi = readRadicSIPFuchs(fileName,
verbose=verbose,
**kwargs)
phi *= -np.pi / 180.
elif fnLow.endswith('.txt') or fnLow.endswith('.csv'):
f, amp, phi = readTXTSpectrum(filename)
amp *= 1.0 # scale it with k if available
else:
raise Exception("Don't know how to read data.")
return f, amp, phi
def loadData(self, filename, **kwargs):
"""Import spectral data.
Import Data and try to assume the file format.
"""
verbose = kwargs.pop('verbose', self._verbose)
with codecs.open(filename, 'r', encoding='iso-8859-15',
errors='replace') as f:
firstLine = f.readline()
f.close()
fnLow = filename.lower()
self.basename = filename[:-4]
if 'SIP Fuchs III' in firstLine:
if verbose:
pg.info("Reading SIP Fuchs III file")
self.f, self.amp, self.phi, header = readFuchs3File(
filename, verbose=verbose, **kwargs)
self.phi *= -np.pi/180.
# print(header) # not used?
elif 'SIP-Quad' in firstLine:
if verbose:
pg.info("Reading SIP Quad file")
self.f, self.amp, self.phi, header = readFuchs3File(
filename, verbose=verbose, quad=True, **kwargs)
self.phi *= -np.pi/180.
elif 'SIP-Fuchs' in firstLine:
if verbose:
pg.info("Reading SIP Fuchs file")
self.f, self.amp, self.phi, drhoa, dphi = readRadicSIPFuchs(
filename, verbose=verbose, quad='SIP-Quad' in firstLine,
**kwargs)
self.phi *= -np.pi/180.
elif fnLow.endswith('.txt') or fnLow.endswith('.csv'):
self.f, self.amp, self.phi = readTXTSpectrum(filename)
self.amp *= self.k
else:
raise Exception("Don't know how to read data.")
return self.f, self.amp, self.phi
def showRayPaths(self, model=None, ax=None, **kwargs):
"""Show ray paths for `model` or last model for which the Jacobian was
calculated.
Parameters
----------
model : array
Velocity model for which to calculate and visualize ray paths (the
default is model for last Jacobian calculation in self.velocity).
ax : matplotlib.axes
Axes for the plot (the default is None).
**kwargs : type
Additional arguments passed to LineCollection (alpha, linewidths,
color, linestyles).
Returns
-------
ax : matplotlib.axes object
cb : matplotlib.colorbar object (only if model is provided)
Examples
--------
>>> # No reason to import matplotlib
>>> import pygimli as pg
>>> from pygimli.physics import Refraction
>>> from pygimli.physics.traveltime import createRAData
>>>
>>> x, y = 8, 6
>>> mesh = pg.createGrid(x, y)
>>> data = createRAData([(0,0)] + [(x, i) for i in range(y)], shotdistance=y+1)
>>> data.set("t", pg.RVector(data.size(), 1.0))
>>> rst = Refraction()
>>> rst.setDataContainer(data)
Data: Sensors: 7 data: 6
>>> rst.setMesh(mesh, 5)
>>> ax, cb = rst.showRayPaths()
"""
cbar = None
if model is None and self.velocity is None:
pg.info("No previous inversion result found and no model given.",
"Using homogeneous slowness model.")
self.velocity = pg.RVector(self.mesh.cellCount(), 1.0)
self.fop.createJacobian(1./self.velocity)
if model is not None:
if self.velocity is not None:
if not np.allclose(model, self.velocity):
self.fop.createJacobian(1/model)
ax, cbar = self.showResult(ax=ax, val=model)
_ = kwargs.setdefault("color", "w")
_ = kwargs.setdefault("alpha", 0.5)
_ = kwargs.setdefault("linewidths", 0.8)
else:
ax = self.showMesh(ax=ax)
# Due to different numbering scheme of way matrix
_, shots = np.unique(self.dataContainer("s"), return_inverse=True)
_, receivers = np.unique(self.dataContainer("g"), return_inverse=True)
# Collecting way segments for all shot/receiver combinations
segs = []
for s, g in zip(shots, receivers):
wi = self.fop.way(s, g)
points = self.fop.mesh().positions(withSecNodes=True)[wi]
segs.append(np.column_stack((pg.x(points), pg.y(points))))
line_segments = LineCollection(segs, **kwargs)
ax.add_collection(line_segments)
return ax, cbar
def show(mesh=None, data=None, **kwargs):
"""Mesh and model visualization.
Syntactic sugar to show a mesh with data. Forwards to
:py:mod:`pygimli.viewer.showMesh` or
:py:mod:`pygimli.viewer.mayaview.showMesh3D` to show most of the typical 2D
and 3D content. See tutorials and examples for usage hints. An empty show
call creates an empty ax window.
Parameters
----------
mesh : :gimliapi:`GIMLI::Mesh` or list of meshes
2D or 3D GIMLi mesh
**kwargs :
* fitView : bool [True]
Scale x and y limits to match the view.
* ax : axe [None]
Matplotlib axes object. Create a new if necessary.
* Will be forwarded to the appropriate show functions.
Returns
-------
Return the results from the showMesh* functions.
See Also
--------
showMesh
"""
if "axes" in kwargs:
print("Deprecation Warning: Please use keyword `ax` instead of `axes`")
kwargs['ax'] = kwargs.pop('axes', None)
if isinstance(mesh, list):
ax = kwargs.pop('ax', None)
fitView = kwargs.pop('fitView', True)
ax, cbar = show(mesh[0], data, hold=1, ax=ax, fitView=fitView, **kwargs)
xmin = mesh[0].xmin()
xmax = mesh[0].xmax()
ymin = mesh[0].ymin()
ymax = mesh[0].ymax()
for m in mesh[1:]:
ax, cbar = show(m, data, ax=ax, hold=1, fitView=False, **kwargs)
xmin = min(xmin, m.xmin())
xmax = max(xmax, m.xmax())
ymin = min(ymin, m.ymin())
ymax = max(ymax, m.ymax())
# ax.relim()
# ax.autoscale_view(tight=True)
if fitView is not False:
ax.set_xlim([xmin, xmax])
ax.set_ylim([ymin, ymax])
# print(ax.get_data_interval())
return ax, cbar
if isinstance(mesh, pg.Mesh):
if mesh.dim() == 2:
if pg.zero(pg.y(mesh)):
pg.info("swap z<->y coordinates for visualization.")
meshSwap = pg.Mesh(mesh)
for n in meshSwap.nodes():
n.pos()[1] = n.pos()[2]
return showMesh(meshSwap, data, **kwargs)
return showMesh(mesh, data, **kwargs)
elif mesh.dim() == 3:
from .mayaview import showMesh3D
return showMesh3D(mesh, data, **kwargs)
else:
pg.error("ERROR: Mesh not valid.", mesh)
ax = kwargs.pop('ax', None)
if ax is None:
ax = plt.subplots()[1]
return ax, None
def drawModel(ax, mesh, data=None, logScale=True, cMin=None, cMax=None,
xlabel=None, ylabel=None, verbose=False,
tri=False, rasterized=False, **kwargs):
"""Draw a 2d mesh and color the cell by the data.
Parameters
----------
mesh : :gimliapi:`GIMLI::Mesh`
The plotted mesh to browse through.
ax : mpl axis instance, optional
Axis instance where the mesh is plotted (default is current axis).
data : array, optional
Data to draw. Should either equal numbers of cells or nodes of the
corresponding `mesh`.
tri : boolean, optional
use MPL tripcolor (experimental)
rasterized : boolean, optional
Rasterize mesh patches to reduce file size and avoid zooming artifacts
in some PDF viewers.
**kwargs : Additional keyword arguments
Will be forwarded to the draw functions and matplotlib methods,
respectively.
Returns
-------
gci : matplotlib graphics object
Examples
--------
>>> import numpy as np
>>> import matplotlib.pyplot as plt
>>> import pygimli as pg
>>> from pygimli.mplviewer import drawModel
>>> n = np.linspace(0, -2, 11)
>>> mesh = pg.createGrid(x=n, y=n)
>>> mx = pg.x(mesh.cellCenter())
>>> my = pg.y(mesh.cellCenter())
>>> data = np.cos(1.5 * mx) * np.sin(1.5 * my)
>>> fig, ax = plt.subplots()
>>> drawModel(ax, mesh, data)
<matplotlib.collections.PolyCollection object at ...>
"""
# deprecated .. remove me
if 'cMap' in kwargs or 'cmap' in kwargs:
pg.warn('cMap|cmap argument is deprecated for draw functions. ' +
'Please use show or customize a colorbar.')
# deprecated .. remove me
if mesh.nodeCount() == 0:
pg.error("drawModel: The mesh is empty.", mesh)
if tri:
gci = drawMPLTri(ax, mesh, data,
cMin=cMin, cMax=cMax, logScale=logScale,
**kwargs)
else:
gci = pg.mplviewer.createMeshPatches(ax, mesh, verbose=verbose,
rasterized=rasterized)
ax.add_collection(gci)
if data is None:
data = pg.RVector(mesh.cellCount())
if len(data) != mesh.cellCount():
print(data, mesh)
pg.info("drawModel have wrong data length .. " +
" indexing data from cellMarkers()")
viewdata = data[mesh.cellMarkers()]
else:
viewdata = data
if min(data) <= 0:
logScale = False
pg.mplviewer.setMappableData(gci, viewdata, cMin=cMin, cMax=cMax,
logScale=logScale)
gci.set_antialiased(True)
gci.set_linewidths(0.1)
gci.set_edgecolors("face")
if xlabel is not None:
ax.set_xlabel(xlabel)
if ylabel is not None:
ax.set_ylabel(ylabel)
if kwargs.pop('fitView', True):
ax.set_xlim(mesh.xmin(), mesh.xmax())
ax.set_ylim(mesh.ymin(), mesh.ymax())
ax.set_aspect('equal')
updateAxes_(ax)
return gci
def readFuchs3File(resfile, k=1.0, verbose=False):
"""Read Fuchs III (SIP spectrum) data file.
Parameters
----------
k : float
Overwrite internal geometric factor from device.
"""
activeBlock = ''
header = {}
LINE = []
dataAct = False
with codecs.open(resfile, 'r', encoding='iso-8859-15', errors='replace') as f:
for line in f:
line = line.replace('\r\n', '\n') # correct for carriage return
if dataAct:
LINE.append(line)
if len(line) < 2:
f, amp, phi, kIn = [], [], [], []
for li in LINE:
sline = li.split()
if len(sline) > 12:
fi = float(sline[11])
if np.isfinite(fi):
f.append(fi)
amp.append(float(sline[12]))
phi.append(float(sline[13]))
kIn.append(float(sline[9]))
if k != 1.0 and verbose is True:
pg.info("Geometric value changed to:", k)
return np.array(f), np.array(amp)/np.array(kIn) * k, \
np.array(phi), header
elif len(line):
if line.rfind('Current') >= 0:
if dataAct:
break
else:
dataAct = True
if line[0] == '[':
token = line[1:line.rfind(']')].replace(' ', '_')
if token[:3] == 'End':
header[activeBlock] = np.array(header[activeBlock])
activeBlock = ''
elif token[:5] == 'Begin':
activeBlock = token[6:]
header[activeBlock] = []
else:
value = line[line.rfind(']') + 1:]
try: # direct line information
if '.' in value:
num = float(value)
else:
num = int(value)
header[token] = num
except BaseException as e:
# maybe beginning or end of a block
#print(e)
pass
else:
if activeBlock:
nums = np.array(line.split(), dtype=float)
header[activeBlock].append(nums)
import logging
import pygimli as pg
#log = logging.getLogger('pyGIMLi')
#logging.basicConfig(level=logging.DEBUG,
#format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
#datefmt='%m/%d/%Y %H:%M:%S',
##filename='example.log'
#)
pg.version()
# test pygimli log
pg.info("Start numeric log test." + str(pg.log(pg.RVector(1, 1.))))
pg.warn("Start warning test.")
def testTraceback1():
def testTraceback2():
pg.error("Start error test.: int", 1, " vec", pg.RVector(2))
testTraceback2()
testTraceback1()
#pg.critical("Start critical test.")
pg.debug("debug 0")
pg.setDebug(1)
pg.debug("debug ON")
pg.setThreadCount(2)
