def __init__(self, db_name=None, db_host=None):
self._logging = csamtpylog.get_csamtpy_logger(self.__class__.__name__)
self.db_host = db_host
self.db_name = db_name
if self.db_name is not None:
self.connect_DB()
def __init__(self, agso_file=None, **kwargs):
self._logging = csamtpylog.get_csamtpy_logger(self.__class__.__name__)
self._agso_fn = agso_file
for key in list(kwargs.keys()):
self.__setattr__(key, kwargs[key])
if self.agso_fn is not None:
self._set_geo_formation()
def __init__(self, default_kwargs, **kws):
self._logging = csamtpylog.get_csamtpy_logger(self.__class__.__name__)
self.default_kwargs = default_kwargs
self.input_kwargs = kws.pop('input_kwargs', {})
if self.input_kwargs is None: self.input_kwargs = {}
self.sanitize_kwargs = {}
if self.default_kwargs is not None:
self._console_scripts()
class gdal_data_check(object):
_has_checked = False
_gdal_data_found = False
_logger = csamtpylog.get_csamtpy_logger(__name__)
def __init__(self, func, raise_error=False):
"""
this decorator should only be used for the function that requres gdal and gdal-data
to function correctly.
the decorator will check if the GDAL_DATA is set and the path in GDAL_DATA is exist.
If GDAL_DATA is not set, then try to use external program "gdal-config --datadir" to
findout where the data files are installed.
If failed to find the data file, then ImportError will be raised.
:param func: function to be decorated
"""
self._func = func
if not self._has_checked:
self._gdal_data_found = self._check_gdal_data()
self._has_checked = True
if not self._gdal_data_found:
if (raise_error):
raise ImportError("GDAL is NOT installed correctly")
else:
print("Ignore GDAL as it is not working. Will use pyproj")
def __call__(self, *args, **kwargs): # pragma: no cover
return self._func(*args, **kwargs)
def _check_gdal_data(self):
if 'GDAL_DATA' not in os.environ:
# gdal data not defined, try to define
from subprocess import Popen, PIPE
self._logger.warning(
"GDAL_DATA environment variable is not set Please see https://trac.osgeo.org/gdal/wiki/FAQInstallationAndBuilding#HowtosetGDAL_DATAvariable "
)
try:
# try to find out gdal_data path using gdal-config
self._logger.info("Trying to find gdal-data path ...")
process = Popen(['gdal-config', '--datadir'], stdout=PIPE)
(output, err) = process.communicate()
exit_code = process.wait()
output = output.strip()
if exit_code == 0 and os.path.exists(output):
os.environ['GDAL_DATA'] = output
self._logger.info(
"Found gdal-data path: {}".format(output))
return True
else:
self._logger.error(
"\tCannot find gdal-data path. Please find the gdal-data path of your installation and set it to "
"\"GDAL_DATA\" environment variable. Please see "
"https://trac.osgeo.org/gdal/wiki/FAQInstallationAndBuilding#HowtosetGDAL_DATAvariable for "
"more information.")
return False
except Exception:
return False
else:
if os.path.exists(os.environ['GDAL_DATA']):
self._logger.info("GDAL_DATA is set to: {}".format(
os.environ['GDAL_DATA']))
try:
from osgeo import osr
from osgeo.ogr import OGRERR_NONE
except:
self._logger.error(
"Failed to load module osgeo; looks like GDAL is NOT working"
)
# print ("Failed to load module osgeo !!! ")
return False
# end try
return True
else:
self._logger.error(
"GDAL_DATA is set to: {}, but the path does not exist.".
format(os.environ['GDAL_DATA']))
return False
class redirect_cls_or_func(object):
"""
Description:
used to redirected functions or classes. Deprecated functions or class
can call others use functions or classes.
Usage:
.. todo:: use new function or class to replace old function method or class
with multiple parameters.
Author: @Daniel03
Date: 18/10/2020
"""
_logger = csamtpylog.get_csamtpy_logger(__name__)
def __init__(self, *args, **kwargs):
"""
self.new_func_or_cls is just a message of deprecating
warning . It could be a name of new function to let user
tracking its code everytime he needs .
"""
self._reason = [
func_or_reason for func_or_reason in args
if type(func_or_reason) == str
][0]
if self._reason is None:
raise TypeError(" Redirected reason must be supplied")
self._new_func_or_cls = [func_or_reason for func_or_reason in \
args if type(func_or_reason)!=str][0]
if self._new_func_or_cls is None:
raise Exception(
" At least one argument must be a func_method_or class."
"\but it's %s." % type(self._new_func_or_cls))
self._logger.warn(
"\t first input argument argument must be a func_method_or class."
"\but it's %s." % type(self._new_func_or_cls))
def __call__(self, cls_or_func): #pragma :no cover
if inspect.isfunction(self._new_func_or_cls):
if hasattr(self._new_func_or_cls, 'func_code'):
_code = self._new_func_or_cls.__code__
lineno = _code.co_firstlineno + 1
else:
# do it once the method is decorated method like staticmethods
try:
_code = self._new_func_or_cls.__code__
except:
pass
lineno = self._new_func_or_cls.__code__.co_firstlineno
fmt="redirected decorated func/methods .<{reason}> "\
"see line {lineno}."
elif inspect.isclass(self._new_func_or_cls):
_code = self._new_func_or_cls.__module__
# filename=os.path.basename(_code.co_filename)
lineno = 1
fmt="redirected decorated class :<{reason}> "\
"see line {lineno}."
else:
# lineno=cls_or_func.__code__.co_firstlineno
lineno = inspect.getframeinfo(inspect.currentframe())[1]
fmt="redirected decorated method :<{reason}> "\
"see line {lineno}."
msg = fmt.format(reason=self._reason, lineno=lineno)
# print(msg)
self._logger.info(msg)
#count variables : func.__code__.co_argscounts
#find variables in function : func.__code__.co_varnames
@functools.wraps(cls_or_func)
def new_func(*args, **kwargs):
return cls_or_func(*args, **kwargs)
return self._new_func_or_cls
# -*- coding: utf-8 -*-
"""
Created on Tue Apr 6 12:58:45 2021
.Console scripts.
write pseudo-wrapper to interact with python shell environnement.
.. note:: Try to use dict_kwargs to collect input arguments values,
as well as defaults values .
@author: @Daniel03
"""
from pycsamt.utils._csamtpylog import csamtpylog
_logger = csamtpylog.get_csamtpy_logger(__name__)
class wrap_cscripts(object):
"""
get input arguments from user , sanitize it and convoy towards required
modules
:param default_kwargs: dictionnary of kwargs arguments with the defaults
values
:type default_kwargs: dict
.. note:: to sanitize input arguments , provided `*` for arguments which
must be converte into `integer` and double `**` for those who should be
convert into `float`. If conversion is not possible, will
take the default arguments
:Example :
class _sensitive:
"""
.. note :: `sensitive class` . Please keep carefully the indices like
it's ranged. Better Ways to avoid redundancy in code during the program
design. Core of parser of each files except files from :ref:`module-geodrill`.
Aims are:
1. to check file . it is was the right file provided or not
2. to write file . to Zonge Avg_format or J-Format or EDI -format and else.
3. to compute value . Indice are used for computation , set and get specific value.
"""
_logger = csamtpylog.get_csamtpy_logger(__name__)
_j = [
'>AZIMUTH', '>LATITUDE', '>LONGITUDE', '>ELEVATION', 'RXX', 'RXY',
'RYX', 'RYY', 'RTE', 'RTM', 'RAV', 'RDE', 'RZX', 'RZY', 'SXX', 'SXY',
'SYX', 'SYY', 'STE', 'STM', 'SAV', 'SDE', 'SZX', 'SZY', 'ZXX', 'ZXY',
'ZYX', 'ZYY', 'ZTE', 'ZTM', 'ZAV', 'ZDE', 'ZZX', 'ZZY', 'QXX', 'QXY',
'QYX', 'QYY', 'QTE', 'QTM', 'QAV', 'QDE', 'QZX', 'QZY', 'CXX', 'CXY',
'CYX', 'CYY', 'CTE', 'CTM', 'CAV', 'CDE', 'CZX', 'CZY', 'TXX', 'TXY',
'TYX', 'TYY', 'TTE', 'TTM', 'TAV', 'TDE', 'TZX', 'TZY', 'ZXX', 'ZXY',
'ZYX'
]
_avg = [
'skp', 'Station', ' Freq', 'Comp', ' Amps', 'Emag', 'Ephz', 'Hmag',
'Hphz', 'Resistivity', 'Phase', '%Emag', 'sEphz', '%Hmag', 'sHphz',
'%Rho', 'sPhz', 'Tx.Amp', 'E.mag', 'E.phz', 'B.mag', 'B.phz', 'Z.mag',
'Z.phz', 'ARes.mag', 'SRes', 'E.wgt', 'B.wgt', 'E.%err', 'E.perr',
'B.%err', 'B.perr', 'Z.%err', 'Z.perr', 'ARes.%err'
]
_edi = [
'>HEAD',
'>INFO', #Head=Infos-Freuency-Rhorot,Zrot and end blocks
'>=DEFINEMEAS',
'>=MTSECT',
'>FREQ ORDER=INC', #Definitions Measurments Blocks
'>ZROT',
'>RHOROT',
'>!Comment',
'>FREQ',
'>HMEAS',
'>EMEAS',
'>RHOXY',
'>PHSXY',
'>RHOXY.VAR',
'>PHSXY.VAR',
'>RHOXY.ERR',
'>PHSXY.ERR',
'>RHOXY.FIT',
'>PHSXY.FIT', #Apparents Resistivities and Phase Blocks
'>RHOYX',
'>PHSYX',
'>RHOYX.VAR',
'>PHSYX.VAR',
'>RHOYX.ERR',
'>PHSYX.ERR',
'>RHOYX.FIT',
'>PHSYX.FIT',
'>RHOXX',
'>PHSXX',
'>RHOXX.VAR',
'>PHSXX.VAR',
'>RHOXX.ERR',
'>PHSXX.ERR',
'>RHOXX.FIT',
'>PHSXX.FIT',
'>RHOYY',
'>PHSYY',
'>RHOYY.VAR',
'>PHSYY.VAR',
'>RHOYY.ERR',
'>PHSYY.ERR',
'>RHOYY.FIT',
'>PHSYY.FIT',
'>FRHOXY',
'>FPHSXY',
'>FRHOXY.VAR',
'>FPHSXY.VAR',
'>FRHOXY.ERR',
'>FPHSXY.ERR',
'>FRHOXY.FIT',
'>FPHSXY.FIT',
'>FRHOXX',
'>FPHSXX',
'>FRHOXX.VAR',
'>FPHSXX.VAR',
'>FRHOXX.ERR',
'>FPHSXX.ERR',
'>FRHOXX.FIT',
'>FPHSXX.FIT',
'>TSERIESSECT',
'>TSERIES',
'>=SPECTRASECT',
'>=EMAPSECT',
'>=OTHERSECT', #Time series-Sepctra and EM/OTHERSECT
'>ZXYR',
'>ZXYI',
'>ZXY.VAR',
'>ZXYR.VAR',
'>ZXYI.VAR',
'>ZXY.COV', #Impedance Data Blocks
'>ZYXR',
'>ZYXI',
'>ZYX.VAR',
'>ZYXR.VAR',
'>ZYXI.VAR',
'>ZYX.COV',
'>ZYYR',
'>ZYYI',
'>ZYY.VAR',
'>ZYYR.VAR',
'>ZYYI.VAR',
'>ZYY.COV',
'>ZXXR',
'>ZXXI',
'>ZXXR.VAR',
'>ZXXI.VAR',
'>ZXX.VAR',
'>ZXX.COV',
'>FZXXR',
'>FZXXI',
'>FZXYR',
'>FZXYI',
'>RES1DXX',
'>DEP1DXX',
'>RES1DXY',
'>DEP1DXY', #Continuous 1D inversion
'>RES1DYX',
'>DEP1DYX',
'>RES1DYY',
'>DEP1DYY',
'>FRES1DXX',
'>FDEP1DXX',
'>FRES1DXY',
'>FDEP1DXY',
'>COH',
'>EPREDCOH',
'>HPREDCOH',
'>SIGAMP',
'>SIGNOISE', # Coherency and Signal Data Blocks
'>TIPMAG',
'>TIPPHS',
'TIPMAG.ERR',
'>TIPMAG.FIT',
'>TIPPHS.FIT', #Tipper Data blocks
'>TXR.EXP',
'>TXI.EXP',
'>TXVAR.EXP',
'>TYR.EXP',
'>TYI.EXP',
'>TYVAR.EXP',
'>ZSTRIKE',
'>ZSKEW',
'>ZELLIP',
'>TSTRIKE',
'>TSKEW',
'>TELLIP', #Strike, Skew, and Ellipticity Data Blocks
'>FILWIDTH',
'>FILANGLE',
'>EQUIVLEN',
'>END'
] #Spatial filter blocks
_occam_startup = [
'Format', 'Description', 'Model File', 'Data File', 'Date/Time',
'Iterations to run', 'Target Misfit', 'Roughness Type',
'Diagonal Penalties', 'Stepsize Cut Count', 'Model Limits',
'Model Value Steps', 'Debug Level', 'Iteration', 'Lagrange Value',
'Roughness Value', 'Misfit Value', 'Misfit Reached', 'Param Count'
] # end with file or startup
_occam_datafile = [
'FORMAT', 'TITLE', 'SITES', 'OFFSETS (M)', 'FREQUENCIES',
'DATA BLOCKS', 'DATUM', 'ERROR'
] #data of simple file.or dat
_occam_modelfile = [
'FORMAT', 'MODEL NAME', 'DESCRIPTION', 'MESH FILE', 'MESH TYPE',
'STATICS FILE', 'PREJUDICE FILE', 'BINDING OFFSET', 'NUM LAYERS'
] # model of simple file
_occam_meshfile = [
'?????????????', 'ZZZZZZZZZZZZ', 'YYYYYYYYYYYY', '000000000000',
'AAAAAAAAAAAAA'
] # end by mesh or simple file
#Check the firstS layer (LIST NUMBER 0 135 52 0 0 2) SPECIFIED the catacteristik of mesh
#only difference between the startup is at the end , the startup file have the same number so
# the iter is not the same case , number change .
#response file , check if the range or the lenght == 7.
_occam_logfile = [
'STARTING R.M.S', ' ** ITERATION', 'TOFMU: MISFIT', 'AND IS', 'USING',
'STEPSIZE IS', 'ROUGHNESS IS'
] #or end with Logfile.
_stn = [
'""dot', '""e', '""h', '""n', '""sta', '""len', '""east', '""north',
'""lat', '""long', '""elev', '""azim', '""stn'
]
@classmethod
def which_file(cls, filename=None, deep=True):
"""
Which file is class method . List of files are the typical files able
to read by pyCSAMT softwares.
Sensitive class method.
Parameters
----------
**filename** :str
corresponding file to read , pathLike
**deep** : bool ,
control reading : False for just control the extension file ,
not opening file . True control in deeper file and
find which file were inputted.
Returns
---------
str
FileType could be [`avg` | `j` | `edi` | `resp` | `mesh` | `occamdat` |
`stn` | `model` | `iter` | `logfile` | `startup`]
List of files read by pyCSAMT :
============== =======================================================
CodeFile Description
============== =======================================================
*avg* Zonge Engineering file Plainty file of ASTATIC file.
*j* A.G .Jonhson J=Format file.
*edi* SEG (Society of Exploration Geophysics) Electrical
Data Interchange file (SEG-EDI)
*stn* Zonge Engineering station file
*occamdat* deGroot-Hedlin, C., and S. Constable, Occam file.
*mesh* Constable, S. C., R. L. Parker, and C. G. Constable
mesh file
*model* Occam Model file
*startup* Occam startup file
*iter* Occam iteration file get after Inversion
*resp* Occam response file , Get after inversion
*logfile* Occam Logfile, Get after inverson (Inversion file )
============== =======================================================
:Example:
>>> files = ['K1_exp.bln','LCS01.avg' ,'LCS01_2_to_1.avg', 'K1.stn',
... 'csi000.dat','csa250.edi','LogFile.logfile',
... 'Occam2DMesh','Occam2DModel', 'OccamDataFile.dat',
... 'S00_ss.edi', 'Startup','RESP13.resp',
... 'ITER02.iter']
>>> for ii in files :
>>> path = os.path.join(os.environ["pyCSAMT"],
... 'csamtpy','data', ii)
... try :
... print(_sensitive.which_file(path,deep=True))
... except :pass
"""
_code = {
'edi': cls._edi,
'j': cls._j,
'startup': cls._occam_startup,
'occamdat': cls._occam_datafile,
'model': cls._occam_modelfile,
'mesh': cls._occam_meshfile,
'iter': cls._occam_startup,
'avg': cls._avg,
'stn': cls._stn,
'logfile': cls._occam_logfile,
'resp': ['none', 'none'],
}
_filename = filename
if _filename is None:
pyCSAMTError_file_handling(
'Error file: NoneType can not be computed. Check your right path.'
)
if _filename is not None:
if os.path.basename(_filename).split('.')[-1].lower() in list(
_code.keys()): #code file recognized
_flag = os.path.basename(_filename).split('.')[-1].lower()
if deep == False: return _flag
else: _flag = 0 #file not recognized
# Open the file now
with open(_filename, 'r', encoding='utf8') as _f:
_data_lines = _f.readlines()
if _flag != 0:
# file extension has been recognized on dict code keys.
cls._logger.info('Reading %s file' % os.path.basename(_filename))
_value_code = _code[_flag]
if _flag == 'edi': #avoid expensive function then controle 2thing before continue
if _value_code[0] not in _data_lines[0]:
warnings.warn('File probably is incorrect. Please consult EDI_Main file components : {0}'.\
format(''.join(cls._edi)))
raise pyCSAMTError_file_handling(
"Wrong SEG-EDI file. It'seems -File provided doesnt include '>HEAD. Please provide the right EDI-FORMAT"
)
elif _value_code[-1] not in _data_lines[-1]:
warnings.warn('File probably incorrect. Please consult EDI_Main file composents : {0}'.\
format(''.join(cls._edi)))
raise pyCSAMTError_file_handling(
"Wrong SEG-EDI file. It'seems -File provided doesnt include '>END. Please provide the right EDI-FORMAT"
)
elif _flag == 'resp': # control quick response file
#check if the len of resp= 7
if len(_data_lines[0].strip().split()) != 7:
raise pyCSAMTError_file_handling(
"Wrong OCCAM-RESPONSE file. Please provide the right response file."
)
else:
return _flag
elif _flag == 'avg':
_reason = cls.validate_avg(avg_data_lines=_data_lines)[0]
if _reason not in ['yesAST', 'yes']:
warnings(
'File <{0}> does not match Zonge AVGfile. Get info to {1}'
.format(os.path.basename(_filename), notion.AVG))
raise pyCSAMTError_file_handling(
"Wrong <{0}>Zonge AVG-file. Please provide the right file."
.format(os.path.basename(_filename)))
return _flag
elif _flag == 'mesh':
try:
[float(ss) for ss in _data_lines[1].strip().split()
] #check the first occam layer
except:
warnings.warn(
'Wrong OCCAM-MESH file.Get more info on that site :https://marineemlab.ucsd.edu/Projects/Occam/index.html'
)
raise pyCSAMTError_file_handling("Wrong OCCAM-MESH <%s>file .We don't find the characteristic of thefirst layer."\
"Please provide the right OCCAM meshfile." % os.path.basename(_filename))
for _meshcodec in _value_code:
if _meshcodec in _data_lines[-1]: return _flag
else:
warnings.warn(
'Trouble occurs while reading the "OCCAM-params specs.More details in :{0}.\n {1}'
.format(notion.occam_mesh_parameter_specs_info,
notion.occam_mesh_parameter_specs_meaning))
raise pyCSAMTError_file_handling(
'We do not find a "Parameter specs" in your <{0}>'.
format(os.path.basename(_filename)))
#once both thing has already been controled . then continue . avoid avg because already has deep controlled
# with special static method then pass
_ccounter = 0 # threshold control is match to 7 except stn.
for _codec in _value_code: # loop the dict value of code #loop now all the
if _flag not in ['avg', 'resp', 'mesh']:
for num, _data_item in enumerate(_data_lines):
if _codec in _data_item:
_ccounter += 1
if _flag == 'stn':
if _ccounter >= 3: return _flag
elif _flag == 'j':
if _ccounter >= 5: return _flag
if _ccounter >= 7: return _flag
if _ccounter < 7: raise pyCSAMTError_file_handling('ERROR occurs while reading file !<{0}>. '\
'File doesnt not match the typical <{1}> Format.'.format(os.path.basename(_filename), _flag))
elif _flag == 0: #file not recognized in _codekeys ,
#then try to read and to find which file mastch the best.
_ccounter = 0
for _keycode, _value_code in _code.items():
for num, valueitem in enumerate(_data_lines):
for codec in _value_code: #read value in code
if codec.lower() in valueitem.lower():
_ccounter += 1
if _keycode == 'j':
if _ccounter >= 4: return _keycode
if _ccounter >= 7:
if _keycode == 'startup':
if all([
float(val)
for val in _data_lines[len(_value_code) +
1].strip().split()
]) == False:
return 'iter'
else:
return _keycode
return _keycode
#Now we assune that the file must be the response file then read according response control
try:
resp0 = [float(ss) for ss in _data_lines[0]]
resp_end = [float(ss) for ss in _data_lines[-1]]
resp_mid = [
float(ss) for ss in _data_lines[int(len(_data_lines) / 2)]
] # get the middele of file and try float converter
assert (len(resp0) == len(resp_end)) and (len(resp_end)==len(resp_mid)),\
pyCSAMTError_file_handling('We presume that RESP <{0}> file doesnt not match a OCCAM-RESPONSE file.'.format(os.path.basename(_filename)))
except:
message = ''.join(
'Error reading File. pyCSAMT can read only <{0}> Format file.',
'Format file. Program can not find the type of that file : <{1}>.',
'You may use script <rewriteXXX> of pyCSAMT to try to rewrite file',
'either to SEG-EDI |j|STN|AVG| or OCCAM{DAT, RESP,MODEL,ITER} files.'
)
cls._logger.error(
message.format('|'.join(list(_code.keys())),
os.path.basename(_filename)))
warnings.warn(
message.format('|'.join(list(_code.keys())),
os.path.basename(_filename)))
return
return 'resp'
@staticmethod
def validate_avg(avg_data_lines):
"""
Core function to validate avg file .
:param avg_data_lines: list of avgfile
:type avg_data_lines: list
:returns: 'yesAST' or 'yes' where 'yesAST' is Astatic file
and 'yes' is plainty avg file (the main file)
:rtype: str
:returns: item, spliting the headAvg components strutured by file.
:rtype: list
:Example:
>>> from csamtpy.etc.infos.Infos inmport _sensistive as SB
>>> path = os.path.join(os.environ["pyCSAMT"],
... 'csamtpy','data', LCS01_2_to_1.avg)
... with open (path, 'r', encoding ='utf8') as f :
... datalines = f.readlines()
... ss =SB._sensitive.validate_avg(avg_data_lines=datalines)
"""
num = [0]
for jj, code in enumerate([
'ASTATIC', '$Stn', '$Survey', '$Tx', '$Rx', 'ARes.mag', 'SRes',
'Z.phz', 'Tx.Amp', 'ARes.%err'
]):
for items in avg_data_lines:
if items.find(code) > 0: num.append(1)
if jj >= 5 and (sum(num) >= 3):
if items.find(code) > 0: item = items.strip().split(',')
return ('yesAST', item)
if jj >= 5 and (sum(num) < 3): break
for jj, code in enumerate([
'AMTAVG', 'ASPACE', 'XMTR', 'skp', '\\-', 'Resistivity',
'Phase', 'Comp', ' Amps', '%Rho'
]):
for items in avg_data_lines:
if items.find(code) > 0: num.append(1)
if jj >= 4 and (sum(num) >= 3):
if items.find(code) > 0:
item = items.strip().split()
return ('yes', item)
if sum(num) == 0:
raise pyCSAMTError_file_handling(
'Data format provided are not typically issued from Zonge AVG format .'
)
