def find_1d_distortion(z_object, include_non1d = False):
"""
find 1D distortion tensor from z object
ONly use the 1D part of the Z to determine D.
Treat all frequencies as 1D, if "include_non1d = True".
"""
if not isinstance(z_object, MTz.Z):
raise MTex.MTpyError_inputarguments('first argument must be an '
'instance of the Z class')
z_obj = z_object
lo_dims = MTge.dimensionality(z_object=z_obj)
if include_non1d is True:
lo_dims = [1 for i in lo_dims]
if len(list(np.where(np.array(lo_dims) == 1))) == 0:
raise MTex.MTpyError_inputarguments('Z object does not have '
'frequencies with spatial 1D characteristic')
print lo_dims
return find_distortion(z_obj, lo_dims = lo_dims)
def multiplymatrices_incl_errors(inmatrix1,
inmatrix2,
inmatrix1_err=None,
inmatrix2_err=None):
if inmatrix1 is None or inmatrix2 is None:
raise MTex.MTpyError_inputarguments(
'ERROR - two 2x2 arrays needed as input')
if inmatrix1.shape != inmatrix2.shape:
raise MTex.MTpyError_inputarguments(
'ERROR - two 2x2 arrays with same dimensions needed as input')
prod = np.array(np.dot(np.matrix(inmatrix1), np.matrix(inmatrix2)))
if (inmatrix1_err is None) or (inmatrix1_err is None):
return prod, None
var = np.zeros((2, 2))
var[0,0] = (inmatrix1_err[0,0] * inmatrix2[0,0])**2 + (inmatrix1_err[0,1] * inmatrix2[1,0])**2+\
(inmatrix2_err[0,0] * inmatrix1[0,0])**2 + (inmatrix2_err[1,0] * inmatrix1[0,1])**2
var[0,1] = (inmatrix1_err[0,0] * inmatrix2[0,1])**2 + (inmatrix1_err[0,1] * inmatrix2[1,1])**2+\
(inmatrix2_err[0,1] * inmatrix1[0,0])**2 + (inmatrix2_err[1,1] * inmatrix1[0,1])**2
var[1,0] = (inmatrix1_err[1,0] * inmatrix2[0,0])**2 + (inmatrix1_err[1,1] * inmatrix2[1,0])**2+\
(inmatrix2_err[0,0] * inmatrix1[1,0])**2 + (inmatrix2_err[1,0] * inmatrix1[1,1])**2
var[1,1] = (inmatrix1_err[1,0] * inmatrix2[0,1])**2 + (inmatrix1_err[1,1] * inmatrix2[1,1])**2+\
(inmatrix2_err[0,1] * inmatrix1[1,0])**2 + (inmatrix2_err[1,1] * inmatrix1[1,1])**2
return prod, np.sqrt(var)
def find_2d_distortion(z_object, include_non2d=False):
"""
find 2D distortion tensor from z object
ONly use the 2D part of the Z to determine D.
Treat all frequencies as 2D, if "include_non2d = True".
"""
if not isinstance(z_object, MTz.Z):
raise MTex.MTpyError_inputarguments('first argument must be an '
'instance of the Z class')
z_obj = z_object
lo_dims = MTge.dimensionality(z_object = z_obj)
#avoid the (standard) 1D distortion call -> remove all 1
lo_dims = [ 4 if i == 1 else i for i in lo_dims ]
if include_non2d is True:
lo_dims = [2 for i in lo_dims]
if len(list(np.where(np.array(lo_dims) == 2))) == 0:
raise MTex.MTpyError_inputarguments('Z object does not have'
' frequencies with spatial 2D characteristic')
return find_distortion(z_obj, lo_dims = lo_dims)
def read_data_header(fn_raw):
"""
Deprecated!!!
USE
read_ts_header
INSTEAD
Read the header line of MTpy TS data files.
input
-----
MTpy TS data file name
output
-------
list of header elements:
stationname, channel, sampling rate, starttime first sample,
starttime last sample, unit, lat, lon, elevation
"""
fn = op.abspath(op.realpath(fn_raw))
if not op.isfile(fn):
raise MTex.MTpyError_inputarguments('Not a file:%s' % fn)
try:
F = open(fn, 'r')
except:
raise MTex.MTpyError_inputarguments('File not readable:%s' % fn)
firstline = F.readline().strip().split()
if not firstline[0][0] == '#':
raise MTex.MTpyError_ts_data('Time series data file does '
'not have a proper header:%s' % fn)
F.close()
header_list = []
idx_header = 0
if len(firstline[0]) > 1:
header_list.append(firstline[0][1:].upper())
else:
header_list.append(firstline[1].upper())
idx_header += 1
header_list.append(firstline[idx_header + 1].lower())
header_list.append(float(firstline[idx_header + 2]))
header_list.append(float(firstline[idx_header + 3]))
header_list.append(int(float(firstline[idx_header + 4])))
header_list.append(firstline[idx_header + 5].lower())
header_list.append(float(firstline[idx_header + 6]))
header_list.append(float(firstline[idx_header + 7]))
header_list.append(float(firstline[idx_header + 8]))
return header_list
def reorient_data2D(x_values, y_values, x_sensor_angle = 0 , y_sensor_angle = 90):
"""
Re-orient time series data of a sensor pair, which has not been in default (x=0, y=90) orientation.
Input:
- x-values - Numpy array
- y-values - Numpy array
Note: same length for both! - If not, the shorter length is taken
Optional:
- Angle of the x-sensor - measured in degrees, clockwise from North (0)
- Angle of the y-sensor - measured in degrees, clockwise from North (0)
Output:
- corrected x-values (North)
- corrected y-values (East)
"""
x_values = np.array(x_values)
y_values = np.array(y_values)
try:
if x_values.dtype not in ['complex', 'float', 'int']:
raise
if len(x_values) != len(y_values):
raise
except:
raise MTex.MTpyError_inputarguments('ERROR - both input arrays must be of same length')
if len(x_values) != len(y_values):
l = min(len(x_values) , len(y_values))
x_values = x_values[:l]
y_values = y_values[:l]
in_array = np.zeros((len(x_values), 2), x_values.dtype)
in_array[:,0] = x_values
in_array[:,1] = y_values
try:
x_angle = math.radians(x_sensor_angle)
y_angle = math.radians(y_sensor_angle)
except:
raise MTex.MTpyError_inputarguments('ERROR - both angles must be of type int or float')
T = np.matrix( [[ np.real(cmath.rect(1,x_angle)), np.imag(cmath.rect(1,x_angle))],[np.real(cmath.rect(1,y_angle)), np.imag(cmath.rect(1,y_angle))]])
try:
new_array = np.dot(in_array, T.I)
except:
raise MTex.MTpyError_inputarguments('ERROR - angles must define independent axes to span 2D')
#print new_array.shape
return new_array[:,0], new_array[:,1]
def rotatematrix_incl_errors(inmatrix, angle, inmatrix_err=None):
if inmatrix is None:
raise MTex.MTpyError_inputarguments(
'Matrix AND eror matrix must be defined')
if (inmatrix_err is not None) and (inmatrix.shape != inmatrix_err.shape):
raise MTex.MTpyError_inputarguments(
'Matrix and err-matrix shapes do not match: %s - %s' %
(str(inmatrix.shape), str(inmatrix_err.shape)))
try:
degreeangle = angle % 360
except:
raise MTex.MTpyError_inputarguments(
'"Angle" must be a valid number (in degrees)')
phi = math.radians(degreeangle)
cphi = np.cos(phi)
sphi = np.sin(phi)
# JP: Changed the rotation matrix to be formulated to rotate
# counter clockwise, I cannot find a good reason for this except that
# when you plot the strike and phase tensors the look correct with this
# formulation.
rotmat = np.array([[cphi, sphi], [-sphi, cphi]])
# rotmat = np.array([[ cphi, -sphi], [sphi, cphi]])
rotated_matrix = np.dot(np.dot(rotmat, inmatrix), np.linalg.inv(rotmat))
errmat = None
if (inmatrix_err is not None):
err_orig = np.real(inmatrix_err)
errmat = np.zeros_like(inmatrix_err)
# standard propagation of errors:
errmat[0,0] = np.sqrt( (cphi**2 * err_orig[0,0])**2 + \
(cphi * sphi * err_orig[0,1])**2 + \
(cphi * sphi * err_orig[1,0])**2 + \
(sphi**2 * err_orig[1,1])**2 )
errmat[0,1] = np.sqrt( (cphi**2 * err_orig[0,1])**2 + \
(cphi * sphi * err_orig[1,1])**2 + \
(cphi * sphi * err_orig[0,0])**2 + \
(sphi**2 * err_orig[1,0])**2 )
errmat[1,0] = np.sqrt( (cphi**2 * err_orig[1,0])**2 + \
(cphi * sphi * err_orig[1,1])**2 +\
(cphi * sphi * err_orig[0,0])**2 + \
(sphi**2 * err_orig[0,1])**2 )
errmat[1,1] = np.sqrt( (cphi**2 * err_orig[1,1])**2 + \
(cphi * sphi * err_orig[0,1])**2 + \
(cphi * sphi * err_orig[1,0])**2 + \
(sphi**2 * err_orig[0,0])**2 )
return rotated_matrix, errmat
def read_configfile(filename):
"""
Read a general config file and return the content as dictionary.
Config files without sections or only DEFAULT section
-> return dictionary
Config files with sections
-> return nested dictionary (main level keys are section heads)
Config files with sections as well as section-less entries
-> return nested dictionary, which includes a top level 'DEFAULT' key
"""
#check, if file is present
if not op.isfile(filename):
raise MTex.MTpyError_inputarguments(
'File does not exist: {0}'.format(filename))
# try to parse file - exit, if not a config file
try:
#generate config parser instance
configobject = ConfigParser.SafeConfigParser()
#do NOT ask, why it does not work with reading from filename directly...:
with open(filename) as F:
d = F.read()
FH = StringIO.StringIO(d)
configobject.readfp(d) #filename)
except:
try:
dummy_String = '[DEFAULT]\n' + open(filename, 'r').read()
FH = StringIO.StringIO(dummy_String)
#generate config parser instance
configobject = ConfigParser.SafeConfigParser()
configobject.readfp(FH)
except:
raise MTex.MTpyError_inputarguments(
'File is not a proper '
'configuration file: {0}'.format(filename))
config_dict = configobject._sections
if len(config_dict.keys()) != 0:
defaults = configobject.defaults()
if len(defaults.keys()) != 0:
config_dict['DEFAULT'] = configobject.defaults()
else:
config_dict = configobject.defaults()
return config_dict
def main():
if len(sys.argv) < 4:
print('\nNeed at least 3 arguments: <path to BIRRP executable> '\
'<station name> <directory for time series>\n\n'\
'Optional arguments: \n [coherence threshold]\n'\
' [start time] \n [end time]\n\n')
return
try:
coherence_th = float(sys.argv[4])
if not 0 <= coherence_th <= 1:
raise
except:
print(
'coherence value invalid (float from interval [0,1]) - set to 0 instead'
)
coherence_th = 0
try:
starttime = float(sys.argv[5])
except:
starttime = None
try:
endtime = float(sys.argv[6])
except:
endtime = None
birrp_exe_raw = sys.argv[1]
birrp_exe = op.abspath(op.realpath(birrp_exe_raw))
if not op.isfile(birrp_exe):
raise MTex.MTpyError_inputarguments(
'Birrp executable not existing: %s' % (birrp_exe))
stationname = sys.argv[2].upper()
ts_dir_raw = sys.argv[3]
ts_dir = op.abspath(op.realpath(ts_dir_raw))
if not op.isdir(ts_dir):
raise MTex.MTpyError_inputarguments(
'Time series directory not existing: %s' % (ts_dir))
if 1:
MTbp.runbirrp_Nin2out_simple(birrp_exe, stationname, ts_dir,
coherence_th, None, 3, None, starttime,
endtime)
def rotatematrix_incl_errors(inmatrix, angle, inmatrix_err = None) :
if inmatrix is None :
raise MTex.MTpyError_inputarguments('Matrix AND eror matrix must be defined')
if (inmatrix_err is not None) and (inmatrix.shape != inmatrix_err.shape):
raise MTex.MTpyError_inputarguments('Matrix and err-matrix shapes do not match: %s - %s'%(str(inmatrix.shape), str(inmatrix_err.shape)))
try:
degreeangle = angle%360
except:
raise MTex.MTpyError_inputarguments('"Angle" must be a valid number (in degrees)')
phi = math.radians(degreeangle)
cphi = np.cos(phi)
sphi = np.sin(phi)
rotmat = np.matrix([[ cphi,sphi],[-sphi,cphi] ])
rotated_matrix = np.dot(np.dot( rotmat, inmatrix ),rotmat.I )
#print rotmat
#print inmatrix
#print rotated_matrix
#sys.exit()
errmat = None
if (inmatrix_err is not None) :
err_orig = np.real(inmatrix_err)
errmat = np.zeros_like(inmatrix_err)
# squared propagation of errors
# z_err_rot[idx_freq,0,0] = np.sqrt( (cphi**2 * z_err_orig[0,0])**2 + cphi**2 * sphi**2 * ( (z_err_orig[0,1])**2 + (z_err_orig[1,0])**2) + (sphi**2 * z_err_orig[1,1])**2)
# z_err_rot[idx_freq,0,1] = np.sqrt( (cphi**2 * z_err_orig[0,1])**2 + cphi**2 * sphi**2 * ( (z_err_orig[1,1])**2 + (z_err_orig[0,0])**2) + (sphi**2 * z_err_orig[1,0])**2)
# z_err_rot[idx_freq,1,0] = np.sqrt( (cphi**2 * z_err_orig[1,0])**2 + cphi**2 * sphi**2 * ( (z_err_orig[1,1])**2 + (z_err_orig[0,0])**2) + (sphi**2 * z_err_orig[0,1])**2)
# z_err_rot[idx_freq,1,1] = np.sqrt( (sphi**2 * z_err_orig[0,0])**2 + cphi**2 * sphi**2 * ( (z_err_orig[0,1])**2 + (z_err_orig[1,0])**2) + (cphi**2 * z_err_orig[1,1])**2)
# standard propagation of errors:
errmat[0,0] = np.sqrt( (cphi**2 * err_orig[0,0])**2 + (cphi * sphi * err_orig[0,1])**2 + (cphi * sphi * err_orig[1,0])**2 + (sphi**2 * err_orig[1,1])**2 )
errmat[0,1] = np.sqrt( (cphi**2 * err_orig[0,1])**2 + (cphi * sphi * err_orig[1,1])**2 + (cphi * sphi * err_orig[0,0])**2 + (sphi**2 * err_orig[1,0])**2 )
errmat[1,0] = np.sqrt( (cphi**2 * err_orig[1,0])**2 + (cphi * sphi * err_orig[1,1])**2 + (cphi * sphi * err_orig[0,0])**2 + (sphi**2 * err_orig[0,1])**2 )
errmat[1,1] = np.sqrt( (cphi**2 * err_orig[1,1])**2 + (cphi * sphi * err_orig[0,1])**2 + (cphi * sphi * err_orig[1,0])**2 + (sphi**2 * err_orig[0,0])**2 )
return rotated_matrix, errmat
def main():
if len(sys.argv) < 2:
sys.exit(
'\n usage:\n\t get_sampling_interval_from_data_file.py <filename>'
'[optional: <file length in seconds>]\n\n'
'If no second argument is given, a file length of 3600 seconds is assumed.\n\n'
)
filename_raw = sys.argv[1]
filename = op.abspath(op.realpath(filename_raw))
if not op.isfile(filename):
raise MTex.MTpyError_inputarguments('File not existing: %s' %
(filename))
try:
length = float(sys.argv[2])
if length <= 0:
raise
except:
print 'Could not understand second argument - must be a length in seconds (int/float) - set to 3600 '
length = 3600
print MTfh.get_sampling_interval_fromdatafile(filename, length=length)
def rhophi2z(rho, phi, freq):
"""
Convert impedance-style information given in Rho/Phi format into complex valued Z.
Input:
rho - 2x2 array (real) - in Ohm m
phi - 2x2 array (real) - in degrees
freq - scalar - frequency in Hz
Output:
Z - 2x2 array (complex)
"""
try:
if rho.shape != (2, 2) or phi.shape != (2, 2):
raise
if not (rho.dtype in ['float', 'int']
and phi.dtype in ['float', 'int']):
raise
except:
raise MTex.MTpyError_inputarguments(
'ERROR - arguments must be two 2x2 arrays (real)')
z = np.zeros((2, 2), 'complex')
for i in range(2):
for j in range(2):
abs_z = np.sqrt(5 * freq * rho[i, j])
z[i, j] = cmath.rect(abs_z, math.radians(phi[i, j]))
return z
def convert_degmin_tuple2degrees(latlon_list):
"""
Convert a 2tuple (list, tuple, array) of form "(degrees, minutes)" into degrees.
Validity of the triple is assumed and has to be asserted in advance.
"""
sign = 1.
try:
latlon_list = [float(i) for i in latlon_list]
if str(latlon_list[0]).startswith('-'):
sign = -1.
except:
raise
deg = latlon_list[0]
minutes = latlon_list[1]
if not (0<=minutes<60):
raise MTex.MTpyError_inputarguments('Minutes value invalid')
#take out sign for easier conversion into degrees
if deg < 0:
deg *= -1.
degrees = deg + 1/60. * minutes
return degrees * sign
def write_ts_file_from_tuple(outfile, ts_tuple, fmt='%.8e'):
"""
Write an MTpy TS data file, where the content is provided as tuple:
(station, channel,samplingrate,t_min,nsamples,unit,lat,lon,elev, data)
todo:
needs tuple-validation
"""
header_dict = {}
for i in range(len(ts_tuple) - 1):
if ts_tuple[i] is not None:
header_dict[lo_headerelements[i]] = ts_tuple[i]
header_string = get_ts_header_string(header_dict)
data = ts_tuple[-1]
outfilename = make_unique_filename(outfile)
try:
outF = open(outfilename, 'w')
outF.write(header_string)
np.savetxt(outF, data, fmt=fmt)
outF.close()
except:
raise MTex.MTpyError_inputarguments(
'ERROR - could not write content'
' of TS tuple to file : {0}'.format(outfilename))
return outfilename
def rotatevector_incl_errors(invector, angle, invector_err=None):
#check for row or column vector
if invector is None:
raise MTex.MTpyError_inputarguments(
'Vector AND error-vector must be defined')
if (invector_err is not None) and (invector.shape != invector_err.shape):
raise MTex.MTpyError_inputarguments(
'Vector and errror-vector shapes do not match: %s - %s' %
(str(invector.shape), str(invector_err.shape)))
try:
degreeangle = angle % 360
except:
raise MTex.MTpyError_inputarguments(
'"Angle" must be a valid number (in degrees)')
phi = math.radians(degreeangle)
cphi = np.cos(phi)
sphi = np.sin(phi)
# JP: Changed the rotation matrix to be formulated to rotate
# counter clockwise, I cannot find a good reason for this except that
# when you plot the strike and phase tensors the look correct with this
# formulation.
rotmat = np.array([[cphi, sphi], [-sphi, cphi]])
# rotmat = np.array([[ cphi, -sphi],[sphi, cphi]])
if invector.shape == (1, 2):
rotated_vector = np.dot(invector, np.linalg.inv(rotmat))
else:
rotated_vector = np.dot(rotmat, invector)
errvec = None
if (invector_err is not None):
errvec = np.zeros_like(invector_err)
if invector_err.shape == (1, 2):
errvec = np.dot(invector_err, np.abs(np.linalg.inv(rotmat)))
else:
errvec = np.dot(np.abs(rotmat), invector_err)
return rotated_vector, errvec
def read_ts_header(tsfile):
""" Read in the header line from MTpy timeseries data files.
Return header as dictionary. Return empty dict,
if no header line was found.
"""
header_dict = {}
tsfile = op.abspath(tsfile)
if not op.isfile(tsfile):
raise MTex.MTpyError_inputarguments('Error - '
'input file not existing: {0}'.format(tsfile))
try:
with open(tsfile,'r') as F:
firstline =''
#ignoring empty lines or lines with just the '#' character in it
while len(firstline) == 0:
firstline = F.readline().strip()
if firstline == '#':
firstline = ''
if firstline[0] != '#':
raise
except:
raise MTex.MTpyError_ts_data('No header line found -'
' check file: {0}'.format(tsfile))
firstline = firstline.replace('#','')
headerlist = firstline.split()
for i in range(len(headerlist)):
header_dict[lo_headerelements[i]] = headerlist[i]
#old header had tmax instead of n_samples:
if ((i == 4) and float(headerlist[4])%1 != 0
and float(headerlist[i]) > float(headerlist[i-1])):
header_dict[lo_headerelements[i]] = int(
(float(headerlist[i]) - float(headerlist[i-1])
)*float(headerlist[i-2]) )+1
headerlements = ['samplingrate','t_min','nsamples','lat','lon','elev']
for h in headerlements:
try:
header_dict[h] = float(header_dict[h])
except:
pass
try:
if header_dict[h]%1==0:
header_dict[h] = int(header_dict[h])
except:
pass
return header_dict
def _data_instrument_consistency_check(data, field, dipole_length, instrument,
amplification, logger, gain):
"""
Check, if the input values given for the calibration make any sense at all.
"""
if len(data) == 0 :
raise MTex.MTpyError_ts_data( 'no data provided for calibration' )
if not field.lower() in ['e','b']:
raise MTex.MTpyError_inputarguments( 'Field must be E or B' )
# if float(dipole_length) <= 0:
# raise MTpyError_inputarguments( 'Dipole length must be positive' )
if float(amplification) <= 0:
raise MTex.MTpyError_inputarguments('Amplification factor must be positive')
if float(gain) <= 0:
raise MTex.MTpyError_inputarguments('Instrument gain must be positive')
try:
if not logger.lower() in list_of_loggers:
raise
except:
raise MTex.MTpyError_inputarguments( 'wrong choice of logger')
try:
if not instrument.lower() in list_of_instruments:
raise
except:
raise MTex.MTpyError_inputarguments( 'wrong choice of instrument')
if field.lower == 'b':
if logger.lower() == 'elogger':
raise MTex.MTpyError_inputarguments( 'wrong choice of logger')
if instrument.lower() == 'electrodes':
raise MTex.MTpyError_inputarguments( 'wrong choice of instrument')
if not float(dipole_length) == 1:
raise MTex.MTpyError_inputarguments( 'Dipole length must be "1" for'
' B-field calibration')
if field.lower == 'e':
if not instrument.lower() == 'electrodes':
raise MTex.MTpyError_inputarguments( 'wrong choice of instrument')
def rotatevector_incl_errors(invector, angle, invector_err=None):
#check for row or column vector
if invector is None:
raise MTex.MTpyError_inputarguments(
'Vector AND error-vector must be defined')
if (invector_err is not None) and (invector.shape != invector_err.shape):
raise MTex.MTpyError_inputarguments(
'Vector and errror-vector shapes do not match: %s - %s' %
(str(invector.shape), str(invector_err.shape)))
try:
degreeangle = angle % 360
except:
raise MTex.MTpyError_inputarguments(
'"Angle" must be a valid number (in degrees)')
phi = math.radians(degreeangle)
cphi = np.cos(phi)
sphi = np.sin(phi)
rotmat = np.matrix([[cphi, sphi], [-sphi, cphi]])
if invector.shape == (1, 2):
rotated_vector = np.dot(invector, rotmat.I)
else:
rotated_vector = np.dot(rotmat, invector)
errvec = None
if (invector_err is not None):
err_orig = np.real(invector_err)
errvec = np.zeros_like(invector_err)
if invector_err.shape == (1, 2):
errvec = np.dot(invector_err, np.abs(rotmat.I))
else:
errvec = np.dot(np.abs(rotmat), invector_err)
return rotated_vector, errvec
def read_ts_file(mtdatafile):
"""
Read an MTpy TS data file and provide the content as tuple:
(station, channel,samplingrate,t_min,nsamples,unit,lat,lon,elev, data)
If header information is incomplete, the tuple is filled up with 'None'
"""
infile = op.abspath(mtdatafile)
if not op.isfile(infile):
raise MTex.MTpyError_inputarguments('ERROR - Data file not '
'existing: {0}'.format(infile))
header = read_ts_header(infile)
if len(header) == 0:
raise MTex.MTpyError_inputarguments(
'ERROR - Data file not valid - '
'header is missing : {0}'.format(infile))
data = np.loadtxt(infile)
if len(data) != int(float(header['nsamples'])):
raise MTex.MTpyError_inputarguments(
'ERROR - Data file not valid '
'- wrong number of samples in data ({1} '
'instead of {2}): {0}'.format(infile, len(data),
int(float(header['nsamples']))))
lo_header_contents = []
for i in lo_headerelements:
if i in header:
lo_header_contents.append(header[i])
else:
lo_header_contents.append(None)
lo_header_contents.append(data)
return tuple(lo_header_contents)
def convert_ms2ts(indir, outdir, combine=True, invert=False):
lo_dirs = []
for i, j, k in os.walk(indir):
lofolders = [op.join(i, f) for f in j]
lo_dirs.extend(lofolders)
lo_dirs.append(indir)
pathname = list(set(lo_dirs))
if len(pathname) == 0:
pathname = [indir]
lo_indirs = [op.relpath(i, start=indir) for i in pathname]
lo_outdirs = []
#'pathname' is a list of relative pathnames. to be reconstructed under the given 'outdir'
try:
for i in lo_indirs:
outpath = op.abspath(op.join(outdir, i))
if not op.isdir(outpath):
os.makedirs(outpath)
lo_outdirs.append(outpath)
except:
raise MTex.MTpyError_inputarguments(
'ERROR - Cannot set up output directory {0}'.format(outpath))
lo_indirs = [op.join(indir, i) for i in lo_indirs]
for idx_ipath, inpath in enumerate(lo_indirs):
lo_infiles = [
i for i in os.listdir(inpath)
if op.isfile(op.abspath(op.join(inpath, i)))
]
lo_outfiles = [
op.abspath(op.join(lo_outdirs[idx_ipath], i)) for i in lo_infiles
]
lo_infiles = [
op.abspath(op.join(indir, inpath, i)) for i in lo_infiles
]
for idx_fn, fn in enumerate(lo_infiles):
print 'reading file {0}'.format(fn)
try:
outfn = MTms.quadrupol_convertfile_miniseed2ts(
fn, lo_outfiles[idx_fn], combine, invert)
print 'wrote file(s) {0}'.format(outfn)
except:
print 'Warning - file {0} is not in valid miniseed format!!!'.format(
fn)
continue
def read_survey_config_fn(self, survey_config_fn=None):
"""
read in survey configuration file and output into a useful dictionary
"""
if survey_config_fn is not None:
self.surve_config_fn = survey_config_fn
if not os.path.isfile(self.survey_config_fn):
raise mtex.MTpyError_inputarguments(
'Could not find {0}, check path'.format(survey_config_fn))
# read in survey information
self.survey_config_dict = mtcfg.read_survey_configfile(
self.survey_config_fn)[self.station]
def __init__(self, **kwargs):
fn_list = kwargs.pop('fn_list', None)
mt_object_list = kwargs.pop('mt_object_list', None)
#----set attributes for the class-------------------------
self.mt_list = mtpt.MTplot_list(fn_list=fn_list,
mt_object_list=mt_object_list)
#--> set plot properties
self.fig_num = kwargs.pop('fig_num', 1)
self.plot_title = kwargs.pop('plot_title', None)
self.fig_dpi = kwargs.pop('fig_dpi', 300)
self.fig_size = kwargs.pop('fig_size', None)
self.font_size = kwargs.pop('font_size', 7)
self.stationid = kwargs.pop('stationid', [0, 4])
self.xlimits = kwargs.pop('xlimits', None)
self.ylimits = kwargs.pop('ylimits', None)
self.ref_point = kwargs.pop('ref_point', (0, 0))
self.map_scale = kwargs.pop('map_scale', 'latlon')
self.marker = kwargs.pop('marker', 'v')
self.marker_size = kwargs.pop('marker_size', 10)
self.marker_color = kwargs.pop('marker_color', 'k')
self.plot_names = kwargs.pop('plot_names', True)
self.text_size = kwargs.pop('text_size', 7)
self.text_weight = kwargs.pop('text_weight', 'normal')
self.text_color = kwargs.pop('text_color', 'k')
self.text_ha = kwargs.pop('text_ha', 'center')
self.text_va = kwargs.pop('text_va', 'baseline')
self.text_angle = kwargs.pop('text_angle', 0)
self.text_x_pad = kwargs.pop('text_x_pad', None)
self.text_y_pad = kwargs.pop('text_y_pad', None)
self.image_file = kwargs.pop('image_file', None)
self.image_extent = kwargs.pop('image_extent', None)
if self.image_file is not None:
if self.image_extent is None:
raise mtex.MTpyError_inputarguments('Need to input extents '+\
'of the image as' +\
'(x0, y0, x1, y1)')
#--> plot if desired
self.plot_yn = kwargs.pop('plot_yn', 'y')
if self.plot_yn == 'y':
self.plot()
def convert_dms_tuple2degrees(latlon_list):
"""
Convert a triple (list, tuple, array) of degrees, minuts, seconds into degrees.
Validity of the triple is assumed and has to be asserted in advance.
"""
sign = 1.
try:
latlon_list = [float(i) for i in latlon_list]
if str(latlon_list[0]).startswith('-'):
sign = -1.
except:
# if triple is given as string:
latlon_raw = latlon_list
# allow separation by :,. or space for (deg min sec) format
try:
latlon_list = re.split('[ :,]', latlon_raw)
except:
raise MTex.MTpyError_config_file(
'Config file error: lat/lon is in invalid format')
try:
latlon_list = [float(i) for i in latlon_list]
except:
raise MTex.MTpyError_config_file(
'Config file error: lat/lon is in invalid format')
if str(latlon_list[0])[0] == '-':
sign = -1.
deg = latlon_list[0]
minutes = latlon_list[1]
seconds = latlon_list[2]
if not (-180 <= deg <= 360 and 0 <= minutes < 60 and 0 <= seconds < 60):
print 'Value for degrees, minutes, or seconds invalid'
raise MTex.MTpyError_inputarguments()
# take out sign for easier conversion into degrees
if deg < 0:
deg *= -1
degrees = deg + 1 / 60. * minutes + 1 / 3600. * seconds
return degrees * sign
def invertmatrix_incl_errors(inmatrix, inmatrix_err=None):
if inmatrix is None:
raise MTex.MTpyError_inputarguments('Matrix must be defined')
if (inmatrix_err is not None) and (inmatrix.shape != inmatrix_err.shape):
raise MTex.MTpyError_inputarguments(
'Matrix and err-matrix shapes do not match: %s - %s' %
(str(inmatrix.shape), str(inmatrix_err.shape)))
if (inmatrix.shape[-2] != inmatrix.shape[-1]):
raise MTex.MTpyError_inputarguments('Matrices must be square!')
if (inmatrix_err is not None) and (inmatrix_err.shape[-2] !=
inmatrix_err.shape[-1]):
raise MTex.MTpyError_inputarguments('Matrices must be square!')
dim = inmatrix.shape[-1]
det = np.linalg.det(inmatrix)
if det == 0:
raise MTex.MTpyError_inputarguments(
'Matrix is singular - I cannot invert that!')
inv_matrix = np.zeros_like(inmatrix)
if dim != 2:
raise MTex.MTpyError_inputarguments('Only 2D matrices supported yet')
inv_matrix = np.linalg.inv(inmatrix)
inv_matrix_err = None
if (inmatrix_err is not None):
inmatrix_err = np.real(inmatrix_err)
inv_matrix_err = np.zeros_like(inmatrix_err)
for i in range(2):
for j in range(2):
#looping over the entries of the error matrix
err = 0.
for k in range(2):
for l in range(2):
#each entry has 4 summands
err += np.abs(-inv_matrix[i, k] * inv_matrix[l, j] *
inmatrix_err[k, l])
inv_matrix_err[i, j] = err
return inv_matrix, inv_matrix_err
def get_j_file(self, birrp_dir=None):
"""
get .j file output by birrp
"""
if birrp_dir is not None:
self.birrp_dir = birrp_dir
if self.birrp_dir is None or not os.path.exists(self.birrp_dir):
raise mtex.MTpyError_inputarguments('No birrp directory input,'
'check path {0}'.format(
self.birrp_dir))
try:
self.j_fn = [
os.path.join(self.birrp_dir, fn)
for fn in os.listdir(self.birrp_dir) if fn.endswith('.j')
][0]
except IndexError:
print('Could not find a .j file in {0}, check path.'.format(
self.birrp_dir))
self.j_fn = None
def main():
if len(sys.argv) < 3:
raise MTex.MTpyError_inputarguments(
'Need at least 2 arguments: <path to files> <response file> [<output dir>] [<channel(s)>] '
)
pathname_raw = sys.argv[1]
directory = op.abspath(op.realpath(pathname_raw))
responsefilename_raw = sys.argv[2]
responsefile = op.abspath(op.realpath(responsefilename_raw))
if not op.isdir(directory):
raise MTex.MTpyError_inputarguments('Directory not existing: %s' %
(directory))
if not op.isfile(responsefile):
raise MTex.MTpyError_inputarguments('Response file not existing: %s' %
(responsefile))
#check, if response file is in proper shape (3 columns freq,re,im of real values):
try:
responsedata = np.loadtxt(responsefile)
s = responsedata.shape
if s[1] != 3:
raise
freq_min = responsedata[0, 0]
freq_max = responsedata[-1, 0]
except:
raise MTex.MTpyError_inputarguments(
'Response file (%s) in wrong format - must be 3 columns: freq,real,imag'
% (responsefile))
#set up output directory:
try:
outdir_raw = sys.argv[3]
outdir = op.abspath(outdir_raw)
except:
outdir = op.join(directory, 'instr_resp_corrected')
try:
if not op.isdir(outdir):
os.makedirs(outdir)
except:
raise MTex.MTpyError_inputarguments(
'Output directory cannot be generated: %s' % (outdir))
#define channels to be considered for correction:
try:
lo_channels = list(
set([
i.upper() if len(i) == 2 else 'B' + i.upper()
for i in sys.argv[4].split(',')
]))
except:
print 'No channel list found - using BX, BY, HX, HY'
lo_channels = ['BX', 'BY', 'HX', 'HY', 'BZ', 'HZ']
#collect file names within the folder
lo_allfiles = os.listdir(directory)
lo_allfiles = [
op.abspath(op.join(directory, i)) for i in lo_allfiles
if op.isfile(op.abspath(op.join(directory, i)))
] # if op.isfile(i)==True]
#generate list of list-of-files-for-each-channel:
lo_lo_files_for_channels = [[] for i in lo_channels]
#check the files for information about the determined channels:
for fn in lo_allfiles:
header_dict = MTfh.read_ts_header(fn)
if len(header_dict.keys()) == 0:
continue
ch = header_dict['channel'].upper()
if ch not in lo_channels:
continue
try:
ch_idx = lo_channels.index(ch)
except ValueError:
continue
# use the current file, if it contains a header line and contains signal from the requested channel:
lo_lo_files_for_channels[ch_idx].append(fn)
#if no files had header lines or did not contain data from the appropriate channel(s):
if np.sum([len(i) for i in lo_lo_files_for_channels]) == 0:
print 'channels: ', lo_channels, ' - directory: ', directory
raise MTex.MTpyError_inputarguments(
'No information for channels found in directory {0} - Check header lines!'
.format(directory))
#=============================================
# start the instrument correction
# looping over all requested channels:
for ch in lo_channels:
#skip, if no data are available for the current channel:
if [len(i)
for i in lo_lo_files_for_channels][lo_channels.index(ch)] == 0:
continue
#set up lists for the infos needed later, esp. for the file handling
lo_files = lo_lo_files_for_channels[lo_channels.index(ch)]
lo_t_mins = []
lo_headers = []
#read in header lines and sort files by increasing starttimes t_min
for fn in lo_files:
header_dict = MTfh.read_ts_header(fn)
lo_t_mins.append(header_dict['t_min'])
lo_headers.append(header_dict)
#sort all the collected lists by t_min
idxs = np.array(lo_t_mins).argsort()
lo_t_mins = [lo_t_mins[i] for i in idxs]
lo_files = [lo_files[i] for i in idxs]
lo_headers = [lo_headers[i] for i in idxs]
# finding consecutive, continuous time axes:
lo_timeaxes = []
ta_old = None
for idx, header in enumerate(lo_headers):
ta_cur = np.arange(int(header['nsamples'])) / float(
header['samplingrate']) + float(header['t_min'])
#if there is no old ta:
if ta_old == None:
ta_old = ta_cur
continue
# if gap between old and new ta is too big:
if (ta_cur[0] - ta_old[-1]) > (2 * 1. /
float(header['samplingrate'])):
lo_timeaxes.append(np.array(ta_old))
ta_old = ta_cur
continue
#find index of new ta which is closest to the end of old_ta - most commonly it's '0' !
overlap = np.abs(ta_cur - ta_old[-1]).argmin()
ta_cur = ta_cur[overlap:]
ta_old = np.concatenate([ta_old, ta_cur])
#append last active time axis ta:
lo_timeaxes.append(np.array(ta_old))
#determine maximal period from response file and existinng time axes.
#win = get_windowlength() = max([ i[-1]-i[0] for i in lo_timeaxes] )
# the minimum of the maximal resolvable signal period and the longest continuous time axis:
winmax = 1. / freq_min
#for debugging set large window size:
#winmax = 5e5
#later on, if the TS is longer than 3 times this time window, we want to cut out subsections of the time series. These cuts shall consist of triplets of subwindows, each of which shall not be longer than this maximum period.
#Now the data set has to be corrected/deconvolved by looping over the collected time axes:
for ta in lo_timeaxes:
print '\nhandling time axis: {0} - {1} ({2} samples) '.format(
ta[0], ta[-1], len(ta))
#if the time section is shorter than 3 times the maximum defined by the response function, read in the whole data set at once for this interval
if (ta[-1] - ta[0]) < (3 * winmax):
print 'time axis short enough ({0} seconds) - reading all at once'.format(
ta[-1] - ta[0])
#collect the appropriate files in a list
#after the MTpy preprocessing the start end end of the time series coincide with files start and endings, so no "half files" are involved.
cur_time = ta[0]
data = []
files = []
headers = []
starttimes = []
while cur_time < ta[-1]:
for idx, header in enumerate(lo_headers):
ta_cur = np.arange(int(header['nsamples'])) / float(
header['samplingrate']) + float(header['t_min'])
if cur_time in ta_cur:
start_idx = np.where(ta_cur == cur_time)[0][0]
break
fn = lo_files[idx]
files.append(fn)
headers.append(header)
starttimes.append(float(header['t_min']))
cur_data = np.loadtxt(fn)
print 'current data section length: ', len(cur_data)
if ta_cur[-1] <= ta[-1]:
data.extend(cur_data[start_idx:].tolist())
cur_time = ta_cur[-1] + 1. / float(
header['samplingrate'])
else:
end_idx = np.where(ta_cur == ta[-1])[0][0]
data.extend(cur_data[start_idx:end_idx + 1].tolist())
cur_time = ta[-1]
print 'current data length: ', len(data)
data = np.array(data)
data = scipy.signal.detrend(data)
#at this point, the data set should be set up for the given time axis
corrected_timeseries = MTin.correct_for_instrument_response(
data, float(header['samplingrate']), responsedata)
print 'corrected TS starting at {0}, length {1}'.format(
ta[0], len(corrected_timeseries))
#now, save this TS back into the appropriate files, including headers
for idx, fn in enumerate(files):
# output file name: use input file name and append '_true'
inbasename = op.basename(fn)
outbasename = ''.join([
op.splitext(inbasename)[0] + '_true',
op.splitext(inbasename)[1]
])
outfn = op.join(outdir, outbasename)
outF = open(outfn, 'w')
header = headers[idx]
unit = header['unit']
if unit[-6:].lower() != '(true)':
unit += '(true)'
header['unit'] = unit
headerline = MTfh.get_ts_header_string(header)
outF.write(headerline)
starttime = starttimes[idx]
length = int(float(header['nsamples']))
startidx = (np.abs(starttime - ta)).argmin()
print startidx, length, len(corrected_timeseries), len(ta)
print '\nhandling file {0} - starttime {1}, - nsamples {2}'.format(
outfn, starttime, length)
print outdir, outfn
data = corrected_timeseries[startidx:startidx + length]
np.savetxt(outF, data)
outF.close()
#To do so, use the time axis and run over the input files again,determine the filenames. Use them, and put them (slightly modified, perhaps?) into the given output directory
#return corrected_timeseries
else:
#find partition into pieces of length 'winmax'. the remainder is equally split between start and end:
#assume constant sampling rate, just use the last opened header (see above):
samplingrate = float(header['samplingrate'])
#total time axis length:
ta_length = ta[-1] - ta[0] + 1. / samplingrate
#partition into winmax long windows
n_windows = int(ta_length / winmax)
remainder = ta_length % winmax
lo_windowstarts = [ta[0]]
for i in range(n_windows + 1):
t0 = ta[0] + remainder / 2. + i * winmax
lo_windowstarts.append(t0)
print 'time axis long ({0} seconds) - processing in {1} sections (window: {2})'.format(
ta_length, n_windows, winmax)
# lists of input file(s) containing the data - for all 3 sections of the moving window
section1_lo_input_files = []
section2_lo_input_files = []
section3_lo_input_files = []
section1_data = []
section2_data = []
section3_data = []
section1_ta = []
section2_ta = []
section3_ta = []
file_open = False
# loop over the winmax long sections:
for idx_t0, t0 in enumerate(lo_windowstarts):
#the last two window-starts do not get an own window:
if idx_t0 > n_windows - 1:
# since there are 'n_windows' full intervals for the moving window
break
print '\n----- section {0} ----\n'.format(idx_t0 + 1)
if idx_t0 == n_windows - 1:
endtime = ta[-1]
else:
endtime = lo_windowstarts[idx_t0 +
3] - 1. / samplingrate
print 's1 = {0} - {1}, s2 = {2} - {3}, s3 = {4} - {5}\n'.format(
t0, lo_windowstarts[idx_t0 + 1] - 1. / samplingrate,
lo_windowstarts[idx_t0 + 1],
lo_windowstarts[idx_t0 + 2] - 1. / samplingrate,
lo_windowstarts[idx_t0 + 2], endtime)
#for each step (except for he last one obviously), 3 consecutive parts are read in, concatenated and deconvolved. Then the central part is taken as 'true' data.
#only for the first and the last sections (start and end pieces) are handled together with the respective following/preceding section
#the data currently under processing:
data = []
timeaxis = []
#list of current input data files and their starting times - only files, which contain parts of the time interval for the output!
lo_infiles = []
lo_t0s = []
#if old data are present from the step before:
if (len(section2_data) > 0) and (len(section3_data) > 0):
print 'old data found....moving window'
section1_data = section2_data
section2_data = section3_data
section1_ta = section2_ta
section2_ta = section3_ta
section1_lo_input_files = section2_lo_input_files
section2_lo_input_files = section3_lo_input_files
section1_lo_t0s = section2_lo_t0s
section2_lo_t0s = section3_lo_t0s
#otherwise, it's the first step, so all 3 sections have to be read
else:
print 'first section...initialise data collection'
section1_data, section1_lo_input_files, section1_lo_t0s = read_ts_data_from_files(
t0, lo_windowstarts[idx_t0 + 1], lo_t_mins,
lo_files)
section2_data, section2_lo_input_files, section2_lo_t0s = read_ts_data_from_files(
lo_windowstarts[idx_t0 + 1],
lo_windowstarts[idx_t0 + 2], lo_t_mins, lo_files)
section1_ta = np.arange(
len(section1_data)) / samplingrate + t0
section2_ta = np.arange(
len(section2_data
)) / samplingrate + lo_windowstarts[idx_t0 + 1]
if idx_t0 < n_windows - 1:
print 'lll'
section3_data, section3_lo_input_files, section3_lo_t0s = read_ts_data_from_files(
lo_windowstarts[idx_t0 + 2],
lo_windowstarts[idx_t0 + 3], lo_t_mins, lo_files)
else:
print 'jjjj'
#for the last section, there is no lo_windowstarts[idx_t0 +3], so it must be the end of the overall time axis
section3_data, section3_lo_input_files, section3_lo_t0s = read_ts_data_from_files(
lo_windowstarts[idx_t0 + 2],
ta[-1] + 1. / samplingrate, lo_t_mins, lo_files)
section3_ta = np.arange(
len(section3_data)) / samplingrate + lo_windowstarts[
idx_t0 + 2]
data = np.concatenate(
[section1_data, section2_data, section3_data])
timeaxis = np.concatenate(
[section1_ta, section2_ta, section3_ta])
print 'sections ta: ', section1_ta[0], section1_ta[-1], len(
section1_ta), section2_ta[0], section2_ta[-1], len(
section2_ta), section3_ta[0], section3_ta[-1], len(
section3_ta)
#continue
data = np.array(data)
# remove mean and linear trend (precaution, since it should be removed as low frequency
# content anyway)
data = scipy.signal.detrend(data)
#----------------------------
# the actual deconvolution:
corrected_data = MTin.correct_for_instrument_response(
data, samplingrate, responsedata)
#----------------------------
if idx_t0 == 0:
# for the first window, the first section is output as well
startidx = 0
lo_infiles.extend(section1_lo_input_files)
lo_t0s.extend(section1_lo_t0s)
else:
#otherwise, just the central section, so take the start of this middle section
startidx = (
np.abs(timeaxis -
lo_windowstarts[idx_t0 + 1])).argmin()
#collect the respective input filenames and their starting times
for fn in section2_lo_input_files:
if fn not in lo_infiles:
lo_infiles.append(fn)
for t0 in section2_lo_t0s:
if t0 not in lo_t0s:
lo_t0s.append(t0)
if idx_t0 == n_windows - 1:
#for the last window, the last section is added
for fn in section3_lo_input_files:
if fn not in lo_infiles:
lo_infiles.append(fn)
for t0 in section3_lo_t0s:
if t0 not in lo_t0s:
lo_t0s.append(t0)
data2write = corrected_data[startidx:]
timeaxis2write = timeaxis[startidx:]
else:
#for all windows but the last, get the middle section end
endidx = (
np.abs(timeaxis -
lo_windowstarts[idx_t0 + 2])).argmin()
#and cut out the section
data2write = corrected_data[startidx:endidx]
timeaxis2write = timeaxis[startidx:endidx]
#print 'indizes:',len(timeaxis),startidx,len(timeaxis2write),timeaxis2write[0],timeaxis2write[-1]
# at this point, we got
# - the corrected data
# - the sorted list of the involved input files (and their starting times)
#
# write data to the output files, whose structure is following the input file(s):
# maximum time stamp of the current data set (section)
tmax = timeaxis2write[-1]
#initialise the current time to the beginning of the section to be written
t = timeaxis2write[0]
print '\nfiles involved in this section', lo_infiles
while t < tmax:
print t
if file_open == False:
print 'no file open...preparing new one'
# take the first of the input files in the list:
print 'read header of input file {0} '.format(
lo_infiles[0])
header = MTfh.read_ts_header(lo_infiles[0])
ta_tmp = np.arange(float(
header['nsamples'])) / float(
header['samplingrate']) + float(
header['t_min'])
unit = header['unit']
if unit[-6:].lower() != '(true)':
unit += '(true)'
header['unit'] = unit
headerline = MTfh.get_ts_header_string(header)
# output file name: use input file name and append '_true'
inbasename = op.basename(fn)
outbasename = ''.join([
op.splitext(inbasename)[0] + '_true',
op.splitext(inbasename)[1]
])
outfn = op.join(outdir, outbasename)
print 'write header to output file {0} '.format(
outfn)
outF = open(outfn, 'w')
outF.write(headerline)
# if the section exceeds the time axis of the file:
if tmax > ta_tmp[-1]:
print 'data longer than space left in file...storing first part '
#write as many samples to the files as there belong
np.savetxt(
outF,
data2write[:int(float(header['nsamples'])
)])
print 'write data to output file {0} '.format(
outfn)
#close the file
outF.close()
file_open = False
print ' output file closed : {0} '.format(
outfn)
datalength = 0
#drop out the first elements of the lists
dummy = lo_infiles.pop(0)
print 'dropped {0}'.format(dummy)
dummy = lo_t0s.pop(0)
print 'dropped {0}'.format(dummy)
#cut the written part of the data
print 'data cut from {0} to {1}'.format(
len(data2write),
len(data2write[
int(float(header['nsamples'])):]))
data2write = data2write[
int(float(header['nsamples'])):]
timeaxis2write = timeaxis2write[
int(float(header['nsamples'])):]
#define the current time as one sample after the end of the file, which was just closed
t = ta_tmp[-1] + 1. / float(
header['samplingrate'])
print 'current time set to {0}'.format(t)
# and back to the while condition, since there are unwritten data
#if the section is not longer than the time axis of the newly opened file:
else:
print 'data fits into open file : {0}'.format(
outF.name)
#write everything
np.savetxt(outF, data2write)
print 'wrote data to file'
#check, if by chance this is exactly the correct number of samples for this file:
if tmax == ta_tmp[-1]:
#if so, close it
print 'file full....closing...'
outF.close()
file_open = False
datalength = 0
print 'written data to file and closed it: {0}'.format(
outF.name)
dummy = lo_infiles.pop(0)
dummy = lo_t0s.pop(0)
#actually, the infile list should be empty now, since the whole section has been written to a file!!
# otherwise, the section is shorter, so the file (potentially) misses entries
else:
datalength = len(data2write)
print 'file not full...{0} remains open - {1} of max. {2} samples written '.format(
outF.name, datalength,
int(float(header['nsamples'])))
file_open = True
#define the current time as at the end of the time axis of the section, i.e. 'go to next section':
t = tmax
print 'current time set to ', t
#otherwise, a file is already open and the next section has to be appended there:
else:
print 'open file {0} is waiting for data...'.format(
outF.name)
header = MTfh.read_ts_header(lo_infiles[0])
ta_tmp = np.arange(float(
header['nsamples'])) / float(
header['samplingrate']) + float(
header['t_min'])
print tmax, ta_tmp[
-1], '{0} of {1} samples used...{2} waiting\n'.format(
datalength, len(ta_tmp), len(data2write))
#check, if the data exceeds the time axis of the open file:
if tmax > ta_tmp[-1]:
print 'data longer than space left in file...storing first {0} here and cut of the rest ({1}) for further processing...\n'.format(
len(ta_tmp) - datalength,
len(data2write) - len(ta_tmp) + datalength)
#determine the index of the section time axis, which belongs to the last entry of the currently open file - including the last value!
endidx = (np.abs(timeaxis2write -
ta_tmp[-1])).argmin() + 1
#write the respective part of the data to the file and close it then
np.savetxt(outF, data2write[:endidx])
outF.close()
file_open = False
print 'closed full file {0}'.format(outF.name)
#cut out the first bit, which is already written
print 'cutting data to remaining bit ({0}->{1})\n'.format(
len(data2write),
len(data2write) - len(data2write[:endidx]))
data2write = data2write[endidx:]
timeaxis2write = timeaxis2write[endidx:]
# drop the file, which is used and done
print 'dropping name of closed file', lo_infiles[
0]
dummy = lo_infiles.pop(0)
dummy = lo_t0s.pop(0)
#set the current time to the start of the next file
t = ta_tmp[-1] + 1. / float(
header['samplingrate'])
print 'current time set to ', t
#if the section is not longer than the time axis of the open file:
else:
print 'data fits into the open file :', outF.name
#write everything
np.savetxt(outF, data2write)
datalength += len(data2write)
print 'file contains {0} samples - {1} open '.format(
datalength,
int(float(header['nsamples'])) -
datalength)
#check, if by chance this is exactly the correct number of samples for this file:
if tmax == ta_tmp[-1]:
#if so, close it
outF.close()
print 'written data to file {0}'.format(
outfn)
file_open = False
dummy = lo_infiles.pop(0)
dummy = lo_t0s.pop(0)
#actually, the infile list should be empty now, since the whole section has been written to a file!!
# otherwise, the section is shorter, so the file (potentially) misses entries
else:
file_open = True
#define the current time as at the end of the time axis of the section => go to next section:
t = tmax
# after looping over all sections, check, if the last file has been closed:
if file_open == True:
outF.close()
print 'written data to file {0}'.format(outF.name)
def EDL_make_dayfiles(inputdir, sampling, stationname=None, outputdir=None):
"""
Concatenate ascii time series to dayfiles (calendar day, UTC reference).
Data can be within a single directory or a list of directories.
However, the files in the directory(ies) 'inputdir' have to be for
one station only, and named with a 2 character suffix, defining the channel!
If the time series are interrupted/discontinuous at some point, a new file
will be started after that point, where the file index 'idx' is increased by 1.
If no stationname is given, the leading non-datetime characters in the first
filename are used.
Files are named as 'stationname_samplingrate_date_idx.channel'
Stationname, channel, and sampling are written to a header line.
Output data consists of a single column float data array. The data are
stored into one directory. If 'outputdir' is not specified, a subdirectory
'dayfiles' will be created witihn the current working directory.
Note:
Midnight cannot be in the middle of a file, because only file starts are
checked for a new day!!
"""
try:
if type(inputdir) == str:
raise
lo_foldernames = [i for i in inputdir]
except TypeError:
lo_foldernames = [inputdir]
#typical suffixes for EDL output file names
components = ['ex', 'ey', 'bx', 'by', 'bz']
lo_allfiles = []
pattern = '*.[ebEB][xyzXYZ]'
if stationname is not None:
pattern = '*{0}*.[ebEB][xyzXYZ]'.format(stationname.lower())
print '\nSearching for files with pattern: ', pattern
for folder in lo_foldernames:
wd = op.abspath(op.realpath(folder))
if not op.isdir(wd):
#print 'Directory not existing: %s' % (wd)
lo_foldernames.remove(wd)
continue
lo_dirfiles = [
op.abspath(op.join(wd, i)) for i in os.listdir(wd)
if fnmatch.fnmatch(i.lower(), pattern.lower()) is True
]
lo_allfiles.extend(lo_dirfiles)
#check, if list of files is empty
if len(lo_allfiles) == 0:
if stationname is not None:
raise MTex.MTpyError_inputarguments('Directory(ies) do(es) not contain'\
' files to combine for station {0}:\n {1}'.format(stationname, inputdir))
raise MTex.MTpyError_inputarguments('Directory does not contain files'\
' to combine:\n {0}'.format(inputdir))
#define subfolder for storing dayfiles
outpath = op.join(os.curdir, 'dayfiles')
if outputdir is not None:
try:
outpath = op.abspath(op.join(os.curdir, outputdir))
if not op.exists(outpath):
try:
os.makedirs(outpath)
except:
raise
if not os.access(outpath, os.W_OK):
raise
except:
print 'Cannot generate writable output directory {0} - using'\
' generic location "dayfiles" instead'.format(outpath)
outpath = op.join(wd, 'dayfiles')
pass
#generate subfolder, if not existing
if not op.exists(outpath):
try:
os.makedirs(outpath)
except:
MTex.MTpyError_inputarguments('Cannot generate output'\
' directory {0} '.format(outpath))
#outer loop over all components
for comp in components:
#make list of files for the current component
lo_files = np.array(
[op.join(wd, i) for i in lo_allfiles if (i.lower()[-2:] == comp)])
#make list of starting times for the respective files
lo_starttimes = np.array(
[EDL_get_starttime_fromfilename(f) for f in lo_files])
#sort the files by their starting times
idx_chronologic = np.argsort(lo_starttimes)
#obtain sorted lists of files and starting times
lo_sorted_files = list(lo_files[idx_chronologic])
lo_sorted_starttimes = list(lo_starttimes[idx_chronologic])
idx = 0
while idx < len(lo_sorted_starttimes):
try:
val = lo_sorted_starttimes[idx]
except:
break
if val is None:
dummy = lo_sorted_files.pop(idx)
dummy = lo_sorted_starttimes.pop(idx)
else:
idx += 1
#set stationname, either from arguments or from filename
if stationname is None:
stationname = EDL_get_stationname_fromfilename(
lo_sorted_files[0]).upper()
#set counting variables - needed for handling of consecutive files
sameday = 0
fileopen = 0
incomplete = 0
fileindex = 0
#allocate a data array to fill
# this is more memory efficient than extending lists!!
#cater for potential rounding errors:
if sampling < 1:
max_n_data = 86400 * (int(1. / sampling) + 1)
else:
max_n_data = int(86400. / sampling) + 1
day_data = np.zeros(max_n_data, 'int')
#loop over all (sorted) files for the current component
for idx_f, f in enumerate(lo_sorted_files):
try:
print 'Reading file %s' % (f)
#starting time of current file
file_start_time = lo_sorted_starttimes[idx_f]
#get tuple with the starting time of the current file
file_start = time.gmtime(file_start_time)
#read in raw data
data_in = []
Fin = open(f)
for line in Fin: #.readlines():
# try:
data_in.append(int(float(line.strip())))
# except:
# pass
data_in = np.array(data_in)
Fin.close()
#data_in = np.loadtxt(f)
except:
print 'WARNING - could not read file - skipping...'
continue
no_samples = len(data_in)
tmp_file_time_axis = np.arange(
no_samples) * sampling + file_start_time
#file_time_axis = (np.arange(no_samples)*sampling +
# file_start_time).tolist()
#time of the last sample + 1x sampling-interval
#file_end_time = file_time_axis[-1] + sampling
file_end_time = tmp_file_time_axis[-1] + sampling
#set the time as starting time for output file, if no output file is open already
if fileopen == 0:
outfile_starttime = file_start_time
#outfile_timeaxis = file_time_axis
old_time_axis = tmp_file_time_axis[:]
arrayindex = 0
#if it's a single column of data
if np.size(data_in.shape) == 1:
day_data[arrayindex:arrayindex + len(data_in)] = data_in
#outfile_data = data_in.tolist()
#otherwise assuming that the first column is time, so just take the second one
else:
day_data[arrayindex:arrayindex + len(data_in)] = data_in[:,
1]
#outfile_data = data_in[:,1].tolist()
#jump with index to current point on time axis
arrayindex += len(data_in)
outfile_endtime = file_end_time
file_date = '{0}{1:02}{2:02}'.format(file_start[0],
file_start[1],
file_start[2])
#define output filename
new_fn = '{0}_1day_{1}_{2}.{3}'.format(stationname, file_date,
fileindex, comp)
new_file = op.abspath(op.join(outpath, new_fn))
#open output file
F = open(new_file, 'w')
fileopen = 1
else:
#check, if the new file ends earlier than data in buffer.
#if yes, just skip this file:
if file_end_time < outfile_endtime:
continue
#if current file starts earlier than the endtime of data in buffer then delete ambiguous parts of the buffer:
#elif (outfile_timeaxis[-1] - file_start_time) > epsilon:
elif (outfile_endtime - file_start_time) > epsilon:
#find point on the outfile time axis for the beginning of current file:
overlap_idx = arrayindex - int(
(outfile_endtime - file_start_time) / sampling)
#set the array index back
arrayindex = overlap_idx
#re-define outfile time axis and data
# outfile_timeaxis = np.delete(outfile_timeaxis,
# np.arange(len(outfile_timeaxis) -
# overlap_idx) +
# overlap_idx).tolist()
# outfile_data = np.delete(outfile_data,
# np.arange(len(outfile_data) -
# overlap_idx) +
# overlap_idx).tolist()
#old_time_axis = tmp_file_time_axis[:]
#append current file's time axis
#outfile_timeaxis.extend(file_time_axis)
#append current data
#if it's a single column of data
if np.size(data_in.shape) == 1:
day_data[arrayindex:arrayindex + len(data_in)] = data_in
#outfile_data.extend(data_in.tolist())
#otherwise assuming that the first column is time, so just take the second one
else:
day_data[arrayindex:arrayindex + len(data_in)] = data_in[:,
1]
#outfile_data.extend(data_in[:,1].tolist())
arrayindex += len(data_in)
print len(data_in), arrayindex
arrayindex += len(data_in)
outfile_endtime = (arrayindex +
1) * sampling + outfile_starttime
#-----------
#check, if there is a next file:
try:
next_file_start_time = lo_sorted_starttimes[idx_f + 1]
except:
incomplete = 1
#if there is a next file,
# - check, if it's the same day
# - check, if it continues at the end of the current one:
if incomplete == 0:
next_file_start_time = lo_sorted_starttimes[idx_f + 1]
next_file_start = time.gmtime(next_file_start_time)
if next_file_start[2] == file_start[2]:
#print 'sameday',file_start[:]
sameday = 1
else:
incomplete = 1
sameday = 0
fileindex = 0
#print '\t NOT sameday', fileindex
if next_file_start_time - file_end_time > epsilon:
incomplete = 1
if incomplete == 1 and sameday == 1:
fileindex += 1
#check, if the file has to be closed and written now
if incomplete == 1:
#define header info
if outfile_starttime % 1 == 0:
outfile_starttime = int(outfile_starttime)
headerline = '# {0} {1} {2:.1f} {3} {4} \n'.format(
stationname, comp.lower(), 1. / sampling,
outfile_starttime, arrayindex)
else:
headerline = '# {0} {1} {2:.1f} {3:f} {4} \n'.format(
stationname, comp.lower(), 1. / sampling,
outfile_starttime, arrayindex)
F.write(headerline)
#outfile_array = np.zeros((len(outfile_timeaxis),2))
#outfile_array[:,0] = outfile_timeaxis
#outfile_array[:,1] = outfile_data
for i in range(arrayindex):
F.write('{0}\n'.format(int(day_data[i])))
#outstring = '\n'.join(['{0:d}'.format(i) for i in day_data[:arrayindex]])
#F.write(outstring)
#np.savetxt(F,day_data[:arrayindex],fmt='%d')
#np.savetxt(F, np.array(outfile_data))
arrayindex = 0
F.close()
print '\t wrote file %s' % (new_file)
fileopen = 0
incomplete = 0
def EDL_make_Nhour_files(n_hours,
inputdir,
sampling,
stationname=None,
outputdir=None):
"""
See 'EDL_make_dayfiles' for description and syntax.
Only difference: output files are blocks of (max) N hours, starting to count
at midnight (00:00h) each day.
Conditions:
1. 24%%N = 0
2. input data files start on the hour marks
Not working yet!!
"""
#print '\n\tnot working yet - code under development !!\n'
#return
try:
if 24 % n_hours != 0:
raise
except:
sys.exit('ERROR - File block length must be on of: 1,2,3,4,6,8,12 \n')
n_hours = int(n_hours)
#list of starting hours for the data blocks:
lo_hours = [int(i) for i in np.arange(int(24 / n_hours)) * n_hours]
no_blocks = len(lo_hours)
#build a list that contains the respective groups of starting hours belonging
# to the same block
lo_blockhours = []
counter = 0
dummylist = []
for i in range(24):
dummylist.append(i)
counter += 1
if counter == n_hours:
lo_blockhours.append(dummylist)
dummylist = []
counter = 0
# most of the following code is redundant/taken from the
# 'EDL_make_dayfiles' function
# This can be cleaned up later
try:
if type(inputdir) == str:
raise
lo_foldernames = [i for i in inputdir]
except TypeError:
lo_foldernames = [inputdir]
#typical suffixes for EDL output file names
components = ['ex', 'ey', 'bx', 'by', 'bz']
lo_allfiles = []
pattern = '*.[ebEB][xyzXYZ]'
if stationname is not None:
pattern = '*{0}*.[ebEB][xyzXYZ]'.format(stationname.lower())
print '\nSearching for files with pattern: ', pattern
for folder in lo_foldernames:
wd = op.abspath(op.realpath(folder))
if not op.isdir(wd):
#print 'Directory not existing: %s' % (wd)
lo_foldernames.remove(wd)
continue
lo_dirfiles = [
op.abspath(op.join(wd, i)) for i in os.listdir(wd)
if fnmatch.fnmatch(i.lower(), pattern.lower()) is True
]
lo_allfiles.extend(lo_dirfiles)
#check, if list of files is empty
if len(lo_allfiles) == 0:
if stationname is not None:
raise MTex.MTpyError_inputarguments('Directory(ies) do(es) not contain'\
' files to combine for station {0}:\n {1}'.format(stationname, inputdir))
raise MTex.MTpyError_inputarguments('Directory does not contain files'\
' to combine:\n {0}'.format(inputdir))
#define subfolder for storing dayfiles
outpath = op.join(os.curdir, '{0}hourfiles'.format(int(n_hours)))
if outputdir is not None:
try:
outpath = op.abspath(op.join(os.curdir, outputdir))
if not op.exists(outpath):
try:
os.makedirs(outpath)
except:
raise
if not os.access(outpath, os.W_OK):
raise
except:
print 'Cannot generate writable output directory {0} - using'\
' generic location "dayfiles" instead'.format(outpath)
outpath = op.join(wd, '{0}hourfiles'.format(int(n_hours)))
pass
#generate subfolder, if not existing
if not op.exists(outpath):
try:
os.makedirs(outpath)
except:
MTex.MTpyError_inputarguments('Cannot generate output'\
' directory {0} '.format(outpath))
#outer loop over all components
for comp in components:
#make list of files for the current component
lo_files = np.array(
[op.join(wd, i) for i in lo_allfiles if (i.lower()[-2:] == comp)])
#make list of starting times for the respective files
lo_starttimes = np.array(
[EDL_get_starttime_fromfilename(f) for f in lo_files])
#sort the files by their starting times
idx_chronologic = np.argsort(lo_starttimes)
#obtain sorted lists of files and starting times
lo_sorted_files = list(lo_files[idx_chronologic])
lo_sorted_starttimes = list(lo_starttimes[idx_chronologic])
idx = 0
while idx < len(lo_sorted_starttimes):
try:
val = lo_sorted_starttimes[idx]
except:
break
if val is None:
dummy = lo_sorted_files.pop(idx)
dummy = lo_sorted_starttimes.pop(idx)
else:
idx += 1
#set stationname, either from arguments or from filename
if stationname is None:
stationname = EDL_get_stationname_fromfilename(
lo_sorted_files[0]).upper()
#set counting variables - needed for handling of consecutive files
#flags, checking, if data has to written to file
sameday = True
sameblock = True
fileopen = False
complete = False
#numerical index of files for same combination of date and hour
fileindex = 0
#current file's daily block number
blockindex = 0
#allocate a data array to fill
# this is more memory efficient than extending lists!!
#cater for potential rounding errors:
if sampling < 1:
max_n_data = 3600 * int(n_hours) * (int(1. / sampling) + 1)
else:
max_n_data = int(3600. * int(n_hours) / sampling) + 1
block_data = np.zeros(max_n_data, 'int')
#loop over all (sorted) files for the current component
for idx_f, f in enumerate(lo_sorted_files):
try:
print 'Reading file %s' % (f)
#starting time of current file
file_start_time = lo_sorted_starttimes[idx_f]
#get tuple with the starting time of the current file
file_start = time.gmtime(file_start_time)
#read in raw data
data_in = []
Fin = open(f)
for line in Fin: #.readlines():
# try:
data_in.append(int(float(line.strip())))
# except:
# pass
data_in = np.array(data_in)
Fin.close()
#data_in = np.loadtxt(f)
except:
print 'WARNING - could not read file - skipping...'
continue
no_samples = len(data_in)
#time axis of the file read in :
tmp_file_time_axis = np.arange(
no_samples) * sampling + file_start_time
#end: time of the last sample + 1x sampling-interval
file_end_time = tmp_file_time_axis[-1] + sampling
#set the current file's starting time as starting time for output file,
#if no output file is open already
if fileopen is False:
#starttime of output file
outfile_starttime = file_start_time
#output data sample index:
arrayindex = 0
#if it's a single column of data
if np.size(data_in.shape) == 1:
block_data[:len(data_in)] = data_in
#otherwise assuming that the first column is time, so just take the second one
else:
block_data[:len(data_in)] = data_in[:, 1]
#jump with index to current point on time axis
arrayindex += len(data_in)
#current (virtual) end time of open file
outfile_endtime = file_end_time
#find the date code of the outputfile
file_date = '{0}{1:02}{2:02}'.format(file_start[0],
file_start[1],
file_start[2])
#...aaaand the hour code as well
data_hour = file_start[3]
#determine, which of the daily data blocks we are currently
#processing/writing
blockindex = no_blocks - 1
file_hour = lo_hours[-1]
for t in range(len(lo_hours) - 1):
if lo_hours[t + 1] > data_hour:
blockindex = t
file_hour = lo_hours[blockindex]
break
#define output filename
new_fn = '{0}_{5}hours_{1}_{2:02d}_{3}.{4}'.format(
stationname, file_date, file_hour, fileindex, comp,
n_hours)
#absolute filename:
new_file = op.abspath(op.join(outpath, new_fn))
#open output file
F = open(new_file, 'w')
#set flag for further loop steps
fileopen = True
else:
#check, if the new file ends earlier than data in buffer.
#if yes, just skip this file:
if file_end_time < outfile_endtime:
continue
#if current file starts earlier than the endtime of data in buffer, but extends the time span
#then delete ambiguous parts of the buffer:
elif (outfile_endtime - file_start_time) > epsilon:
#find point on the outfile time axis for the beginning of current file:
overlap_idx = arrayindex - int(
(outfile_endtime - file_start_time) / sampling)
#set the array index back to the appropriate value corresponding to the
#start of the new file
arrayindex = overlap_idx
#append current data
#if it's a single column of data
if np.size(data_in.shape) == 1:
block_data[arrayindex:arrayindex + len(data_in)] = data_in
#outfile_data.extend(data_in.tolist())
#otherwise assuming that the first column is time, so just take the second one
else:
block_data[arrayindex:arrayindex +
len(data_in)] = data_in[:, 1]
#outfile_data.extend(data_in[:,1].tolist())
#update position in time
arrayindex += len(data_in)
#update (virtual) end of outfile data
outfile_endtime = (arrayindex +
1) * sampling + outfile_starttime
#-----------
# current file has been read in, data in buffer have been updated
# now check, if it has to be written to file....
#check, if there is a next file at all:
try:
next_file_start_time = lo_sorted_starttimes[idx_f + 1]
except:
complete = True
#if there is a next file,
# - check, if it's the same day
# - check, if it' the same block
# - check, if it continues at the end of the current one:
if complete is False:
next_file_start_time = lo_sorted_starttimes[idx_f + 1]
next_file_start = time.gmtime(next_file_start_time)
nextfile_hour = next_file_start[3]
nextfile_blockindex = no_blocks - 1
for t in range(len(lo_hours) - 1):
if lo_hours[t + 1] > nextfile_hour:
nextfile_blockindex = t
break
nextfile_day = '{0}{1:02}{2:02}'.format(
next_file_start[0], next_file_start[1], next_file_start[2])
if nextfile_day != file_date:
complete = True
sameday = False
if nextfile_blockindex != blockindex:
complete = True
sameblock = False
if complete is False:
if next_file_start_time - file_end_time > epsilon:
complete = True
sameblock = True
sameday = True
#ipdb.set_trace()
#check, if the file has to be closed and written now
if complete is True:
#define header info
if outfile_starttime % 1 == 0:
outfile_starttime = int(outfile_starttime)
headerline = '# {0} {1} {2:.1f} {3} {4} \n'.format(
stationname, comp.lower(), 1. / sampling,
outfile_starttime, arrayindex)
else:
headerline = '# {0} {1} {2:.1f} {3:f} {4} \n'.format(
stationname, comp.lower(), 1. / sampling,
outfile_starttime, arrayindex)
F.write(headerline)
#outfile_array = np.zeros((len(outfile_timeaxis),2))
#outfile_array[:,0] = outfile_timeaxis
#outfile_array[:,1] = outfile_data
for i in range(arrayindex):
F.write('{0}\n'.format(int(block_data[i])))
#outstring = '\n'.join(['{0:d}'.format(i) for i in day_data[:arrayindex]])
#F.write(outstring)
#np.savetxt(F,day_data[:arrayindex],fmt='%d')
#np.savetxt(F, np.array(outfile_data))
arrayindex = 0
F.close()
print '\t wrote file %s' % (new_file)
fileopen = False
complete = False
#blockindex = (blockindex+1)%no_blocks
if sameday is True and sameblock is True:
fileindex += 1
else:
fileindex = 0
def reorient_files(lo_files, configfile, lo_stations=None, outdir=None):
#read config file
try:
config_dict = MTcf.read_survey_configfile(configfile)
except:
raise MTex.MTpyError_config_file('Config file cannot be read:'
' {0}'.format(configfile))
if lo_stations is not None:
try:
if type(lo_stations) == str:
raise
#check, if it's iterable:
dummy = [i for i in lo_stations]
except:
raise MTex.MTpyError_inputarguments('ERROR - "lo_stations"'
' argument must be iterable!')
print '\t re-orienting data for collection of stations:\n{0}'.format(
lo_stations)
#Do not require list of headers as input, as this function can be called directly rather than from a 'calibratefiles.py'-like script - so the list not necessarily exists in beforehand -
#collect header lines of files in list
lo_headers = []
lo_stationnames = []
for file_idx, filename in enumerate(lo_files):
header = read_ts_header(filename)
station = header['station']
if station.upper() not in [i.upper() for i in lo_stations]:
#TODO: check, if this causes problems with the indices for the current loop:
lo_files.remove(filename)
continue
lo_headers.append(header)
lo_stationnames.append(station.upper())
if len(lo_headers) == 0:
if lo_stations is not None:
print 'ERROR - No files with header lines found for station(s)'\
' {0}'.format(lo_stations)
else:
print 'ERROR - No files with header lines found'
return 1
lo_stationnames = list(set(lo_stationnames))
# set up output directory
ori_outdir = op.abspath(op.join(os.curdir, 'reoriented'))
if outdir is not None:
try:
ori_outdir = op.abspath(op.join(os.curdir, outdir))
if not op.isdir(ori_outdir):
os.makedirs(ori_outdir)
except:
print 'Output directory cannot be generated: {0} - using generic'\
' location'.format(ori_outdir)
ori_outdir = op.abspath(op.join(os.curdir, 'reoriented'))
try:
if not op.isdir(ori_outdir):
os.makedirs(ori_outdir)
except:
#this only comes up, if the generic location cannot be generated
raise MTex.MTpyError_inputarguments(
'Generic directory cannot be'
' generated: {0}'.format(ori_outdir))
#----------------------
#start re-orientation
#present: list of all files, list of all headers, list of all stations
for sta_idx, sta in enumerate(lo_stationnames):
#print sta
try:
stationconfig = config_dict[sta]
except:
print 'Warning - No config file entry for station {0} -'\
' no processing possible'.format(sta)
continue
declination = float(stationconfig.get('declination', 0.))
for sensor in ['e', 'b']:
#TODO:
# reduce this function to the re-orientation of files that have the same length for X and Y.
#Do the puzzlling for varying lengths later!!
for idx_h_x, header_x in enumerate(lo_headers):
#looking for one specific station
if not header_x['station'].upper() == sta.upper():
continue
#looking for the specific sensor type
if not header_x['channel'].lower()[0] == sensor:
continue
#looking for the X channel (the to-be-North)
if not header_x['channel'].lower()[1] == 'x':
continue
x_file = lo_files[idx_h_x]
x_header_string = get_ts_header_string(header_x)
t0 = float(header_x['t_min'])
#print t0
#now look for the respective y-file and possible z-file - unfortunately by another loop over all headers:
y_file = None
z_file = None
for idx_h_y, header_y in enumerate(lo_headers):
if (header_y['station'].upper() == sta.upper()) and \
(header_y['channel'].lower()[0] == sensor) and \
(float(header_y['t_min']) == float(header_x['t_min'] ) ):
if (header_y['channel'].lower()[1] == 'y'):
y_file = lo_files[idx_h_y]
y_header_string = get_ts_header_string(header_y)
elif (header_y['channel'].lower()[1] == 'z'):
z_file = lo_files[idx_h_y]
else:
continue
if y_file == None:
continue
x_outfn = op.abspath(op.join(ori_outdir, op.basename(x_file)))
y_outfn = op.abspath(op.join(ori_outdir, op.basename(y_file)))
if z_file is not None:
z_outfn = op.abspath(
op.join(ori_outdir, op.basename(z_file)))
xdata = np.loadtxt(x_file)
ydata = np.loadtxt(y_file)
#declination is positive, if magnetic North is east of true North.
# the measured angles are w.r.t. magnetic North, so the given
# azimuths do not include the declination
#-> thus the declination value is added to azimuths
if sensor == 'e':
xangle = float(stationconfig.get('e_xaxis_azimuth',
0.)) + declination
yangle = float(stationconfig.get('e_yaxis_azimuth',
90.)) + declination
else:
xangle = float(stationconfig.get('b_xaxis_azimuth',
0.)) + declination
yangle = float(stationconfig.get('b_yaxis_azimuth',
90.)) + declination
newx, newy = MTcc.reorient_data2D(xdata,
ydata,
x_sensor_angle=xangle,
y_sensor_angle=yangle)
#print xdata.shape, ydata.shape, newx.shape, newy.shape
#continue
outFx = open(x_outfn, 'w')
outFx.write(x_header_string)
np.savetxt(outFx, newx)
outFx.close()
outFy = open(y_outfn, 'w')
outFy.write(y_header_string)
np.savetxt(outFy, newy)
outFy.close()
written_files = [x_outfn, y_outfn]
if z_file is not None:
shutil.copyfile(z_file, z_outfn)
written_files.append(z_outfn)
print '\tSuccessfullly written files {0}'.format(written_files)
return 0
def calibrate_file(filename, outdir, instrument, instrument_amplification,
logger, gain, dipole, stationname, channel, latitude,
longitude, elevation, offset = 0 ):
"""
Calibrate data from one given file and store the output to another file.
If the channel is not given explicitly, it's taken from the filename suffix.
E field values will be present in microvolt/meter
B fields are given in nanotesla
input:
- data file name
- foldername for saving the output
- instrument type
- instrument amplification factor
- data logger type
- logger gain factor
- station name
- channel
"""
time_axis = None
if not instrument.lower() in list_of_instruments:
raise MTex.MTpyError_inputarguments('instrument type not known')
if not logger.lower() in list_of_loggers:
raise MTex.MTpyError_inputarguments('data logger type not known')
if not op.isfile(filename):
raise MTex.MTpyError_inputarguments('data file not existing')
infile_base = op.basename(filename)
try:
data_in = np.loadtxt(filename)
except:
raise MTex.MTpyError_inputarguments('cannot read data file')
data_out = copy.copy(data_in)
#read in first line of input file, checking, if header line exists
FH = open(filename,'r')
firstline = FH.readline().strip()
FH.close()
if np.size(data_in.shape) > 1:
if data_in.shape[1] > 1:
#at least 2 columns - assume, first is time, second data - ignore, if there are more
time_axis = data_in[:,0]
data_in = data_in[:,1]
if not op.isdir(outdir):
try:
os.makedirs(outdir)
except:
raise MTex.MTpyError_inputarguments('output directory is not '
'existing and cannot be generated')
if channel == None:
channel = filename[-2:].lower()
if not channel in list_of_channels:
raise MTex.MTpyError_inputarguments('wrong channel specification')
field = channel[0]
#print 'channel:...........',channel, field
#print 'read file',filename ,'wrote file...'
#return
#separate way for B and E fields here:
if field == 'e':
if dipole <= 1:
print 'Check dipole length value ! - It is highly improbable to '\
'have a 1 meter dipole!!'
answer = raw_input('\t\tContinue anyway? [y/N] \n')
if not answer[0].lower() == 'y':
sys.exit('Calibration process interrupted by user input!')
instrument = 'electrodes'
logger = logger.lower()
if logger == 'elogger':
if not type(gain) in [float, int]:#list_of_elogger_gain_factors:
raise MTex.MTpyError_inputarguments('invalid gain for elogger:'
' {0}'.format(gain))
#instrument_amplification = dict_of_efield_amplification[logger]
outfile_data = elogger_e_field(data_in, gain, dipole,
instrument_amplification)
elif logger == 'edl':
if not type(gain) in [float, int, str]:
raise MTex.MTpyError_inputarguments('invalid gain for EDL: '
'{0}'.format(gain))
#instrument_amplification = dict_of_efield_amplification[logger]
if type(gain) == str:
EDLgain = dict_of_EDL_gain_factors[gain]
else:
EDLgain = float(gain)
outfile_data = EDL_e_field(data_in, EDLgain, dipole,
instrument_amplification)
dataunit ='microvoltpermeter'
#B-field part
elif field == 'b':
instrument = instrument.lower()
if not instrument in list_of_bfield_instruments:
raise MTex.MTpyError_inputarguments('invalid instrument for B-'
'field measurements')
logger = logger.lower()
if not logger in list_of_bfield_loggers:
raise MTex.MTpyError_inputarguments('invalid logger for B-field'
' measurements')
#instrument_amplification = 1.
#calibration_factor = dict_of_calibration_factors_volt2nanotesla[instrument]
if logger == 'edl':
if not type(gain) in [float,int,str]:
raise MTex.MTpyError_inputarguments('invalid gain: '
'{0}'.format(gain))
if type(gain) == str:
EDLgain = dict_of_EDL_gain_factors[gain]
else:
EDLgain = float(gain)
if instrument == 'fluxgate' and channel == 'bz':
instrument_amplification *= dict_of_bz_instrument_amplification[logger]
outfile_data = EDL_b_field(data_in, EDLgain, instrument,
instrument_amplification)
dataunit = 'nanotesla'
newbasename = '{0}_{1}.{2}'.format(op.splitext(infile_base)[0], dataunit,
infile_base.split('.')[-1].lower())
#set up output file
outfile = op.join(outdir, newbasename)
additional_header_info = ' {0} {1:02.5f} {2:03.5f} {3:.1f} \n'.format(
dataunit, latitude, longitude, elevation)
if firstline[0][0] == '#':
newfirstline = firstline + additional_header_info
else:
newfirstline = '# {0} {1} {2}'.format(stationname, channel,
additional_header_info)
if time_axis != None:
data_out[:,1] = outfile_data
else:
data_out = outfile_data
Fout = open(outfile,'w')
Fout.write(newfirstline)
np.savetxt(Fout,data_out,fmt='%.8e')
Fout.close()
print 'read file',filename ,' -> wrote file %s'%(outfile)
def main():
if len(sys.argv) < 2:
sys.exit(
'\n\tNeed at least 1 argument:\n\n <path to files>\n[optional:'
'<output dir>] \n')
outdir = None
if len(sys.argv) > 2:
outdir = sys.argv[2]
# if len(sys.argv) > 3:
# network = sys.argv[3]
# if len(sys.argv) > 4:
# location = sys.argv[4]
pathname_raw = sys.argv[1]
#we need relative paths here!!!
indir = pathname_raw #op.abspath(op.realpath(pathname_raw))
if not op.isdir(indir):
raise MTex.MTpyError_inputarguments(
'Data file(s) path not existing: {0}'.format(indir))
#define output directory for storing miniSeed files
#outpath = op.join(os.curdir,'miniSeed')
if outdir is not None:
try:
outpath = op.abspath(op.join(os.curdir, outdir))
if not op.exists(outpath):
try:
os.makedirs(outpath)
except:
raise
if not os.access(outpath, os.W_OK):
raise
except:
print 'Cannot generate writable output directory {0} - using generic'\
' location "ascii" instead'.format(outpath)
outdir = None
if outdir is None:
outpath = op.join(os.curdir, 'ascii')
try:
if not op.exists(outpath):
try:
os.makedirs(outpath)
except:
raise
if not os.access(outpath, os.W_OK):
raise
except:
sys.exit('Error ! - Cannot generate writable output directory '
'"ascii" - abort...')
outdir = op.abspath(outpath)
lo_dirs = []
for i, j, k in os.walk(indir):
lofolders = [op.join(i, f) for f in j]
lo_dirs.extend(lofolders)
lo_dirs.append(indir)
pathname = list(set(lo_dirs))
if len(pathname) == 0:
pathname = [indir]
lo_indirs = pathname
lo_outdirs = []
#'pathname' is a list of relative pathnames. to be reconstructed under the given 'outdir'
try:
for i in lo_indirs:
outpath = op.abspath(op.join(outdir, i))
if not op.isdir(outpath):
os.makedirs(outpath)
lo_outdirs.append(outpath)
except:
raise MTex.MTpyError_inputarguments(
'ERROR - Cannot set up output directory {0}'.format(outpath))
for idx_ipath, inpath in enumerate(lo_indirs):
lo_infiles = [
i for i in os.listdir(inpath)
if op.isfile(op.abspath(op.join(inpath, i)))
]
lo_outfiles = [
op.abspath(op.join(lo_outdirs[idx_ipath], i)) for i in lo_infiles
]
lo_infiles = [op.abspath(op.join(inpath, i)) for i in lo_infiles]
for idx_fn, fn in enumerate(lo_infiles):
print 'reading file {0}'.format(fn)
try:
outfn = MTms.convertfile_miniseed2ts(fn, lo_outfiles[idx_fn])
print 'wrote file(s) {0}'.format(outfn)
except:
print 'Warning - file {0} is not in valid miniseed format!!!'.format(
fn)
continue
