def read_data_header(fn_raw):
"""
Deprecated!!!
USE
read_ts_header
INSTEAD
Read the header line of MTpy TS data files.
input
-----
MTpy TS data file name
output
-------
list of header elements:
stationname, channel, sampling rate, starttime first sample,
starttime last sample, unit, lat, lon, elevation
"""
fn = op.abspath(op.realpath(fn_raw))
if not op.isfile(fn):
raise MTex.MTpyError_inputarguments('Not a file:%s' % fn)
try:
F = open(fn, 'r')
except:
raise MTex.MTpyError_inputarguments('File not readable:%s' % fn)
firstline = F.readline().strip().split()
if not firstline[0][0] == '#':
raise MTex.MTpyError_ts_data('Time series data file does '
'not have a proper header:%s' % fn)
F.close()
header_list = []
idx_header = 0
if len(firstline[0]) > 1:
header_list.append(firstline[0][1:].upper())
else:
header_list.append(firstline[1].upper())
idx_header += 1
header_list.append(firstline[idx_header + 1].lower())
header_list.append(float(firstline[idx_header + 2]))
header_list.append(float(firstline[idx_header + 3]))
header_list.append(int(float(firstline[idx_header + 4])))
header_list.append(firstline[idx_header + 5].lower())
header_list.append(float(firstline[idx_header + 6]))
header_list.append(float(firstline[idx_header + 7]))
header_list.append(float(firstline[idx_header + 8]))
return header_list
def read_ts_header(tsfile):
""" Read in the header line from MTpy timeseries data files.
Return header as dictionary. Return empty dict,
if no header line was found.
"""
header_dict = {}
tsfile = op.abspath(tsfile)
if not op.isfile(tsfile):
raise MTex.MTpyError_inputarguments('Error - '
'input file not existing: {0}'.format(tsfile))
try:
with open(tsfile,'r') as F:
firstline =''
#ignoring empty lines or lines with just the '#' character in it
while len(firstline) == 0:
firstline = F.readline().strip()
if firstline == '#':
firstline = ''
if firstline[0] != '#':
raise
except:
raise MTex.MTpyError_ts_data('No header line found -'
' check file: {0}'.format(tsfile))
firstline = firstline.replace('#','')
headerlist = firstline.split()
for i in range(len(headerlist)):
header_dict[lo_headerelements[i]] = headerlist[i]
#old header had tmax instead of n_samples:
if ((i == 4) and float(headerlist[4])%1 != 0
and float(headerlist[i]) > float(headerlist[i-1])):
header_dict[lo_headerelements[i]] = int(
(float(headerlist[i]) - float(headerlist[i-1])
)*float(headerlist[i-2]) )+1
headerlements = ['samplingrate','t_min','nsamples','lat','lon','elev']
for h in headerlements:
try:
header_dict[h] = float(header_dict[h])
except:
pass
try:
if header_dict[h]%1==0:
header_dict[h] = int(header_dict[h])
except:
pass
return header_dict
def _data_instrument_consistency_check(data, field, dipole_length, instrument,
amplification, logger, gain):
"""
Check, if the input values given for the calibration make any sense at all.
"""
if len(data) == 0 :
raise MTex.MTpyError_ts_data( 'no data provided for calibration' )
if not field.lower() in ['e','b']:
raise MTex.MTpyError_inputarguments( 'Field must be E or B' )
# if float(dipole_length) <= 0:
# raise MTpyError_inputarguments( 'Dipole length must be positive' )
if float(amplification) <= 0:
raise MTex.MTpyError_inputarguments('Amplification factor must be positive')
if float(gain) <= 0:
raise MTex.MTpyError_inputarguments('Instrument gain must be positive')
try:
if not logger.lower() in list_of_loggers:
raise
except:
raise MTex.MTpyError_inputarguments( 'wrong choice of logger')
try:
if not instrument.lower() in list_of_instruments:
raise
except:
raise MTex.MTpyError_inputarguments( 'wrong choice of instrument')
if field.lower == 'b':
if logger.lower() == 'elogger':
raise MTex.MTpyError_inputarguments( 'wrong choice of logger')
if instrument.lower() == 'electrodes':
raise MTex.MTpyError_inputarguments( 'wrong choice of instrument')
if not float(dipole_length) == 1:
raise MTex.MTpyError_inputarguments( 'Dipole length must be "1" for'
' B-field calibration')
if field.lower == 'e':
if not instrument.lower() == 'electrodes':
raise MTex.MTpyError_inputarguments( 'wrong choice of instrument')
def correct_for_instrument_response(data, samplingrate, responsedata):
"""Correct input time series for instrument response.
Instr.Resp. is given as 3 column array: frequency, real part, imaginary part
The given section is demeaned, window tapered, zero padded, (potentially bandpassed with the extreme frequencies of the response frequency axis), FFT-ed, "deconvolved" (straight division by the array values or interpolated values inbetween - in frequency domain), re-transformed, mean-re-added and returned.
"""
datamean = np.mean(data)
data -= datamean
freqmin = responsedata[0, 0]
freqmax = responsedata[-1, 0]
#print freqmin,freqmax
N = len(data)
if N < 1:
raise MTex.MTpyError_ts_data(
'Error - Length of TS to correct is zero!')
#use double sided cosine taper function
window = MTfi.tukey(N, 0.2)
tapered_data = data * window
#check, if length is directly equal to power of 2:
if np.log2(N) % 1 == 0:
next2power = int(np.log2(N))
else:
next2power = int(np.log2(N)) + 1
#zero pad data for significantly faster fft - NO, Numpy does not do that automatically
padded_data = np.zeros((2**next2power))
padded_data[:len(tapered_data)] = tapered_data
#bandpass data for excluding all frequencies that are not covered by the known instrument response
bp_data = padded_data #butter_bandpass_filter(padded_data, freqmin, freqmax, samplingrate)
#get the spectrum of the data
data_spectrum = np.fft.rfft(bp_data)
data_freqs = np.fft.fftfreq(len(bp_data), 1. / samplingrate)
#and the same for the instrument
instr_spectrum = responsedata[:, 1] + np.complex(0, 1) * responsedata[:, 2]
instr_freqs = responsedata[:, 0]
lo_mags = []
lo_freqs = []
#return data_freqs,data_spectrum,instr_freqs,instr_spectrum
#now correct for all frequencies on the data_freqs-axis:
corrected_spectrum = np.zeros((len(data_spectrum)), 'complex')
for i in range(len(data_spectrum)):
freq = data_freqs[i]
spec = data_spectrum[i]
#this is effectively a boxcar window - maybe to be replaced by proper windowing function ?
if not (freqmin <= np.abs(freq) <= freqmax):
print 'no instrument response in this frequency range - spectrum set to zero here: ', freq
corrected_spectrum[i] = 0 #spec
#lo_mags.append([np.abs(spec),np.abs(corrected_spectrum[i]), 0 ])
#lo_freqs.append([freq,0,0,0,0,0,0])
continue
#find the appropriate frequencies ( since the current freq-value is most likely inbetween two values on the instr_freqs-axis)
#get the respective value by linear interpolation
#find the value closest to the current freq, assume it's lower
closest_lower = np.abs(freq - instr_freqs).argmin()
#if it coincides with the highest frequency/last entry:
if closest_lower == len(instr_freqs) - 1:
factor = data_spectrum[closest_lower]
# or the lowest
elif closest_lower == 0:
factor = data_spectrum[0]
else:
#in case the closest frequency value is not lower but higher, take the freq value below as lower bound for the interval:
if instr_freqs[closest_lower] > freq:
closest_lower -= 1
#define the interval:
instrfreq1 = instr_freqs[closest_lower]
instrfreq2 = instr_freqs[closest_lower + 1]
instrfactor1 = instr_spectrum[closest_lower]
instrfactor2 = instr_spectrum[closest_lower + 1]
intervallength = instrfreq2 - instrfreq1
weight = (freq - instrfreq1) / intervallength
factor = weight * instrfactor2 + (1 - weight) * instrfactor1
#lo_freqs.append([freq,instrfreq1,instrfreq2,weight,instrfactor1,instrfactor2,factor])
#finally correct the data for the instrument influence by dividing the complex data spectral value by the instrument response value:
corrected_spectrum[i] = spec / factor
#lo_mags.append([np.abs(spec),np.abs(corrected_spectrum[i]), factor ])
#invert into time domain
correctedTS = np.fft.irfft(corrected_spectrum)
#cut the zero padding
correctedTS = correctedTS[:N]
#re-attach the mean
correctedTS += datamean
return correctedTS #, lo_mags,lo_freqs