def find_1d_distortion(z_object, include_non1d = False): """ find 1D distortion tensor from z object ONly use the 1D part of the Z to determine D. Treat all frequencies as 1D, if "include_non1d = True". """ if not isinstance(z_object, MTz.Z): raise MTex.MTpyError_inputarguments('first argument must be an ' 'instance of the Z class') z_obj = z_object lo_dims = MTge.dimensionality(z_object=z_obj) if include_non1d is True: lo_dims = [1 for i in lo_dims] if len(list(np.where(np.array(lo_dims) == 1))) == 0: raise MTex.MTpyError_inputarguments('Z object does not have ' 'frequencies with spatial 1D characteristic') print lo_dims return find_distortion(z_obj, lo_dims = lo_dims)
def multiplymatrices_incl_errors(inmatrix1, inmatrix2, inmatrix1_err=None, inmatrix2_err=None): if inmatrix1 is None or inmatrix2 is None: raise MTex.MTpyError_inputarguments( 'ERROR - two 2x2 arrays needed as input') if inmatrix1.shape != inmatrix2.shape: raise MTex.MTpyError_inputarguments( 'ERROR - two 2x2 arrays with same dimensions needed as input') prod = np.array(np.dot(np.matrix(inmatrix1), np.matrix(inmatrix2))) if (inmatrix1_err is None) or (inmatrix1_err is None): return prod, None var = np.zeros((2, 2)) var[0,0] = (inmatrix1_err[0,0] * inmatrix2[0,0])**2 + (inmatrix1_err[0,1] * inmatrix2[1,0])**2+\ (inmatrix2_err[0,0] * inmatrix1[0,0])**2 + (inmatrix2_err[1,0] * inmatrix1[0,1])**2 var[0,1] = (inmatrix1_err[0,0] * inmatrix2[0,1])**2 + (inmatrix1_err[0,1] * inmatrix2[1,1])**2+\ (inmatrix2_err[0,1] * inmatrix1[0,0])**2 + (inmatrix2_err[1,1] * inmatrix1[0,1])**2 var[1,0] = (inmatrix1_err[1,0] * inmatrix2[0,0])**2 + (inmatrix1_err[1,1] * inmatrix2[1,0])**2+\ (inmatrix2_err[0,0] * inmatrix1[1,0])**2 + (inmatrix2_err[1,0] * inmatrix1[1,1])**2 var[1,1] = (inmatrix1_err[1,0] * inmatrix2[0,1])**2 + (inmatrix1_err[1,1] * inmatrix2[1,1])**2+\ (inmatrix2_err[0,1] * inmatrix1[1,0])**2 + (inmatrix2_err[1,1] * inmatrix1[1,1])**2 return prod, np.sqrt(var)
def find_2d_distortion(z_object, include_non2d=False): """ find 2D distortion tensor from z object ONly use the 2D part of the Z to determine D. Treat all frequencies as 2D, if "include_non2d = True". """ if not isinstance(z_object, MTz.Z): raise MTex.MTpyError_inputarguments('first argument must be an ' 'instance of the Z class') z_obj = z_object lo_dims = MTge.dimensionality(z_object = z_obj) #avoid the (standard) 1D distortion call -> remove all 1 lo_dims = [ 4 if i == 1 else i for i in lo_dims ] if include_non2d is True: lo_dims = [2 for i in lo_dims] if len(list(np.where(np.array(lo_dims) == 2))) == 0: raise MTex.MTpyError_inputarguments('Z object does not have' ' frequencies with spatial 2D characteristic') return find_distortion(z_obj, lo_dims = lo_dims)
def read_data_header(fn_raw): """ Deprecated!!! USE read_ts_header INSTEAD Read the header line of MTpy TS data files. input ----- MTpy TS data file name output ------- list of header elements: stationname, channel, sampling rate, starttime first sample, starttime last sample, unit, lat, lon, elevation """ fn = op.abspath(op.realpath(fn_raw)) if not op.isfile(fn): raise MTex.MTpyError_inputarguments('Not a file:%s' % fn) try: F = open(fn, 'r') except: raise MTex.MTpyError_inputarguments('File not readable:%s' % fn) firstline = F.readline().strip().split() if not firstline[0][0] == '#': raise MTex.MTpyError_ts_data('Time series data file does ' 'not have a proper header:%s' % fn) F.close() header_list = [] idx_header = 0 if len(firstline[0]) > 1: header_list.append(firstline[0][1:].upper()) else: header_list.append(firstline[1].upper()) idx_header += 1 header_list.append(firstline[idx_header + 1].lower()) header_list.append(float(firstline[idx_header + 2])) header_list.append(float(firstline[idx_header + 3])) header_list.append(int(float(firstline[idx_header + 4]))) header_list.append(firstline[idx_header + 5].lower()) header_list.append(float(firstline[idx_header + 6])) header_list.append(float(firstline[idx_header + 7])) header_list.append(float(firstline[idx_header + 8])) return header_list
def reorient_data2D(x_values, y_values, x_sensor_angle = 0 , y_sensor_angle = 90): """ Re-orient time series data of a sensor pair, which has not been in default (x=0, y=90) orientation. Input: - x-values - Numpy array - y-values - Numpy array Note: same length for both! - If not, the shorter length is taken Optional: - Angle of the x-sensor - measured in degrees, clockwise from North (0) - Angle of the y-sensor - measured in degrees, clockwise from North (0) Output: - corrected x-values (North) - corrected y-values (East) """ x_values = np.array(x_values) y_values = np.array(y_values) try: if x_values.dtype not in ['complex', 'float', 'int']: raise if len(x_values) != len(y_values): raise except: raise MTex.MTpyError_inputarguments('ERROR - both input arrays must be of same length') if len(x_values) != len(y_values): l = min(len(x_values) , len(y_values)) x_values = x_values[:l] y_values = y_values[:l] in_array = np.zeros((len(x_values), 2), x_values.dtype) in_array[:,0] = x_values in_array[:,1] = y_values try: x_angle = math.radians(x_sensor_angle) y_angle = math.radians(y_sensor_angle) except: raise MTex.MTpyError_inputarguments('ERROR - both angles must be of type int or float') T = np.matrix( [[ np.real(cmath.rect(1,x_angle)), np.imag(cmath.rect(1,x_angle))],[np.real(cmath.rect(1,y_angle)), np.imag(cmath.rect(1,y_angle))]]) try: new_array = np.dot(in_array, T.I) except: raise MTex.MTpyError_inputarguments('ERROR - angles must define independent axes to span 2D') #print new_array.shape return new_array[:,0], new_array[:,1]
def rotatematrix_incl_errors(inmatrix, angle, inmatrix_err=None): if inmatrix is None: raise MTex.MTpyError_inputarguments( 'Matrix AND eror matrix must be defined') if (inmatrix_err is not None) and (inmatrix.shape != inmatrix_err.shape): raise MTex.MTpyError_inputarguments( 'Matrix and err-matrix shapes do not match: %s - %s' % (str(inmatrix.shape), str(inmatrix_err.shape))) try: degreeangle = angle % 360 except: raise MTex.MTpyError_inputarguments( '"Angle" must be a valid number (in degrees)') phi = math.radians(degreeangle) cphi = np.cos(phi) sphi = np.sin(phi) # JP: Changed the rotation matrix to be formulated to rotate # counter clockwise, I cannot find a good reason for this except that # when you plot the strike and phase tensors the look correct with this # formulation. rotmat = np.array([[cphi, sphi], [-sphi, cphi]]) # rotmat = np.array([[ cphi, -sphi], [sphi, cphi]]) rotated_matrix = np.dot(np.dot(rotmat, inmatrix), np.linalg.inv(rotmat)) errmat = None if (inmatrix_err is not None): err_orig = np.real(inmatrix_err) errmat = np.zeros_like(inmatrix_err) # standard propagation of errors: errmat[0,0] = np.sqrt( (cphi**2 * err_orig[0,0])**2 + \ (cphi * sphi * err_orig[0,1])**2 + \ (cphi * sphi * err_orig[1,0])**2 + \ (sphi**2 * err_orig[1,1])**2 ) errmat[0,1] = np.sqrt( (cphi**2 * err_orig[0,1])**2 + \ (cphi * sphi * err_orig[1,1])**2 + \ (cphi * sphi * err_orig[0,0])**2 + \ (sphi**2 * err_orig[1,0])**2 ) errmat[1,0] = np.sqrt( (cphi**2 * err_orig[1,0])**2 + \ (cphi * sphi * err_orig[1,1])**2 +\ (cphi * sphi * err_orig[0,0])**2 + \ (sphi**2 * err_orig[0,1])**2 ) errmat[1,1] = np.sqrt( (cphi**2 * err_orig[1,1])**2 + \ (cphi * sphi * err_orig[0,1])**2 + \ (cphi * sphi * err_orig[1,0])**2 + \ (sphi**2 * err_orig[0,0])**2 ) return rotated_matrix, errmat
def read_configfile(filename): """ Read a general config file and return the content as dictionary. Config files without sections or only DEFAULT section -> return dictionary Config files with sections -> return nested dictionary (main level keys are section heads) Config files with sections as well as section-less entries -> return nested dictionary, which includes a top level 'DEFAULT' key """ #check, if file is present if not op.isfile(filename): raise MTex.MTpyError_inputarguments( 'File does not exist: {0}'.format(filename)) # try to parse file - exit, if not a config file try: #generate config parser instance configobject = ConfigParser.SafeConfigParser() #do NOT ask, why it does not work with reading from filename directly...: with open(filename) as F: d = F.read() FH = StringIO.StringIO(d) configobject.readfp(d) #filename) except: try: dummy_String = '[DEFAULT]\n' + open(filename, 'r').read() FH = StringIO.StringIO(dummy_String) #generate config parser instance configobject = ConfigParser.SafeConfigParser() configobject.readfp(FH) except: raise MTex.MTpyError_inputarguments( 'File is not a proper ' 'configuration file: {0}'.format(filename)) config_dict = configobject._sections if len(config_dict.keys()) != 0: defaults = configobject.defaults() if len(defaults.keys()) != 0: config_dict['DEFAULT'] = configobject.defaults() else: config_dict = configobject.defaults() return config_dict
def main(): if len(sys.argv) < 4: print('\nNeed at least 3 arguments: <path to BIRRP executable> '\ '<station name> <directory for time series>\n\n'\ 'Optional arguments: \n [coherence threshold]\n'\ ' [start time] \n [end time]\n\n') return try: coherence_th = float(sys.argv[4]) if not 0 <= coherence_th <= 1: raise except: print( 'coherence value invalid (float from interval [0,1]) - set to 0 instead' ) coherence_th = 0 try: starttime = float(sys.argv[5]) except: starttime = None try: endtime = float(sys.argv[6]) except: endtime = None birrp_exe_raw = sys.argv[1] birrp_exe = op.abspath(op.realpath(birrp_exe_raw)) if not op.isfile(birrp_exe): raise MTex.MTpyError_inputarguments( 'Birrp executable not existing: %s' % (birrp_exe)) stationname = sys.argv[2].upper() ts_dir_raw = sys.argv[3] ts_dir = op.abspath(op.realpath(ts_dir_raw)) if not op.isdir(ts_dir): raise MTex.MTpyError_inputarguments( 'Time series directory not existing: %s' % (ts_dir)) if 1: MTbp.runbirrp_Nin2out_simple(birrp_exe, stationname, ts_dir, coherence_th, None, 3, None, starttime, endtime)
def rotatematrix_incl_errors(inmatrix, angle, inmatrix_err = None) : if inmatrix is None : raise MTex.MTpyError_inputarguments('Matrix AND eror matrix must be defined') if (inmatrix_err is not None) and (inmatrix.shape != inmatrix_err.shape): raise MTex.MTpyError_inputarguments('Matrix and err-matrix shapes do not match: %s - %s'%(str(inmatrix.shape), str(inmatrix_err.shape))) try: degreeangle = angle%360 except: raise MTex.MTpyError_inputarguments('"Angle" must be a valid number (in degrees)') phi = math.radians(degreeangle) cphi = np.cos(phi) sphi = np.sin(phi) rotmat = np.matrix([[ cphi,sphi],[-sphi,cphi] ]) rotated_matrix = np.dot(np.dot( rotmat, inmatrix ),rotmat.I ) #print rotmat #print inmatrix #print rotated_matrix #sys.exit() errmat = None if (inmatrix_err is not None) : err_orig = np.real(inmatrix_err) errmat = np.zeros_like(inmatrix_err) # squared propagation of errors # z_err_rot[idx_freq,0,0] = np.sqrt( (cphi**2 * z_err_orig[0,0])**2 + cphi**2 * sphi**2 * ( (z_err_orig[0,1])**2 + (z_err_orig[1,0])**2) + (sphi**2 * z_err_orig[1,1])**2) # z_err_rot[idx_freq,0,1] = np.sqrt( (cphi**2 * z_err_orig[0,1])**2 + cphi**2 * sphi**2 * ( (z_err_orig[1,1])**2 + (z_err_orig[0,0])**2) + (sphi**2 * z_err_orig[1,0])**2) # z_err_rot[idx_freq,1,0] = np.sqrt( (cphi**2 * z_err_orig[1,0])**2 + cphi**2 * sphi**2 * ( (z_err_orig[1,1])**2 + (z_err_orig[0,0])**2) + (sphi**2 * z_err_orig[0,1])**2) # z_err_rot[idx_freq,1,1] = np.sqrt( (sphi**2 * z_err_orig[0,0])**2 + cphi**2 * sphi**2 * ( (z_err_orig[0,1])**2 + (z_err_orig[1,0])**2) + (cphi**2 * z_err_orig[1,1])**2) # standard propagation of errors: errmat[0,0] = np.sqrt( (cphi**2 * err_orig[0,0])**2 + (cphi * sphi * err_orig[0,1])**2 + (cphi * sphi * err_orig[1,0])**2 + (sphi**2 * err_orig[1,1])**2 ) errmat[0,1] = np.sqrt( (cphi**2 * err_orig[0,1])**2 + (cphi * sphi * err_orig[1,1])**2 + (cphi * sphi * err_orig[0,0])**2 + (sphi**2 * err_orig[1,0])**2 ) errmat[1,0] = np.sqrt( (cphi**2 * err_orig[1,0])**2 + (cphi * sphi * err_orig[1,1])**2 + (cphi * sphi * err_orig[0,0])**2 + (sphi**2 * err_orig[0,1])**2 ) errmat[1,1] = np.sqrt( (cphi**2 * err_orig[1,1])**2 + (cphi * sphi * err_orig[0,1])**2 + (cphi * sphi * err_orig[1,0])**2 + (sphi**2 * err_orig[0,0])**2 ) return rotated_matrix, errmat
def main(): if len(sys.argv) < 2: sys.exit( '\n usage:\n\t get_sampling_interval_from_data_file.py <filename>' '[optional: <file length in seconds>]\n\n' 'If no second argument is given, a file length of 3600 seconds is assumed.\n\n' ) filename_raw = sys.argv[1] filename = op.abspath(op.realpath(filename_raw)) if not op.isfile(filename): raise MTex.MTpyError_inputarguments('File not existing: %s' % (filename)) try: length = float(sys.argv[2]) if length <= 0: raise except: print 'Could not understand second argument - must be a length in seconds (int/float) - set to 3600 ' length = 3600 print MTfh.get_sampling_interval_fromdatafile(filename, length=length)
def rhophi2z(rho, phi, freq): """ Convert impedance-style information given in Rho/Phi format into complex valued Z. Input: rho - 2x2 array (real) - in Ohm m phi - 2x2 array (real) - in degrees freq - scalar - frequency in Hz Output: Z - 2x2 array (complex) """ try: if rho.shape != (2, 2) or phi.shape != (2, 2): raise if not (rho.dtype in ['float', 'int'] and phi.dtype in ['float', 'int']): raise except: raise MTex.MTpyError_inputarguments( 'ERROR - arguments must be two 2x2 arrays (real)') z = np.zeros((2, 2), 'complex') for i in range(2): for j in range(2): abs_z = np.sqrt(5 * freq * rho[i, j]) z[i, j] = cmath.rect(abs_z, math.radians(phi[i, j])) return z
def convert_degmin_tuple2degrees(latlon_list): """ Convert a 2tuple (list, tuple, array) of form "(degrees, minutes)" into degrees. Validity of the triple is assumed and has to be asserted in advance. """ sign = 1. try: latlon_list = [float(i) for i in latlon_list] if str(latlon_list[0]).startswith('-'): sign = -1. except: raise deg = latlon_list[0] minutes = latlon_list[1] if not (0<=minutes<60): raise MTex.MTpyError_inputarguments('Minutes value invalid') #take out sign for easier conversion into degrees if deg < 0: deg *= -1. degrees = deg + 1/60. * minutes return degrees * sign
def write_ts_file_from_tuple(outfile, ts_tuple, fmt='%.8e'): """ Write an MTpy TS data file, where the content is provided as tuple: (station, channel,samplingrate,t_min,nsamples,unit,lat,lon,elev, data) todo: needs tuple-validation """ header_dict = {} for i in range(len(ts_tuple) - 1): if ts_tuple[i] is not None: header_dict[lo_headerelements[i]] = ts_tuple[i] header_string = get_ts_header_string(header_dict) data = ts_tuple[-1] outfilename = make_unique_filename(outfile) try: outF = open(outfilename, 'w') outF.write(header_string) np.savetxt(outF, data, fmt=fmt) outF.close() except: raise MTex.MTpyError_inputarguments( 'ERROR - could not write content' ' of TS tuple to file : {0}'.format(outfilename)) return outfilename
def rotatevector_incl_errors(invector, angle, invector_err=None): #check for row or column vector if invector is None: raise MTex.MTpyError_inputarguments( 'Vector AND error-vector must be defined') if (invector_err is not None) and (invector.shape != invector_err.shape): raise MTex.MTpyError_inputarguments( 'Vector and errror-vector shapes do not match: %s - %s' % (str(invector.shape), str(invector_err.shape))) try: degreeangle = angle % 360 except: raise MTex.MTpyError_inputarguments( '"Angle" must be a valid number (in degrees)') phi = math.radians(degreeangle) cphi = np.cos(phi) sphi = np.sin(phi) # JP: Changed the rotation matrix to be formulated to rotate # counter clockwise, I cannot find a good reason for this except that # when you plot the strike and phase tensors the look correct with this # formulation. rotmat = np.array([[cphi, sphi], [-sphi, cphi]]) # rotmat = np.array([[ cphi, -sphi],[sphi, cphi]]) if invector.shape == (1, 2): rotated_vector = np.dot(invector, np.linalg.inv(rotmat)) else: rotated_vector = np.dot(rotmat, invector) errvec = None if (invector_err is not None): errvec = np.zeros_like(invector_err) if invector_err.shape == (1, 2): errvec = np.dot(invector_err, np.abs(np.linalg.inv(rotmat))) else: errvec = np.dot(np.abs(rotmat), invector_err) return rotated_vector, errvec
def read_ts_header(tsfile): """ Read in the header line from MTpy timeseries data files. Return header as dictionary. Return empty dict, if no header line was found. """ header_dict = {} tsfile = op.abspath(tsfile) if not op.isfile(tsfile): raise MTex.MTpyError_inputarguments('Error - ' 'input file not existing: {0}'.format(tsfile)) try: with open(tsfile,'r') as F: firstline ='' #ignoring empty lines or lines with just the '#' character in it while len(firstline) == 0: firstline = F.readline().strip() if firstline == '#': firstline = '' if firstline[0] != '#': raise except: raise MTex.MTpyError_ts_data('No header line found -' ' check file: {0}'.format(tsfile)) firstline = firstline.replace('#','') headerlist = firstline.split() for i in range(len(headerlist)): header_dict[lo_headerelements[i]] = headerlist[i] #old header had tmax instead of n_samples: if ((i == 4) and float(headerlist[4])%1 != 0 and float(headerlist[i]) > float(headerlist[i-1])): header_dict[lo_headerelements[i]] = int( (float(headerlist[i]) - float(headerlist[i-1]) )*float(headerlist[i-2]) )+1 headerlements = ['samplingrate','t_min','nsamples','lat','lon','elev'] for h in headerlements: try: header_dict[h] = float(header_dict[h]) except: pass try: if header_dict[h]%1==0: header_dict[h] = int(header_dict[h]) except: pass return header_dict
def _data_instrument_consistency_check(data, field, dipole_length, instrument, amplification, logger, gain): """ Check, if the input values given for the calibration make any sense at all. """ if len(data) == 0 : raise MTex.MTpyError_ts_data( 'no data provided for calibration' ) if not field.lower() in ['e','b']: raise MTex.MTpyError_inputarguments( 'Field must be E or B' ) # if float(dipole_length) <= 0: # raise MTpyError_inputarguments( 'Dipole length must be positive' ) if float(amplification) <= 0: raise MTex.MTpyError_inputarguments('Amplification factor must be positive') if float(gain) <= 0: raise MTex.MTpyError_inputarguments('Instrument gain must be positive') try: if not logger.lower() in list_of_loggers: raise except: raise MTex.MTpyError_inputarguments( 'wrong choice of logger') try: if not instrument.lower() in list_of_instruments: raise except: raise MTex.MTpyError_inputarguments( 'wrong choice of instrument') if field.lower == 'b': if logger.lower() == 'elogger': raise MTex.MTpyError_inputarguments( 'wrong choice of logger') if instrument.lower() == 'electrodes': raise MTex.MTpyError_inputarguments( 'wrong choice of instrument') if not float(dipole_length) == 1: raise MTex.MTpyError_inputarguments( 'Dipole length must be "1" for' ' B-field calibration') if field.lower == 'e': if not instrument.lower() == 'electrodes': raise MTex.MTpyError_inputarguments( 'wrong choice of instrument')
def rotatevector_incl_errors(invector, angle, invector_err=None): #check for row or column vector if invector is None: raise MTex.MTpyError_inputarguments( 'Vector AND error-vector must be defined') if (invector_err is not None) and (invector.shape != invector_err.shape): raise MTex.MTpyError_inputarguments( 'Vector and errror-vector shapes do not match: %s - %s' % (str(invector.shape), str(invector_err.shape))) try: degreeangle = angle % 360 except: raise MTex.MTpyError_inputarguments( '"Angle" must be a valid number (in degrees)') phi = math.radians(degreeangle) cphi = np.cos(phi) sphi = np.sin(phi) rotmat = np.matrix([[cphi, sphi], [-sphi, cphi]]) if invector.shape == (1, 2): rotated_vector = np.dot(invector, rotmat.I) else: rotated_vector = np.dot(rotmat, invector) errvec = None if (invector_err is not None): err_orig = np.real(invector_err) errvec = np.zeros_like(invector_err) if invector_err.shape == (1, 2): errvec = np.dot(invector_err, np.abs(rotmat.I)) else: errvec = np.dot(np.abs(rotmat), invector_err) return rotated_vector, errvec
def read_ts_file(mtdatafile): """ Read an MTpy TS data file and provide the content as tuple: (station, channel,samplingrate,t_min,nsamples,unit,lat,lon,elev, data) If header information is incomplete, the tuple is filled up with 'None' """ infile = op.abspath(mtdatafile) if not op.isfile(infile): raise MTex.MTpyError_inputarguments('ERROR - Data file not ' 'existing: {0}'.format(infile)) header = read_ts_header(infile) if len(header) == 0: raise MTex.MTpyError_inputarguments( 'ERROR - Data file not valid - ' 'header is missing : {0}'.format(infile)) data = np.loadtxt(infile) if len(data) != int(float(header['nsamples'])): raise MTex.MTpyError_inputarguments( 'ERROR - Data file not valid ' '- wrong number of samples in data ({1} ' 'instead of {2}): {0}'.format(infile, len(data), int(float(header['nsamples'])))) lo_header_contents = [] for i in lo_headerelements: if i in header: lo_header_contents.append(header[i]) else: lo_header_contents.append(None) lo_header_contents.append(data) return tuple(lo_header_contents)
def convert_ms2ts(indir, outdir, combine=True, invert=False): lo_dirs = [] for i, j, k in os.walk(indir): lofolders = [op.join(i, f) for f in j] lo_dirs.extend(lofolders) lo_dirs.append(indir) pathname = list(set(lo_dirs)) if len(pathname) == 0: pathname = [indir] lo_indirs = [op.relpath(i, start=indir) for i in pathname] lo_outdirs = [] #'pathname' is a list of relative pathnames. to be reconstructed under the given 'outdir' try: for i in lo_indirs: outpath = op.abspath(op.join(outdir, i)) if not op.isdir(outpath): os.makedirs(outpath) lo_outdirs.append(outpath) except: raise MTex.MTpyError_inputarguments( 'ERROR - Cannot set up output directory {0}'.format(outpath)) lo_indirs = [op.join(indir, i) for i in lo_indirs] for idx_ipath, inpath in enumerate(lo_indirs): lo_infiles = [ i for i in os.listdir(inpath) if op.isfile(op.abspath(op.join(inpath, i))) ] lo_outfiles = [ op.abspath(op.join(lo_outdirs[idx_ipath], i)) for i in lo_infiles ] lo_infiles = [ op.abspath(op.join(indir, inpath, i)) for i in lo_infiles ] for idx_fn, fn in enumerate(lo_infiles): print 'reading file {0}'.format(fn) try: outfn = MTms.quadrupol_convertfile_miniseed2ts( fn, lo_outfiles[idx_fn], combine, invert) print 'wrote file(s) {0}'.format(outfn) except: print 'Warning - file {0} is not in valid miniseed format!!!'.format( fn) continue
def read_survey_config_fn(self, survey_config_fn=None): """ read in survey configuration file and output into a useful dictionary """ if survey_config_fn is not None: self.surve_config_fn = survey_config_fn if not os.path.isfile(self.survey_config_fn): raise mtex.MTpyError_inputarguments( 'Could not find {0}, check path'.format(survey_config_fn)) # read in survey information self.survey_config_dict = mtcfg.read_survey_configfile( self.survey_config_fn)[self.station]
def __init__(self, **kwargs): fn_list = kwargs.pop('fn_list', None) mt_object_list = kwargs.pop('mt_object_list', None) #----set attributes for the class------------------------- self.mt_list = mtpt.MTplot_list(fn_list=fn_list, mt_object_list=mt_object_list) #--> set plot properties self.fig_num = kwargs.pop('fig_num', 1) self.plot_title = kwargs.pop('plot_title', None) self.fig_dpi = kwargs.pop('fig_dpi', 300) self.fig_size = kwargs.pop('fig_size', None) self.font_size = kwargs.pop('font_size', 7) self.stationid = kwargs.pop('stationid', [0, 4]) self.xlimits = kwargs.pop('xlimits', None) self.ylimits = kwargs.pop('ylimits', None) self.ref_point = kwargs.pop('ref_point', (0, 0)) self.map_scale = kwargs.pop('map_scale', 'latlon') self.marker = kwargs.pop('marker', 'v') self.marker_size = kwargs.pop('marker_size', 10) self.marker_color = kwargs.pop('marker_color', 'k') self.plot_names = kwargs.pop('plot_names', True) self.text_size = kwargs.pop('text_size', 7) self.text_weight = kwargs.pop('text_weight', 'normal') self.text_color = kwargs.pop('text_color', 'k') self.text_ha = kwargs.pop('text_ha', 'center') self.text_va = kwargs.pop('text_va', 'baseline') self.text_angle = kwargs.pop('text_angle', 0) self.text_x_pad = kwargs.pop('text_x_pad', None) self.text_y_pad = kwargs.pop('text_y_pad', None) self.image_file = kwargs.pop('image_file', None) self.image_extent = kwargs.pop('image_extent', None) if self.image_file is not None: if self.image_extent is None: raise mtex.MTpyError_inputarguments('Need to input extents '+\ 'of the image as' +\ '(x0, y0, x1, y1)') #--> plot if desired self.plot_yn = kwargs.pop('plot_yn', 'y') if self.plot_yn == 'y': self.plot()
def convert_dms_tuple2degrees(latlon_list): """ Convert a triple (list, tuple, array) of degrees, minuts, seconds into degrees. Validity of the triple is assumed and has to be asserted in advance. """ sign = 1. try: latlon_list = [float(i) for i in latlon_list] if str(latlon_list[0]).startswith('-'): sign = -1. except: # if triple is given as string: latlon_raw = latlon_list # allow separation by :,. or space for (deg min sec) format try: latlon_list = re.split('[ :,]', latlon_raw) except: raise MTex.MTpyError_config_file( 'Config file error: lat/lon is in invalid format') try: latlon_list = [float(i) for i in latlon_list] except: raise MTex.MTpyError_config_file( 'Config file error: lat/lon is in invalid format') if str(latlon_list[0])[0] == '-': sign = -1. deg = latlon_list[0] minutes = latlon_list[1] seconds = latlon_list[2] if not (-180 <= deg <= 360 and 0 <= minutes < 60 and 0 <= seconds < 60): print 'Value for degrees, minutes, or seconds invalid' raise MTex.MTpyError_inputarguments() # take out sign for easier conversion into degrees if deg < 0: deg *= -1 degrees = deg + 1 / 60. * minutes + 1 / 3600. * seconds return degrees * sign
def invertmatrix_incl_errors(inmatrix, inmatrix_err=None): if inmatrix is None: raise MTex.MTpyError_inputarguments('Matrix must be defined') if (inmatrix_err is not None) and (inmatrix.shape != inmatrix_err.shape): raise MTex.MTpyError_inputarguments( 'Matrix and err-matrix shapes do not match: %s - %s' % (str(inmatrix.shape), str(inmatrix_err.shape))) if (inmatrix.shape[-2] != inmatrix.shape[-1]): raise MTex.MTpyError_inputarguments('Matrices must be square!') if (inmatrix_err is not None) and (inmatrix_err.shape[-2] != inmatrix_err.shape[-1]): raise MTex.MTpyError_inputarguments('Matrices must be square!') dim = inmatrix.shape[-1] det = np.linalg.det(inmatrix) if det == 0: raise MTex.MTpyError_inputarguments( 'Matrix is singular - I cannot invert that!') inv_matrix = np.zeros_like(inmatrix) if dim != 2: raise MTex.MTpyError_inputarguments('Only 2D matrices supported yet') inv_matrix = np.linalg.inv(inmatrix) inv_matrix_err = None if (inmatrix_err is not None): inmatrix_err = np.real(inmatrix_err) inv_matrix_err = np.zeros_like(inmatrix_err) for i in range(2): for j in range(2): #looping over the entries of the error matrix err = 0. for k in range(2): for l in range(2): #each entry has 4 summands err += np.abs(-inv_matrix[i, k] * inv_matrix[l, j] * inmatrix_err[k, l]) inv_matrix_err[i, j] = err return inv_matrix, inv_matrix_err
def get_j_file(self, birrp_dir=None): """ get .j file output by birrp """ if birrp_dir is not None: self.birrp_dir = birrp_dir if self.birrp_dir is None or not os.path.exists(self.birrp_dir): raise mtex.MTpyError_inputarguments('No birrp directory input,' 'check path {0}'.format( self.birrp_dir)) try: self.j_fn = [ os.path.join(self.birrp_dir, fn) for fn in os.listdir(self.birrp_dir) if fn.endswith('.j') ][0] except IndexError: print('Could not find a .j file in {0}, check path.'.format( self.birrp_dir)) self.j_fn = None
def main(): if len(sys.argv) < 3: raise MTex.MTpyError_inputarguments( 'Need at least 2 arguments: <path to files> <response file> [<output dir>] [<channel(s)>] ' ) pathname_raw = sys.argv[1] directory = op.abspath(op.realpath(pathname_raw)) responsefilename_raw = sys.argv[2] responsefile = op.abspath(op.realpath(responsefilename_raw)) if not op.isdir(directory): raise MTex.MTpyError_inputarguments('Directory not existing: %s' % (directory)) if not op.isfile(responsefile): raise MTex.MTpyError_inputarguments('Response file not existing: %s' % (responsefile)) #check, if response file is in proper shape (3 columns freq,re,im of real values): try: responsedata = np.loadtxt(responsefile) s = responsedata.shape if s[1] != 3: raise freq_min = responsedata[0, 0] freq_max = responsedata[-1, 0] except: raise MTex.MTpyError_inputarguments( 'Response file (%s) in wrong format - must be 3 columns: freq,real,imag' % (responsefile)) #set up output directory: try: outdir_raw = sys.argv[3] outdir = op.abspath(outdir_raw) except: outdir = op.join(directory, 'instr_resp_corrected') try: if not op.isdir(outdir): os.makedirs(outdir) except: raise MTex.MTpyError_inputarguments( 'Output directory cannot be generated: %s' % (outdir)) #define channels to be considered for correction: try: lo_channels = list( set([ i.upper() if len(i) == 2 else 'B' + i.upper() for i in sys.argv[4].split(',') ])) except: print 'No channel list found - using BX, BY, HX, HY' lo_channels = ['BX', 'BY', 'HX', 'HY', 'BZ', 'HZ'] #collect file names within the folder lo_allfiles = os.listdir(directory) lo_allfiles = [ op.abspath(op.join(directory, i)) for i in lo_allfiles if op.isfile(op.abspath(op.join(directory, i))) ] # if op.isfile(i)==True] #generate list of list-of-files-for-each-channel: lo_lo_files_for_channels = [[] for i in lo_channels] #check the files for information about the determined channels: for fn in lo_allfiles: header_dict = MTfh.read_ts_header(fn) if len(header_dict.keys()) == 0: continue ch = header_dict['channel'].upper() if ch not in lo_channels: continue try: ch_idx = lo_channels.index(ch) except ValueError: continue # use the current file, if it contains a header line and contains signal from the requested channel: lo_lo_files_for_channels[ch_idx].append(fn) #if no files had header lines or did not contain data from the appropriate channel(s): if np.sum([len(i) for i in lo_lo_files_for_channels]) == 0: print 'channels: ', lo_channels, ' - directory: ', directory raise MTex.MTpyError_inputarguments( 'No information for channels found in directory {0} - Check header lines!' .format(directory)) #============================================= # start the instrument correction # looping over all requested channels: for ch in lo_channels: #skip, if no data are available for the current channel: if [len(i) for i in lo_lo_files_for_channels][lo_channels.index(ch)] == 0: continue #set up lists for the infos needed later, esp. for the file handling lo_files = lo_lo_files_for_channels[lo_channels.index(ch)] lo_t_mins = [] lo_headers = [] #read in header lines and sort files by increasing starttimes t_min for fn in lo_files: header_dict = MTfh.read_ts_header(fn) lo_t_mins.append(header_dict['t_min']) lo_headers.append(header_dict) #sort all the collected lists by t_min idxs = np.array(lo_t_mins).argsort() lo_t_mins = [lo_t_mins[i] for i in idxs] lo_files = [lo_files[i] for i in idxs] lo_headers = [lo_headers[i] for i in idxs] # finding consecutive, continuous time axes: lo_timeaxes = [] ta_old = None for idx, header in enumerate(lo_headers): ta_cur = np.arange(int(header['nsamples'])) / float( header['samplingrate']) + float(header['t_min']) #if there is no old ta: if ta_old == None: ta_old = ta_cur continue # if gap between old and new ta is too big: if (ta_cur[0] - ta_old[-1]) > (2 * 1. / float(header['samplingrate'])): lo_timeaxes.append(np.array(ta_old)) ta_old = ta_cur continue #find index of new ta which is closest to the end of old_ta - most commonly it's '0' ! overlap = np.abs(ta_cur - ta_old[-1]).argmin() ta_cur = ta_cur[overlap:] ta_old = np.concatenate([ta_old, ta_cur]) #append last active time axis ta: lo_timeaxes.append(np.array(ta_old)) #determine maximal period from response file and existinng time axes. #win = get_windowlength() = max([ i[-1]-i[0] for i in lo_timeaxes] ) # the minimum of the maximal resolvable signal period and the longest continuous time axis: winmax = 1. / freq_min #for debugging set large window size: #winmax = 5e5 #later on, if the TS is longer than 3 times this time window, we want to cut out subsections of the time series. These cuts shall consist of triplets of subwindows, each of which shall not be longer than this maximum period. #Now the data set has to be corrected/deconvolved by looping over the collected time axes: for ta in lo_timeaxes: print '\nhandling time axis: {0} - {1} ({2} samples) '.format( ta[0], ta[-1], len(ta)) #if the time section is shorter than 3 times the maximum defined by the response function, read in the whole data set at once for this interval if (ta[-1] - ta[0]) < (3 * winmax): print 'time axis short enough ({0} seconds) - reading all at once'.format( ta[-1] - ta[0]) #collect the appropriate files in a list #after the MTpy preprocessing the start end end of the time series coincide with files start and endings, so no "half files" are involved. cur_time = ta[0] data = [] files = [] headers = [] starttimes = [] while cur_time < ta[-1]: for idx, header in enumerate(lo_headers): ta_cur = np.arange(int(header['nsamples'])) / float( header['samplingrate']) + float(header['t_min']) if cur_time in ta_cur: start_idx = np.where(ta_cur == cur_time)[0][0] break fn = lo_files[idx] files.append(fn) headers.append(header) starttimes.append(float(header['t_min'])) cur_data = np.loadtxt(fn) print 'current data section length: ', len(cur_data) if ta_cur[-1] <= ta[-1]: data.extend(cur_data[start_idx:].tolist()) cur_time = ta_cur[-1] + 1. / float( header['samplingrate']) else: end_idx = np.where(ta_cur == ta[-1])[0][0] data.extend(cur_data[start_idx:end_idx + 1].tolist()) cur_time = ta[-1] print 'current data length: ', len(data) data = np.array(data) data = scipy.signal.detrend(data) #at this point, the data set should be set up for the given time axis corrected_timeseries = MTin.correct_for_instrument_response( data, float(header['samplingrate']), responsedata) print 'corrected TS starting at {0}, length {1}'.format( ta[0], len(corrected_timeseries)) #now, save this TS back into the appropriate files, including headers for idx, fn in enumerate(files): # output file name: use input file name and append '_true' inbasename = op.basename(fn) outbasename = ''.join([ op.splitext(inbasename)[0] + '_true', op.splitext(inbasename)[1] ]) outfn = op.join(outdir, outbasename) outF = open(outfn, 'w') header = headers[idx] unit = header['unit'] if unit[-6:].lower() != '(true)': unit += '(true)' header['unit'] = unit headerline = MTfh.get_ts_header_string(header) outF.write(headerline) starttime = starttimes[idx] length = int(float(header['nsamples'])) startidx = (np.abs(starttime - ta)).argmin() print startidx, length, len(corrected_timeseries), len(ta) print '\nhandling file {0} - starttime {1}, - nsamples {2}'.format( outfn, starttime, length) print outdir, outfn data = corrected_timeseries[startidx:startidx + length] np.savetxt(outF, data) outF.close() #To do so, use the time axis and run over the input files again,determine the filenames. Use them, and put them (slightly modified, perhaps?) into the given output directory #return corrected_timeseries else: #find partition into pieces of length 'winmax'. the remainder is equally split between start and end: #assume constant sampling rate, just use the last opened header (see above): samplingrate = float(header['samplingrate']) #total time axis length: ta_length = ta[-1] - ta[0] + 1. / samplingrate #partition into winmax long windows n_windows = int(ta_length / winmax) remainder = ta_length % winmax lo_windowstarts = [ta[0]] for i in range(n_windows + 1): t0 = ta[0] + remainder / 2. + i * winmax lo_windowstarts.append(t0) print 'time axis long ({0} seconds) - processing in {1} sections (window: {2})'.format( ta_length, n_windows, winmax) # lists of input file(s) containing the data - for all 3 sections of the moving window section1_lo_input_files = [] section2_lo_input_files = [] section3_lo_input_files = [] section1_data = [] section2_data = [] section3_data = [] section1_ta = [] section2_ta = [] section3_ta = [] file_open = False # loop over the winmax long sections: for idx_t0, t0 in enumerate(lo_windowstarts): #the last two window-starts do not get an own window: if idx_t0 > n_windows - 1: # since there are 'n_windows' full intervals for the moving window break print '\n----- section {0} ----\n'.format(idx_t0 + 1) if idx_t0 == n_windows - 1: endtime = ta[-1] else: endtime = lo_windowstarts[idx_t0 + 3] - 1. / samplingrate print 's1 = {0} - {1}, s2 = {2} - {3}, s3 = {4} - {5}\n'.format( t0, lo_windowstarts[idx_t0 + 1] - 1. / samplingrate, lo_windowstarts[idx_t0 + 1], lo_windowstarts[idx_t0 + 2] - 1. / samplingrate, lo_windowstarts[idx_t0 + 2], endtime) #for each step (except for he last one obviously), 3 consecutive parts are read in, concatenated and deconvolved. Then the central part is taken as 'true' data. #only for the first and the last sections (start and end pieces) are handled together with the respective following/preceding section #the data currently under processing: data = [] timeaxis = [] #list of current input data files and their starting times - only files, which contain parts of the time interval for the output! lo_infiles = [] lo_t0s = [] #if old data are present from the step before: if (len(section2_data) > 0) and (len(section3_data) > 0): print 'old data found....moving window' section1_data = section2_data section2_data = section3_data section1_ta = section2_ta section2_ta = section3_ta section1_lo_input_files = section2_lo_input_files section2_lo_input_files = section3_lo_input_files section1_lo_t0s = section2_lo_t0s section2_lo_t0s = section3_lo_t0s #otherwise, it's the first step, so all 3 sections have to be read else: print 'first section...initialise data collection' section1_data, section1_lo_input_files, section1_lo_t0s = read_ts_data_from_files( t0, lo_windowstarts[idx_t0 + 1], lo_t_mins, lo_files) section2_data, section2_lo_input_files, section2_lo_t0s = read_ts_data_from_files( lo_windowstarts[idx_t0 + 1], lo_windowstarts[idx_t0 + 2], lo_t_mins, lo_files) section1_ta = np.arange( len(section1_data)) / samplingrate + t0 section2_ta = np.arange( len(section2_data )) / samplingrate + lo_windowstarts[idx_t0 + 1] if idx_t0 < n_windows - 1: print 'lll' section3_data, section3_lo_input_files, section3_lo_t0s = read_ts_data_from_files( lo_windowstarts[idx_t0 + 2], lo_windowstarts[idx_t0 + 3], lo_t_mins, lo_files) else: print 'jjjj' #for the last section, there is no lo_windowstarts[idx_t0 +3], so it must be the end of the overall time axis section3_data, section3_lo_input_files, section3_lo_t0s = read_ts_data_from_files( lo_windowstarts[idx_t0 + 2], ta[-1] + 1. / samplingrate, lo_t_mins, lo_files) section3_ta = np.arange( len(section3_data)) / samplingrate + lo_windowstarts[ idx_t0 + 2] data = np.concatenate( [section1_data, section2_data, section3_data]) timeaxis = np.concatenate( [section1_ta, section2_ta, section3_ta]) print 'sections ta: ', section1_ta[0], section1_ta[-1], len( section1_ta), section2_ta[0], section2_ta[-1], len( section2_ta), section3_ta[0], section3_ta[-1], len( section3_ta) #continue data = np.array(data) # remove mean and linear trend (precaution, since it should be removed as low frequency # content anyway) data = scipy.signal.detrend(data) #---------------------------- # the actual deconvolution: corrected_data = MTin.correct_for_instrument_response( data, samplingrate, responsedata) #---------------------------- if idx_t0 == 0: # for the first window, the first section is output as well startidx = 0 lo_infiles.extend(section1_lo_input_files) lo_t0s.extend(section1_lo_t0s) else: #otherwise, just the central section, so take the start of this middle section startidx = ( np.abs(timeaxis - lo_windowstarts[idx_t0 + 1])).argmin() #collect the respective input filenames and their starting times for fn in section2_lo_input_files: if fn not in lo_infiles: lo_infiles.append(fn) for t0 in section2_lo_t0s: if t0 not in lo_t0s: lo_t0s.append(t0) if idx_t0 == n_windows - 1: #for the last window, the last section is added for fn in section3_lo_input_files: if fn not in lo_infiles: lo_infiles.append(fn) for t0 in section3_lo_t0s: if t0 not in lo_t0s: lo_t0s.append(t0) data2write = corrected_data[startidx:] timeaxis2write = timeaxis[startidx:] else: #for all windows but the last, get the middle section end endidx = ( np.abs(timeaxis - lo_windowstarts[idx_t0 + 2])).argmin() #and cut out the section data2write = corrected_data[startidx:endidx] timeaxis2write = timeaxis[startidx:endidx] #print 'indizes:',len(timeaxis),startidx,len(timeaxis2write),timeaxis2write[0],timeaxis2write[-1] # at this point, we got # - the corrected data # - the sorted list of the involved input files (and their starting times) # # write data to the output files, whose structure is following the input file(s): # maximum time stamp of the current data set (section) tmax = timeaxis2write[-1] #initialise the current time to the beginning of the section to be written t = timeaxis2write[0] print '\nfiles involved in this section', lo_infiles while t < tmax: print t if file_open == False: print 'no file open...preparing new one' # take the first of the input files in the list: print 'read header of input file {0} '.format( lo_infiles[0]) header = MTfh.read_ts_header(lo_infiles[0]) ta_tmp = np.arange(float( header['nsamples'])) / float( header['samplingrate']) + float( header['t_min']) unit = header['unit'] if unit[-6:].lower() != '(true)': unit += '(true)' header['unit'] = unit headerline = MTfh.get_ts_header_string(header) # output file name: use input file name and append '_true' inbasename = op.basename(fn) outbasename = ''.join([ op.splitext(inbasename)[0] + '_true', op.splitext(inbasename)[1] ]) outfn = op.join(outdir, outbasename) print 'write header to output file {0} '.format( outfn) outF = open(outfn, 'w') outF.write(headerline) # if the section exceeds the time axis of the file: if tmax > ta_tmp[-1]: print 'data longer than space left in file...storing first part ' #write as many samples to the files as there belong np.savetxt( outF, data2write[:int(float(header['nsamples']) )]) print 'write data to output file {0} '.format( outfn) #close the file outF.close() file_open = False print ' output file closed : {0} '.format( outfn) datalength = 0 #drop out the first elements of the lists dummy = lo_infiles.pop(0) print 'dropped {0}'.format(dummy) dummy = lo_t0s.pop(0) print 'dropped {0}'.format(dummy) #cut the written part of the data print 'data cut from {0} to {1}'.format( len(data2write), len(data2write[ int(float(header['nsamples'])):])) data2write = data2write[ int(float(header['nsamples'])):] timeaxis2write = timeaxis2write[ int(float(header['nsamples'])):] #define the current time as one sample after the end of the file, which was just closed t = ta_tmp[-1] + 1. / float( header['samplingrate']) print 'current time set to {0}'.format(t) # and back to the while condition, since there are unwritten data #if the section is not longer than the time axis of the newly opened file: else: print 'data fits into open file : {0}'.format( outF.name) #write everything np.savetxt(outF, data2write) print 'wrote data to file' #check, if by chance this is exactly the correct number of samples for this file: if tmax == ta_tmp[-1]: #if so, close it print 'file full....closing...' outF.close() file_open = False datalength = 0 print 'written data to file and closed it: {0}'.format( outF.name) dummy = lo_infiles.pop(0) dummy = lo_t0s.pop(0) #actually, the infile list should be empty now, since the whole section has been written to a file!! # otherwise, the section is shorter, so the file (potentially) misses entries else: datalength = len(data2write) print 'file not full...{0} remains open - {1} of max. {2} samples written '.format( outF.name, datalength, int(float(header['nsamples']))) file_open = True #define the current time as at the end of the time axis of the section, i.e. 'go to next section': t = tmax print 'current time set to ', t #otherwise, a file is already open and the next section has to be appended there: else: print 'open file {0} is waiting for data...'.format( outF.name) header = MTfh.read_ts_header(lo_infiles[0]) ta_tmp = np.arange(float( header['nsamples'])) / float( header['samplingrate']) + float( header['t_min']) print tmax, ta_tmp[ -1], '{0} of {1} samples used...{2} waiting\n'.format( datalength, len(ta_tmp), len(data2write)) #check, if the data exceeds the time axis of the open file: if tmax > ta_tmp[-1]: print 'data longer than space left in file...storing first {0} here and cut of the rest ({1}) for further processing...\n'.format( len(ta_tmp) - datalength, len(data2write) - len(ta_tmp) + datalength) #determine the index of the section time axis, which belongs to the last entry of the currently open file - including the last value! endidx = (np.abs(timeaxis2write - ta_tmp[-1])).argmin() + 1 #write the respective part of the data to the file and close it then np.savetxt(outF, data2write[:endidx]) outF.close() file_open = False print 'closed full file {0}'.format(outF.name) #cut out the first bit, which is already written print 'cutting data to remaining bit ({0}->{1})\n'.format( len(data2write), len(data2write) - len(data2write[:endidx])) data2write = data2write[endidx:] timeaxis2write = timeaxis2write[endidx:] # drop the file, which is used and done print 'dropping name of closed file', lo_infiles[ 0] dummy = lo_infiles.pop(0) dummy = lo_t0s.pop(0) #set the current time to the start of the next file t = ta_tmp[-1] + 1. / float( header['samplingrate']) print 'current time set to ', t #if the section is not longer than the time axis of the open file: else: print 'data fits into the open file :', outF.name #write everything np.savetxt(outF, data2write) datalength += len(data2write) print 'file contains {0} samples - {1} open '.format( datalength, int(float(header['nsamples'])) - datalength) #check, if by chance this is exactly the correct number of samples for this file: if tmax == ta_tmp[-1]: #if so, close it outF.close() print 'written data to file {0}'.format( outfn) file_open = False dummy = lo_infiles.pop(0) dummy = lo_t0s.pop(0) #actually, the infile list should be empty now, since the whole section has been written to a file!! # otherwise, the section is shorter, so the file (potentially) misses entries else: file_open = True #define the current time as at the end of the time axis of the section => go to next section: t = tmax # after looping over all sections, check, if the last file has been closed: if file_open == True: outF.close() print 'written data to file {0}'.format(outF.name)
def EDL_make_dayfiles(inputdir, sampling, stationname=None, outputdir=None): """ Concatenate ascii time series to dayfiles (calendar day, UTC reference). Data can be within a single directory or a list of directories. However, the files in the directory(ies) 'inputdir' have to be for one station only, and named with a 2 character suffix, defining the channel! If the time series are interrupted/discontinuous at some point, a new file will be started after that point, where the file index 'idx' is increased by 1. If no stationname is given, the leading non-datetime characters in the first filename are used. Files are named as 'stationname_samplingrate_date_idx.channel' Stationname, channel, and sampling are written to a header line. Output data consists of a single column float data array. The data are stored into one directory. If 'outputdir' is not specified, a subdirectory 'dayfiles' will be created witihn the current working directory. Note: Midnight cannot be in the middle of a file, because only file starts are checked for a new day!! """ try: if type(inputdir) == str: raise lo_foldernames = [i for i in inputdir] except TypeError: lo_foldernames = [inputdir] #typical suffixes for EDL output file names components = ['ex', 'ey', 'bx', 'by', 'bz'] lo_allfiles = [] pattern = '*.[ebEB][xyzXYZ]' if stationname is not None: pattern = '*{0}*.[ebEB][xyzXYZ]'.format(stationname.lower()) print '\nSearching for files with pattern: ', pattern for folder in lo_foldernames: wd = op.abspath(op.realpath(folder)) if not op.isdir(wd): #print 'Directory not existing: %s' % (wd) lo_foldernames.remove(wd) continue lo_dirfiles = [ op.abspath(op.join(wd, i)) for i in os.listdir(wd) if fnmatch.fnmatch(i.lower(), pattern.lower()) is True ] lo_allfiles.extend(lo_dirfiles) #check, if list of files is empty if len(lo_allfiles) == 0: if stationname is not None: raise MTex.MTpyError_inputarguments('Directory(ies) do(es) not contain'\ ' files to combine for station {0}:\n {1}'.format(stationname, inputdir)) raise MTex.MTpyError_inputarguments('Directory does not contain files'\ ' to combine:\n {0}'.format(inputdir)) #define subfolder for storing dayfiles outpath = op.join(os.curdir, 'dayfiles') if outputdir is not None: try: outpath = op.abspath(op.join(os.curdir, outputdir)) if not op.exists(outpath): try: os.makedirs(outpath) except: raise if not os.access(outpath, os.W_OK): raise except: print 'Cannot generate writable output directory {0} - using'\ ' generic location "dayfiles" instead'.format(outpath) outpath = op.join(wd, 'dayfiles') pass #generate subfolder, if not existing if not op.exists(outpath): try: os.makedirs(outpath) except: MTex.MTpyError_inputarguments('Cannot generate output'\ ' directory {0} '.format(outpath)) #outer loop over all components for comp in components: #make list of files for the current component lo_files = np.array( [op.join(wd, i) for i in lo_allfiles if (i.lower()[-2:] == comp)]) #make list of starting times for the respective files lo_starttimes = np.array( [EDL_get_starttime_fromfilename(f) for f in lo_files]) #sort the files by their starting times idx_chronologic = np.argsort(lo_starttimes) #obtain sorted lists of files and starting times lo_sorted_files = list(lo_files[idx_chronologic]) lo_sorted_starttimes = list(lo_starttimes[idx_chronologic]) idx = 0 while idx < len(lo_sorted_starttimes): try: val = lo_sorted_starttimes[idx] except: break if val is None: dummy = lo_sorted_files.pop(idx) dummy = lo_sorted_starttimes.pop(idx) else: idx += 1 #set stationname, either from arguments or from filename if stationname is None: stationname = EDL_get_stationname_fromfilename( lo_sorted_files[0]).upper() #set counting variables - needed for handling of consecutive files sameday = 0 fileopen = 0 incomplete = 0 fileindex = 0 #allocate a data array to fill # this is more memory efficient than extending lists!! #cater for potential rounding errors: if sampling < 1: max_n_data = 86400 * (int(1. / sampling) + 1) else: max_n_data = int(86400. / sampling) + 1 day_data = np.zeros(max_n_data, 'int') #loop over all (sorted) files for the current component for idx_f, f in enumerate(lo_sorted_files): try: print 'Reading file %s' % (f) #starting time of current file file_start_time = lo_sorted_starttimes[idx_f] #get tuple with the starting time of the current file file_start = time.gmtime(file_start_time) #read in raw data data_in = [] Fin = open(f) for line in Fin: #.readlines(): # try: data_in.append(int(float(line.strip()))) # except: # pass data_in = np.array(data_in) Fin.close() #data_in = np.loadtxt(f) except: print 'WARNING - could not read file - skipping...' continue no_samples = len(data_in) tmp_file_time_axis = np.arange( no_samples) * sampling + file_start_time #file_time_axis = (np.arange(no_samples)*sampling + # file_start_time).tolist() #time of the last sample + 1x sampling-interval #file_end_time = file_time_axis[-1] + sampling file_end_time = tmp_file_time_axis[-1] + sampling #set the time as starting time for output file, if no output file is open already if fileopen == 0: outfile_starttime = file_start_time #outfile_timeaxis = file_time_axis old_time_axis = tmp_file_time_axis[:] arrayindex = 0 #if it's a single column of data if np.size(data_in.shape) == 1: day_data[arrayindex:arrayindex + len(data_in)] = data_in #outfile_data = data_in.tolist() #otherwise assuming that the first column is time, so just take the second one else: day_data[arrayindex:arrayindex + len(data_in)] = data_in[:, 1] #outfile_data = data_in[:,1].tolist() #jump with index to current point on time axis arrayindex += len(data_in) outfile_endtime = file_end_time file_date = '{0}{1:02}{2:02}'.format(file_start[0], file_start[1], file_start[2]) #define output filename new_fn = '{0}_1day_{1}_{2}.{3}'.format(stationname, file_date, fileindex, comp) new_file = op.abspath(op.join(outpath, new_fn)) #open output file F = open(new_file, 'w') fileopen = 1 else: #check, if the new file ends earlier than data in buffer. #if yes, just skip this file: if file_end_time < outfile_endtime: continue #if current file starts earlier than the endtime of data in buffer then delete ambiguous parts of the buffer: #elif (outfile_timeaxis[-1] - file_start_time) > epsilon: elif (outfile_endtime - file_start_time) > epsilon: #find point on the outfile time axis for the beginning of current file: overlap_idx = arrayindex - int( (outfile_endtime - file_start_time) / sampling) #set the array index back arrayindex = overlap_idx #re-define outfile time axis and data # outfile_timeaxis = np.delete(outfile_timeaxis, # np.arange(len(outfile_timeaxis) - # overlap_idx) + # overlap_idx).tolist() # outfile_data = np.delete(outfile_data, # np.arange(len(outfile_data) - # overlap_idx) + # overlap_idx).tolist() #old_time_axis = tmp_file_time_axis[:] #append current file's time axis #outfile_timeaxis.extend(file_time_axis) #append current data #if it's a single column of data if np.size(data_in.shape) == 1: day_data[arrayindex:arrayindex + len(data_in)] = data_in #outfile_data.extend(data_in.tolist()) #otherwise assuming that the first column is time, so just take the second one else: day_data[arrayindex:arrayindex + len(data_in)] = data_in[:, 1] #outfile_data.extend(data_in[:,1].tolist()) arrayindex += len(data_in) print len(data_in), arrayindex arrayindex += len(data_in) outfile_endtime = (arrayindex + 1) * sampling + outfile_starttime #----------- #check, if there is a next file: try: next_file_start_time = lo_sorted_starttimes[idx_f + 1] except: incomplete = 1 #if there is a next file, # - check, if it's the same day # - check, if it continues at the end of the current one: if incomplete == 0: next_file_start_time = lo_sorted_starttimes[idx_f + 1] next_file_start = time.gmtime(next_file_start_time) if next_file_start[2] == file_start[2]: #print 'sameday',file_start[:] sameday = 1 else: incomplete = 1 sameday = 0 fileindex = 0 #print '\t NOT sameday', fileindex if next_file_start_time - file_end_time > epsilon: incomplete = 1 if incomplete == 1 and sameday == 1: fileindex += 1 #check, if the file has to be closed and written now if incomplete == 1: #define header info if outfile_starttime % 1 == 0: outfile_starttime = int(outfile_starttime) headerline = '# {0} {1} {2:.1f} {3} {4} \n'.format( stationname, comp.lower(), 1. / sampling, outfile_starttime, arrayindex) else: headerline = '# {0} {1} {2:.1f} {3:f} {4} \n'.format( stationname, comp.lower(), 1. / sampling, outfile_starttime, arrayindex) F.write(headerline) #outfile_array = np.zeros((len(outfile_timeaxis),2)) #outfile_array[:,0] = outfile_timeaxis #outfile_array[:,1] = outfile_data for i in range(arrayindex): F.write('{0}\n'.format(int(day_data[i]))) #outstring = '\n'.join(['{0:d}'.format(i) for i in day_data[:arrayindex]]) #F.write(outstring) #np.savetxt(F,day_data[:arrayindex],fmt='%d') #np.savetxt(F, np.array(outfile_data)) arrayindex = 0 F.close() print '\t wrote file %s' % (new_file) fileopen = 0 incomplete = 0
def EDL_make_Nhour_files(n_hours, inputdir, sampling, stationname=None, outputdir=None): """ See 'EDL_make_dayfiles' for description and syntax. Only difference: output files are blocks of (max) N hours, starting to count at midnight (00:00h) each day. Conditions: 1. 24%%N = 0 2. input data files start on the hour marks Not working yet!! """ #print '\n\tnot working yet - code under development !!\n' #return try: if 24 % n_hours != 0: raise except: sys.exit('ERROR - File block length must be on of: 1,2,3,4,6,8,12 \n') n_hours = int(n_hours) #list of starting hours for the data blocks: lo_hours = [int(i) for i in np.arange(int(24 / n_hours)) * n_hours] no_blocks = len(lo_hours) #build a list that contains the respective groups of starting hours belonging # to the same block lo_blockhours = [] counter = 0 dummylist = [] for i in range(24): dummylist.append(i) counter += 1 if counter == n_hours: lo_blockhours.append(dummylist) dummylist = [] counter = 0 # most of the following code is redundant/taken from the # 'EDL_make_dayfiles' function # This can be cleaned up later try: if type(inputdir) == str: raise lo_foldernames = [i for i in inputdir] except TypeError: lo_foldernames = [inputdir] #typical suffixes for EDL output file names components = ['ex', 'ey', 'bx', 'by', 'bz'] lo_allfiles = [] pattern = '*.[ebEB][xyzXYZ]' if stationname is not None: pattern = '*{0}*.[ebEB][xyzXYZ]'.format(stationname.lower()) print '\nSearching for files with pattern: ', pattern for folder in lo_foldernames: wd = op.abspath(op.realpath(folder)) if not op.isdir(wd): #print 'Directory not existing: %s' % (wd) lo_foldernames.remove(wd) continue lo_dirfiles = [ op.abspath(op.join(wd, i)) for i in os.listdir(wd) if fnmatch.fnmatch(i.lower(), pattern.lower()) is True ] lo_allfiles.extend(lo_dirfiles) #check, if list of files is empty if len(lo_allfiles) == 0: if stationname is not None: raise MTex.MTpyError_inputarguments('Directory(ies) do(es) not contain'\ ' files to combine for station {0}:\n {1}'.format(stationname, inputdir)) raise MTex.MTpyError_inputarguments('Directory does not contain files'\ ' to combine:\n {0}'.format(inputdir)) #define subfolder for storing dayfiles outpath = op.join(os.curdir, '{0}hourfiles'.format(int(n_hours))) if outputdir is not None: try: outpath = op.abspath(op.join(os.curdir, outputdir)) if not op.exists(outpath): try: os.makedirs(outpath) except: raise if not os.access(outpath, os.W_OK): raise except: print 'Cannot generate writable output directory {0} - using'\ ' generic location "dayfiles" instead'.format(outpath) outpath = op.join(wd, '{0}hourfiles'.format(int(n_hours))) pass #generate subfolder, if not existing if not op.exists(outpath): try: os.makedirs(outpath) except: MTex.MTpyError_inputarguments('Cannot generate output'\ ' directory {0} '.format(outpath)) #outer loop over all components for comp in components: #make list of files for the current component lo_files = np.array( [op.join(wd, i) for i in lo_allfiles if (i.lower()[-2:] == comp)]) #make list of starting times for the respective files lo_starttimes = np.array( [EDL_get_starttime_fromfilename(f) for f in lo_files]) #sort the files by their starting times idx_chronologic = np.argsort(lo_starttimes) #obtain sorted lists of files and starting times lo_sorted_files = list(lo_files[idx_chronologic]) lo_sorted_starttimes = list(lo_starttimes[idx_chronologic]) idx = 0 while idx < len(lo_sorted_starttimes): try: val = lo_sorted_starttimes[idx] except: break if val is None: dummy = lo_sorted_files.pop(idx) dummy = lo_sorted_starttimes.pop(idx) else: idx += 1 #set stationname, either from arguments or from filename if stationname is None: stationname = EDL_get_stationname_fromfilename( lo_sorted_files[0]).upper() #set counting variables - needed for handling of consecutive files #flags, checking, if data has to written to file sameday = True sameblock = True fileopen = False complete = False #numerical index of files for same combination of date and hour fileindex = 0 #current file's daily block number blockindex = 0 #allocate a data array to fill # this is more memory efficient than extending lists!! #cater for potential rounding errors: if sampling < 1: max_n_data = 3600 * int(n_hours) * (int(1. / sampling) + 1) else: max_n_data = int(3600. * int(n_hours) / sampling) + 1 block_data = np.zeros(max_n_data, 'int') #loop over all (sorted) files for the current component for idx_f, f in enumerate(lo_sorted_files): try: print 'Reading file %s' % (f) #starting time of current file file_start_time = lo_sorted_starttimes[idx_f] #get tuple with the starting time of the current file file_start = time.gmtime(file_start_time) #read in raw data data_in = [] Fin = open(f) for line in Fin: #.readlines(): # try: data_in.append(int(float(line.strip()))) # except: # pass data_in = np.array(data_in) Fin.close() #data_in = np.loadtxt(f) except: print 'WARNING - could not read file - skipping...' continue no_samples = len(data_in) #time axis of the file read in : tmp_file_time_axis = np.arange( no_samples) * sampling + file_start_time #end: time of the last sample + 1x sampling-interval file_end_time = tmp_file_time_axis[-1] + sampling #set the current file's starting time as starting time for output file, #if no output file is open already if fileopen is False: #starttime of output file outfile_starttime = file_start_time #output data sample index: arrayindex = 0 #if it's a single column of data if np.size(data_in.shape) == 1: block_data[:len(data_in)] = data_in #otherwise assuming that the first column is time, so just take the second one else: block_data[:len(data_in)] = data_in[:, 1] #jump with index to current point on time axis arrayindex += len(data_in) #current (virtual) end time of open file outfile_endtime = file_end_time #find the date code of the outputfile file_date = '{0}{1:02}{2:02}'.format(file_start[0], file_start[1], file_start[2]) #...aaaand the hour code as well data_hour = file_start[3] #determine, which of the daily data blocks we are currently #processing/writing blockindex = no_blocks - 1 file_hour = lo_hours[-1] for t in range(len(lo_hours) - 1): if lo_hours[t + 1] > data_hour: blockindex = t file_hour = lo_hours[blockindex] break #define output filename new_fn = '{0}_{5}hours_{1}_{2:02d}_{3}.{4}'.format( stationname, file_date, file_hour, fileindex, comp, n_hours) #absolute filename: new_file = op.abspath(op.join(outpath, new_fn)) #open output file F = open(new_file, 'w') #set flag for further loop steps fileopen = True else: #check, if the new file ends earlier than data in buffer. #if yes, just skip this file: if file_end_time < outfile_endtime: continue #if current file starts earlier than the endtime of data in buffer, but extends the time span #then delete ambiguous parts of the buffer: elif (outfile_endtime - file_start_time) > epsilon: #find point on the outfile time axis for the beginning of current file: overlap_idx = arrayindex - int( (outfile_endtime - file_start_time) / sampling) #set the array index back to the appropriate value corresponding to the #start of the new file arrayindex = overlap_idx #append current data #if it's a single column of data if np.size(data_in.shape) == 1: block_data[arrayindex:arrayindex + len(data_in)] = data_in #outfile_data.extend(data_in.tolist()) #otherwise assuming that the first column is time, so just take the second one else: block_data[arrayindex:arrayindex + len(data_in)] = data_in[:, 1] #outfile_data.extend(data_in[:,1].tolist()) #update position in time arrayindex += len(data_in) #update (virtual) end of outfile data outfile_endtime = (arrayindex + 1) * sampling + outfile_starttime #----------- # current file has been read in, data in buffer have been updated # now check, if it has to be written to file.... #check, if there is a next file at all: try: next_file_start_time = lo_sorted_starttimes[idx_f + 1] except: complete = True #if there is a next file, # - check, if it's the same day # - check, if it' the same block # - check, if it continues at the end of the current one: if complete is False: next_file_start_time = lo_sorted_starttimes[idx_f + 1] next_file_start = time.gmtime(next_file_start_time) nextfile_hour = next_file_start[3] nextfile_blockindex = no_blocks - 1 for t in range(len(lo_hours) - 1): if lo_hours[t + 1] > nextfile_hour: nextfile_blockindex = t break nextfile_day = '{0}{1:02}{2:02}'.format( next_file_start[0], next_file_start[1], next_file_start[2]) if nextfile_day != file_date: complete = True sameday = False if nextfile_blockindex != blockindex: complete = True sameblock = False if complete is False: if next_file_start_time - file_end_time > epsilon: complete = True sameblock = True sameday = True #ipdb.set_trace() #check, if the file has to be closed and written now if complete is True: #define header info if outfile_starttime % 1 == 0: outfile_starttime = int(outfile_starttime) headerline = '# {0} {1} {2:.1f} {3} {4} \n'.format( stationname, comp.lower(), 1. / sampling, outfile_starttime, arrayindex) else: headerline = '# {0} {1} {2:.1f} {3:f} {4} \n'.format( stationname, comp.lower(), 1. / sampling, outfile_starttime, arrayindex) F.write(headerline) #outfile_array = np.zeros((len(outfile_timeaxis),2)) #outfile_array[:,0] = outfile_timeaxis #outfile_array[:,1] = outfile_data for i in range(arrayindex): F.write('{0}\n'.format(int(block_data[i]))) #outstring = '\n'.join(['{0:d}'.format(i) for i in day_data[:arrayindex]]) #F.write(outstring) #np.savetxt(F,day_data[:arrayindex],fmt='%d') #np.savetxt(F, np.array(outfile_data)) arrayindex = 0 F.close() print '\t wrote file %s' % (new_file) fileopen = False complete = False #blockindex = (blockindex+1)%no_blocks if sameday is True and sameblock is True: fileindex += 1 else: fileindex = 0
def reorient_files(lo_files, configfile, lo_stations=None, outdir=None): #read config file try: config_dict = MTcf.read_survey_configfile(configfile) except: raise MTex.MTpyError_config_file('Config file cannot be read:' ' {0}'.format(configfile)) if lo_stations is not None: try: if type(lo_stations) == str: raise #check, if it's iterable: dummy = [i for i in lo_stations] except: raise MTex.MTpyError_inputarguments('ERROR - "lo_stations"' ' argument must be iterable!') print '\t re-orienting data for collection of stations:\n{0}'.format( lo_stations) #Do not require list of headers as input, as this function can be called directly rather than from a 'calibratefiles.py'-like script - so the list not necessarily exists in beforehand - #collect header lines of files in list lo_headers = [] lo_stationnames = [] for file_idx, filename in enumerate(lo_files): header = read_ts_header(filename) station = header['station'] if station.upper() not in [i.upper() for i in lo_stations]: #TODO: check, if this causes problems with the indices for the current loop: lo_files.remove(filename) continue lo_headers.append(header) lo_stationnames.append(station.upper()) if len(lo_headers) == 0: if lo_stations is not None: print 'ERROR - No files with header lines found for station(s)'\ ' {0}'.format(lo_stations) else: print 'ERROR - No files with header lines found' return 1 lo_stationnames = list(set(lo_stationnames)) # set up output directory ori_outdir = op.abspath(op.join(os.curdir, 'reoriented')) if outdir is not None: try: ori_outdir = op.abspath(op.join(os.curdir, outdir)) if not op.isdir(ori_outdir): os.makedirs(ori_outdir) except: print 'Output directory cannot be generated: {0} - using generic'\ ' location'.format(ori_outdir) ori_outdir = op.abspath(op.join(os.curdir, 'reoriented')) try: if not op.isdir(ori_outdir): os.makedirs(ori_outdir) except: #this only comes up, if the generic location cannot be generated raise MTex.MTpyError_inputarguments( 'Generic directory cannot be' ' generated: {0}'.format(ori_outdir)) #---------------------- #start re-orientation #present: list of all files, list of all headers, list of all stations for sta_idx, sta in enumerate(lo_stationnames): #print sta try: stationconfig = config_dict[sta] except: print 'Warning - No config file entry for station {0} -'\ ' no processing possible'.format(sta) continue declination = float(stationconfig.get('declination', 0.)) for sensor in ['e', 'b']: #TODO: # reduce this function to the re-orientation of files that have the same length for X and Y. #Do the puzzlling for varying lengths later!! for idx_h_x, header_x in enumerate(lo_headers): #looking for one specific station if not header_x['station'].upper() == sta.upper(): continue #looking for the specific sensor type if not header_x['channel'].lower()[0] == sensor: continue #looking for the X channel (the to-be-North) if not header_x['channel'].lower()[1] == 'x': continue x_file = lo_files[idx_h_x] x_header_string = get_ts_header_string(header_x) t0 = float(header_x['t_min']) #print t0 #now look for the respective y-file and possible z-file - unfortunately by another loop over all headers: y_file = None z_file = None for idx_h_y, header_y in enumerate(lo_headers): if (header_y['station'].upper() == sta.upper()) and \ (header_y['channel'].lower()[0] == sensor) and \ (float(header_y['t_min']) == float(header_x['t_min'] ) ): if (header_y['channel'].lower()[1] == 'y'): y_file = lo_files[idx_h_y] y_header_string = get_ts_header_string(header_y) elif (header_y['channel'].lower()[1] == 'z'): z_file = lo_files[idx_h_y] else: continue if y_file == None: continue x_outfn = op.abspath(op.join(ori_outdir, op.basename(x_file))) y_outfn = op.abspath(op.join(ori_outdir, op.basename(y_file))) if z_file is not None: z_outfn = op.abspath( op.join(ori_outdir, op.basename(z_file))) xdata = np.loadtxt(x_file) ydata = np.loadtxt(y_file) #declination is positive, if magnetic North is east of true North. # the measured angles are w.r.t. magnetic North, so the given # azimuths do not include the declination #-> thus the declination value is added to azimuths if sensor == 'e': xangle = float(stationconfig.get('e_xaxis_azimuth', 0.)) + declination yangle = float(stationconfig.get('e_yaxis_azimuth', 90.)) + declination else: xangle = float(stationconfig.get('b_xaxis_azimuth', 0.)) + declination yangle = float(stationconfig.get('b_yaxis_azimuth', 90.)) + declination newx, newy = MTcc.reorient_data2D(xdata, ydata, x_sensor_angle=xangle, y_sensor_angle=yangle) #print xdata.shape, ydata.shape, newx.shape, newy.shape #continue outFx = open(x_outfn, 'w') outFx.write(x_header_string) np.savetxt(outFx, newx) outFx.close() outFy = open(y_outfn, 'w') outFy.write(y_header_string) np.savetxt(outFy, newy) outFy.close() written_files = [x_outfn, y_outfn] if z_file is not None: shutil.copyfile(z_file, z_outfn) written_files.append(z_outfn) print '\tSuccessfullly written files {0}'.format(written_files) return 0
def calibrate_file(filename, outdir, instrument, instrument_amplification, logger, gain, dipole, stationname, channel, latitude, longitude, elevation, offset = 0 ): """ Calibrate data from one given file and store the output to another file. If the channel is not given explicitly, it's taken from the filename suffix. E field values will be present in microvolt/meter B fields are given in nanotesla input: - data file name - foldername for saving the output - instrument type - instrument amplification factor - data logger type - logger gain factor - station name - channel """ time_axis = None if not instrument.lower() in list_of_instruments: raise MTex.MTpyError_inputarguments('instrument type not known') if not logger.lower() in list_of_loggers: raise MTex.MTpyError_inputarguments('data logger type not known') if not op.isfile(filename): raise MTex.MTpyError_inputarguments('data file not existing') infile_base = op.basename(filename) try: data_in = np.loadtxt(filename) except: raise MTex.MTpyError_inputarguments('cannot read data file') data_out = copy.copy(data_in) #read in first line of input file, checking, if header line exists FH = open(filename,'r') firstline = FH.readline().strip() FH.close() if np.size(data_in.shape) > 1: if data_in.shape[1] > 1: #at least 2 columns - assume, first is time, second data - ignore, if there are more time_axis = data_in[:,0] data_in = data_in[:,1] if not op.isdir(outdir): try: os.makedirs(outdir) except: raise MTex.MTpyError_inputarguments('output directory is not ' 'existing and cannot be generated') if channel == None: channel = filename[-2:].lower() if not channel in list_of_channels: raise MTex.MTpyError_inputarguments('wrong channel specification') field = channel[0] #print 'channel:...........',channel, field #print 'read file',filename ,'wrote file...' #return #separate way for B and E fields here: if field == 'e': if dipole <= 1: print 'Check dipole length value ! - It is highly improbable to '\ 'have a 1 meter dipole!!' answer = raw_input('\t\tContinue anyway? [y/N] \n') if not answer[0].lower() == 'y': sys.exit('Calibration process interrupted by user input!') instrument = 'electrodes' logger = logger.lower() if logger == 'elogger': if not type(gain) in [float, int]:#list_of_elogger_gain_factors: raise MTex.MTpyError_inputarguments('invalid gain for elogger:' ' {0}'.format(gain)) #instrument_amplification = dict_of_efield_amplification[logger] outfile_data = elogger_e_field(data_in, gain, dipole, instrument_amplification) elif logger == 'edl': if not type(gain) in [float, int, str]: raise MTex.MTpyError_inputarguments('invalid gain for EDL: ' '{0}'.format(gain)) #instrument_amplification = dict_of_efield_amplification[logger] if type(gain) == str: EDLgain = dict_of_EDL_gain_factors[gain] else: EDLgain = float(gain) outfile_data = EDL_e_field(data_in, EDLgain, dipole, instrument_amplification) dataunit ='microvoltpermeter' #B-field part elif field == 'b': instrument = instrument.lower() if not instrument in list_of_bfield_instruments: raise MTex.MTpyError_inputarguments('invalid instrument for B-' 'field measurements') logger = logger.lower() if not logger in list_of_bfield_loggers: raise MTex.MTpyError_inputarguments('invalid logger for B-field' ' measurements') #instrument_amplification = 1. #calibration_factor = dict_of_calibration_factors_volt2nanotesla[instrument] if logger == 'edl': if not type(gain) in [float,int,str]: raise MTex.MTpyError_inputarguments('invalid gain: ' '{0}'.format(gain)) if type(gain) == str: EDLgain = dict_of_EDL_gain_factors[gain] else: EDLgain = float(gain) if instrument == 'fluxgate' and channel == 'bz': instrument_amplification *= dict_of_bz_instrument_amplification[logger] outfile_data = EDL_b_field(data_in, EDLgain, instrument, instrument_amplification) dataunit = 'nanotesla' newbasename = '{0}_{1}.{2}'.format(op.splitext(infile_base)[0], dataunit, infile_base.split('.')[-1].lower()) #set up output file outfile = op.join(outdir, newbasename) additional_header_info = ' {0} {1:02.5f} {2:03.5f} {3:.1f} \n'.format( dataunit, latitude, longitude, elevation) if firstline[0][0] == '#': newfirstline = firstline + additional_header_info else: newfirstline = '# {0} {1} {2}'.format(stationname, channel, additional_header_info) if time_axis != None: data_out[:,1] = outfile_data else: data_out = outfile_data Fout = open(outfile,'w') Fout.write(newfirstline) np.savetxt(Fout,data_out,fmt='%.8e') Fout.close() print 'read file',filename ,' -> wrote file %s'%(outfile)
def main(): if len(sys.argv) < 2: sys.exit( '\n\tNeed at least 1 argument:\n\n <path to files>\n[optional:' '<output dir>] \n') outdir = None if len(sys.argv) > 2: outdir = sys.argv[2] # if len(sys.argv) > 3: # network = sys.argv[3] # if len(sys.argv) > 4: # location = sys.argv[4] pathname_raw = sys.argv[1] #we need relative paths here!!! indir = pathname_raw #op.abspath(op.realpath(pathname_raw)) if not op.isdir(indir): raise MTex.MTpyError_inputarguments( 'Data file(s) path not existing: {0}'.format(indir)) #define output directory for storing miniSeed files #outpath = op.join(os.curdir,'miniSeed') if outdir is not None: try: outpath = op.abspath(op.join(os.curdir, outdir)) if not op.exists(outpath): try: os.makedirs(outpath) except: raise if not os.access(outpath, os.W_OK): raise except: print 'Cannot generate writable output directory {0} - using generic'\ ' location "ascii" instead'.format(outpath) outdir = None if outdir is None: outpath = op.join(os.curdir, 'ascii') try: if not op.exists(outpath): try: os.makedirs(outpath) except: raise if not os.access(outpath, os.W_OK): raise except: sys.exit('Error ! - Cannot generate writable output directory ' '"ascii" - abort...') outdir = op.abspath(outpath) lo_dirs = [] for i, j, k in os.walk(indir): lofolders = [op.join(i, f) for f in j] lo_dirs.extend(lofolders) lo_dirs.append(indir) pathname = list(set(lo_dirs)) if len(pathname) == 0: pathname = [indir] lo_indirs = pathname lo_outdirs = [] #'pathname' is a list of relative pathnames. to be reconstructed under the given 'outdir' try: for i in lo_indirs: outpath = op.abspath(op.join(outdir, i)) if not op.isdir(outpath): os.makedirs(outpath) lo_outdirs.append(outpath) except: raise MTex.MTpyError_inputarguments( 'ERROR - Cannot set up output directory {0}'.format(outpath)) for idx_ipath, inpath in enumerate(lo_indirs): lo_infiles = [ i for i in os.listdir(inpath) if op.isfile(op.abspath(op.join(inpath, i))) ] lo_outfiles = [ op.abspath(op.join(lo_outdirs[idx_ipath], i)) for i in lo_infiles ] lo_infiles = [op.abspath(op.join(inpath, i)) for i in lo_infiles] for idx_fn, fn in enumerate(lo_infiles): print 'reading file {0}'.format(fn) try: outfn = MTms.convertfile_miniseed2ts(fn, lo_outfiles[idx_fn]) print 'wrote file(s) {0}'.format(outfn) except: print 'Warning - file {0} is not in valid miniseed format!!!'.format( fn) continue