def load_ts_obj(self, ts_fn): """ load an MT file """ if isinstance(ts_fn, str): ext = os.path.splitext(ts_fn)[-1][1:].lower() if ext == 'z3d': self.logger.info('Opening Z3D file {0}'.format(ts_fn)) z3d_obj = zen.Zen3D(ts_fn) z3d_obj.read_z3d() ts_obj = z3d_obj.ts_obj elif ext in ['ex', 'ey', 'hx', 'hy', 'hz']: self.logger.info('Opening ascii file {0}'.format(ts_fn)) ts_obj = mtts.MTTS() ts_obj.read_file(ts_fn) elif ext in ['bnn', 'bin']: self.logger.info('Opening NIMS file {0}'.format(ts_fn)) nims_obj = nims.NIMS(ts_fn) ts_obj = [ nims_obj.hx, nims_obj.hy, nims_obj.hz, nims_obj.ex, nims_obj.ey ] elif isinstance(ts_fn, mtts.MTTS): ts_obj = ts_fn self.logger.info('Loading MT object') else: raise mtts.MTTSError("Do not understand {0}".format(type(ts_fn))) return ts_obj
def get_z3d_info(self, z3d_fn_list, calibration_path=None): """ Get general z3d information and put information in a dataframe :param z3d_fn_list: List of files Paths to z3d files :type z3d_fn_list: list :return: Dataframe of z3d information :rtype: Pandas.DataFrame :Example: :: >>> zc_obj = zc.Z3DCollection(r"/home/z3d_files") >>> z3d_fn_list = zc.get_z3d_fn_list() >>> z3d_df = zc.get_z3d_info(z3d_fn_list) >>> # write dataframe to a file to use later >>> z3d_df.to_csv(r"/home/z3d_files/z3d_info.csv") """ if len(z3d_fn_list) < 1: raise ValueError('No Z3D files found') cal_dict = self.get_calibrations(calibration_path) z3d_info_list = [] for z3d_fn in z3d_fn_list: z3d_obj = zen.Zen3D(z3d_fn) z3d_obj.read_all_info() z3d_obj.start = z3d_obj.zen_schedule.isoformat() # set some attributes to null to fill later z3d_obj.stop = None z3d_obj.n_samples = 0 z3d_obj.fn_ascii = None z3d_obj.block = 0 z3d_obj.remote = False z3d_obj.zen_num = 'ZEN{0:03.0f}'.format(z3d_obj.header.box_number) try: z3d_obj.cal_fn = cal_dict[z3d_obj.coil_num] except KeyError: z3d_obj.cal_fn = 0 # make a dictionary of values to put into data frame entry = dict([(key, getattr(z3d_obj, value)) for key, value in self._keys_dict.items()]) z3d_info_list.append(entry) # make pandas dataframe and set data types z3d_df = pd.DataFrame(z3d_info_list) z3d_df = z3d_df.astype(self._dtypes) z3d_df.start = pd.to_datetime(z3d_df.start, errors='coerce') z3d_df.stop = pd.to_datetime(z3d_df.stop, errors='coerce') # assign block numbers for sr in z3d_df.sampling_rate.unique(): starts = sorted(z3d_df[z3d_df.sampling_rate == sr].start.unique()) for block_num, start in enumerate(starts): z3d_df.loc[(z3d_df.start == start), 'block'] = block_num return z3d_df
def str_save(): import mtpy.usgs.zen as zen import numpy as np z1 = zen.Zen3D( r"d:\Peacock\MTData\MonoBasin\MB_June2015\mb300\mb300_20150610_070056_4096_EY.Z3D" ) z1.read_z3d() l_header = ','.join( list(('256.0', '1470097901', str(len(z1.time_series)), '.2345', '.1230', '10.'))) ls = z1.convert_counts() / ((100 / 100.) * (2 * np.pi)) ls = ls.astype('S18') with open(r"d:\Peacock\test_4.ex", 'w') as fid: fid.write(l_header + '\n') fid.write('\n'.join(ls))
def str_save(): import mtpy.usgs.zen as zen import numpy as np z1 = zen.Zen3D( r"d:\Peacock\MTData\MonoBasin\MB_June2015\mb300\mb300_20150610_070056_4096_EY.Z3D" ) z1.read_z3d() l_header = ",".join( list(("256.0", "1470097901", str(len(z1.time_series)), ".2345", ".1230", "10."))) ls = z1.convert_counts() / ((100 / 100.0) * (2 * np.pi)) ls = ls.astype("S18") with open(r"d:\Peacock\test_4.ex", "w") as fid: fid.write(l_header + "\n") fid.write("\n".join(ls))
def get_z3d_info(z3d_path): """ get information on z3d files """ if not isinstance(z3d_path, Path): z3d_path = Path(z3d_path) # need to get all the files for one channel fn_dict = dict([(key, []) for key in ['ex', 'ey', 'hx', 'hy', 'hz']]) # get all z3d files within a given folder, will look through recursively fn_list = [ fn_path for fn_path in z3d_path.rglob('*') if fn_path.suffix in ['.z3d', '.Z3D'] ] # loop over files, read just the metadata and get important information for fn in fn_list: z_obj = zen.Zen3D(fn) z_obj.read_all_info() fn_dict[z_obj.component].append({ 'start': z_obj.zen_schedule.isoformat(), 'df': z_obj.df, 'fn': z_obj.fn }) return fn_dict
comps = ["ex", "ey", "hx", "hy"] colors = [ (28 / 255, 146 / 255, 205 / 255), (28 / 255, 52 / 255, 205 / 255), (203 / 255, 158 / 255, 33 / 255), (203 / 255, 61 / 255, 33 / 255), ] fig = plt.figure(1, dpi=150) fig.clf() ax = fig.add_subplot(1, 1, 1) p_dict = {"fs": 256, "nperseg": 2 ** 13} line_list = [] for ii, comp, c in zip(range(len(comps)), comps, colors): fn = Path(f"{fn_stem}{comp.upper()}.Z3D") z1 = zen.Zen3D(fn) z1.read_z3d() z1.apply_adaptive_notch_filter() b = z1.plot_spectrogram(plot_type="all") b.save_figure(save_path.joinpath(f"{fn.stem}.png").as_posix(), fig_dpi=150) f, p = signal.welch(z1.ts_obj.ts.data, **p_dict) (l1,) = ax.loglog(f, p, lw=1.75, color=c) l1.label = comp.capitalize() line_list.append(l1) ax.set_xlabel("Frequency (Hz)", fontdict={"size": 10, "weight": "bold"}) ax.set_ylabel("Power (dB)", fontdict={"size": 10, "weight": "bold"}) ax.axis("tight")
# -*- coding: utf-8 -*- """ Created on Fri May 29 15:20:12 2015 @author: jpeacock-pr """ import mtpy.usgs.zen as zen import numpy as np import struct import os station_dir = r"d:\Peacock\MTData\Test\mb666" fn_list = [ os.path.join(station_dir, fn) for fn in os.listdir(station_dir) if fn.find('165016') > 0 ] zt_list = [] for fn in fn_list: zt = zen.Zen3D(fn) zt.read_z3d() zt_list.append(zt)
# ============================================================================== fn = r"d:\Peacock\MTData\Umatilla\um102\um102_20170607_070018_4096_HY.Z3D" # noise period, this is something you will have to find, should make a # function to do it automatically noise_per = 5 pad_edge = 0.10 pad_width = 0.03 # ============================================================================== # Load time series data into TS format # ============================================================================== st = datetime.datetime.utcnow() z1 = zen.Zen3D() z1.read_z3d(fn) # number of samples n = z1.ts_obj.ts.data.size # relative time array to correspond with data t_arr = np.arange(0, n / z1.ts_obj.sampling_rate, 1.0 / z1.ts_obj.sampling_rate) # ============================================================================== # Window data and average to get the shape of 1 pulse from the pipeline # ============================================================================== # set window length, initialize an average window array for statistical # analysis later window_len = int(noise_per * z1.ts_obj.sampling_rate)
data_type = np.dtype([(st, dt) for st, dt in zip(stamp_lst, data_types)]) block_seconds = 8*60*60 #256 for 8 hours block_df = 256 master_schedule = ['00:00:00', '08:00:00', '16:00:00'] #number of maximum samples per data block after down sampling block_len = (block_df)*block_seconds for z_fn in os.listdir(dirpath): z1 = zen.Zen3D(z_fn) z1.get_info() #round the time to the nearest hour t1 = time.mktime(time.strptime(z1.start_dt, dt_fmt))) #create file instance rfid2 = file(z_fn, 'rb') #--> get header header2 = rfid2.read(header_len) #--> find the first gps stamp gps_find = -1 gt2 = '' zt = zen.Zen3D() while gt2 != sdt_lst[0]:
sdate = time.strftime("%Y%m%d", dt3) stime = time.strftime("%H%M%S", dt3) sdt = time.strftime("%Y-%m-%d,%H:%M:%S", dt3) sdth_lst.append(sdt) t3 += bsec # create file instance rfid2 = file(rr_fn2, "rb") # --> get header header2 = rfid2.read(header_len) # --> find the first gps stamp gps_find = -1 gt2 = "" zt = zen.Zen3D() while gt2 != sdt_lst[0]: test_string = rfid2.read((df0 / 2 + 9) * 4) zt = zen.Zen3D() zt._raw_data = test_string gps_index = zt.get_gps_stamp_location() if gps_index != -1: if gps_index >= (df0 / 2 + 9) * 4 - 36: print "reading extra" zt._raw_data += rfid2.read(gps_stamp_len) test_stamp = zt.get_gps_stamp(gps_index)[0] gt2 = zt.get_date_time(gps_week, test_stamp["time"]) else: pass
def write_cache_file(self, fn_list, save_fn, station='ZEN', decimate=1): """ write a cache file from given filenames """ #sort the files so they are in order fn_sort_list = [] for cs in self.chn_order: for fn in fn_list: if cs in fn.lower(): fn_sort_list.append(fn) fn_list = fn_sort_list print(fn_list) n_fn = len(fn_list) self.zt_list = [] for fn in fn_list: zt1 = zen.Zen3D(fn=fn) zt1.verbose = self.verbose try: zt1.read_3d() except ZenGPSError: zt1._seconds_diff = 59 zt1.read_3d() self.zt_list.append(zt1) #fill in meta data from the time series file self.meta_data['DATA.DATE0'] = ',' + zt1.date_time[0].split(',')[0] self.meta_data['DATA.TIME0'] = ',' + zt1.date_time[0].split(',')[1] self.meta_data['TS.ADFREQ'] = ',{0}'.format(int(zt1.df)) self.meta_data['CH.FACTOR'] += ',' + self._ch_factor self.meta_data['CH.GAIN'] += ',' + self._ch_gain self.meta_data['CH.CMP'] += ',' + zt1.ch_cmp.upper() self.meta_data['CH.LENGTH'] += ',' + zt1.ch_length self.meta_data['CH.EXTGAIN'] += ',1' self.meta_data['CH.NOTCH'] += ',NONE' self.meta_data['CH.HIGHPASS'] += ',NONE' self.meta_data['CH.LOWPASS'] += ','+\ self._ch_lowpass_dict[str(int(zt1.df))] self.meta_data['CH.ADCARDSN'] += ',' + zt1.ch_adcard_sn self.meta_data['CH.NUMBER'] += ',{0}'.format(zt1.ch_number) self.meta_data['RX.STN'] += ',' + zt1.rx_stn #make sure all files have the same sampling rate self.check_sampling_rate(self.zt_list) #make sure the length of time series is the same for all channels self.ts, ts_len = self.check_time_series(self.zt_list, decimate=decimate) self.meta_data['TS.NPNT'] = ',{0}'.format(ts_len) #get the file name to save to if save_fn[-4:] == '.cac': self.save_fn = save_fn elif save_fn[-4] == '.': raise ZenInputFileError('File extension needs to be .cac, not'+\ save_fn[-4:]) else: general_fn = station+'_'+\ self.meta_data['DATA.DATE0'][1:].replace('-','')+\ '_'+self.meta_data['DATA.TIME0'][1:].replace(':','')+\ '_'+self.meta_data['TS.ADFREQ'][1:]+'.cac' if os.path.basename(save_fn) != 'Merged': save_fn = os.path.join(save_fn, 'Merged') if not os.path.exists(save_fn): os.mkdir(save_fn) self.save_fn = os.path.join(save_fn, general_fn) cfid = file(self.save_fn, 'wb+') #--> write navigation records first cfid.write(struct.pack('<i', self._nav_len)) cfid.write(struct.pack('<i', self._flag)) cfid.write(struct.pack('<h', self._type_dict['nav'])) for nd in range(self._nav_len - 2): cfid.write(struct.pack('<b', 0)) cfid.write(struct.pack('<i', self._nav_len)) #--> write meta data meta_str = ''.join([ key + self.meta_data[key] + '\n' for key in np.sort(list(self.meta_data.keys())) ]) meta_len = len(meta_str) cfid.write(struct.pack('<i', meta_len + 2)) cfid.write(struct.pack('<i', self._flag)) cfid.write(struct.pack('<h', self._type_dict['meta'])) cfid.write(meta_str) cfid.write(struct.pack('<i', meta_len + 2)) #--> write calibrations cal_data1 = 'HEADER.TYPE,Calibrate\nCAL.VER,019\nCAL.SYS,0000,'+\ ''.join([' 0.000000: '+'0.000000 0.000000,'*3]*27) cal_data2 = '\nCAL.SYS,0000,'+\ ''.join([' 0.000000: '+'0.000000 0.000000,'*3]*27) cal_data = cal_data1 + (cal_data2 * (n_fn - 1)) cal_len = len(cal_data) cfid.write(struct.pack('<i', cal_len + 2)) cfid.write(struct.pack('<i', self._flag)) cfid.write(struct.pack('<h', self._type_dict['cal'])) cfid.write(cal_data[:-1] + '\n') cfid.write(struct.pack('<i', cal_len + 2)) #--> write data ts_block_len = int(ts_len) * n_fn * 4 + 2 #--> Need to scale the time series into counts cause that is apparently # what MTFT24 expects self.ts = self.ts.astype(np.int32) #--> make sure none of the data is above the allowed level self.ts[np.where(self.ts > 2.14e9)] = 2.14e9 self.ts[np.where(self.ts < -2.14e9)] = -2.14e9 #--> write time series block cfid.write(struct.pack('<i', ts_block_len)) cfid.write(struct.pack('<i', self._flag)) cfid.write(struct.pack('<h', self._type_dict['ts'])) #--> need to pack the data as signed integers for zz in range(ts_len): cfid.write(struct.pack('<' + 'i' * n_fn, *self.ts[zz])) cfid.write(struct.pack('<i', ts_block_len)) cfid.close() if self.verbose: print('Saved File to: ', self.save_fn) self.log_lines.append('=' * 72 + '\n') self.log_lines.append('Saved File to: \n') self.log_lines.append(' ' * 4 + '{0}\n'.format(self.save_fn)) self.log_lines.append('=' * 72 + '\n')
import glob import json from mtpy.usgs import zen survey_dir = r"d:\Peacock\MTData\SCEC" survey_dict = {} for station in os.listdir(survey_dir): station_dir = os.path.join(survey_dir, station) if os.path.isdir(station_dir): z3d_list = glob.glob(os.path.join(station_dir, "*.Z3D")) if len(z3d_list) == 0: continue for z3d_fn in z3d_list: z3d_obj = zen.Zen3D(fn=z3d_fn) z3d_obj.read_all_info() dt_key = z3d_obj.zen_schedule.isoformat() if not dt_key in survey_dict.keys(): survey_dict[dt_key] = {} if not z3d_obj.station in survey_dict[dt_key].keys(): survey_dict[dt_key][z3d_obj.station] = {"comp": [], "df": [], "azm": []} survey_dict[dt_key][z3d_obj.station]["comp"].append(z3d_obj.component) survey_dict[dt_key][z3d_obj.station]["df"].append(z3d_obj.df) survey_dict[dt_key][z3d_obj.station]["azm"].append(z3d_obj.azimuth) with open(os.path.join(survey_dir, "survey_summary.txt"), "w") as fid: json.dump(survey_dict, fid) lines = [] for d_key in sorted(list(survey_dict.keys())):
def combine_z3d_files(z3d_path, new_sampling_rate=4, t_buffer=8 * 3600, comp_list=['ex', 'ey', 'hx', 'hy', 'hz']): """ Combine all z3d files for a given station and given component for processing and getting the long period estimations. :param str z3d_path: full path to z3d files :param str component: component to combine :param int new_sampling_rate: new sampling rate of the data :param int t_buffer: buffer for the last time series, should be length of longest schedule chunk """ st = datetime.datetime.now() attr_list = [ "station", "channel_number", "component", "coordinate_system", "dipole_length", "azimuth", "units", "lat", "lon", "elev", "datum", "data_logger", "instrument_id", "calibration_fn", "declination", "fn", "conversion", "gain", ] fn_df = get_z3d_info(z3d_path) return_fn_list = [] for comp in comp_list: if len(fn_df[comp]) == 0: print('Warning: Skipping {0} because no Z3D files found.'.format( comp)) continue comp_df = pd.DataFrame(fn_df[comp]) # sort the data frame by date comp_df = comp_df.sort_values('start') # get start date and end at last start date, get time difference start_dt = datetime.datetime.fromisoformat(comp_df.start.min()) end_dt = datetime.datetime.fromisoformat(comp_df.start.max()) t_diff = (end_dt - start_dt).total_seconds() ### make a new MTTS object that will have a length that is buffered ### at the end to make sure there is room for the data, will trimmed new_ts = ts.MTTS() new_ts.ts = np.zeros(int((t_diff + t_buffer) * sampling_rate)) new_ts.sampling_rate = sampling_rate new_ts.start_time_utc = start_dt # make an attribute dictionary that can be used to fill in the new # MTTS object attr_dict = dict([(key, []) for key in attr_list]) # loop over each z3d file for the given component for row in comp_df.itertuples(): z_obj = zen.Zen3D(row.fn) print(row.fn) z_obj.read_z3d() t_obj = z_obj.ts_obj # decimate to the required sampling rate t_obj.decimate(int(z_obj.df / sampling_rate)) # fill the new time series with the data at the appropriate times print(f"start = {t_obj.ts.index[0]}, end = {t_obj.ts.index[-1]}") new_ts.ts.data[(new_ts.ts.index >= t_obj.ts.index[0]) & ( new_ts.ts.index <= t_obj.ts.index[-1])] = t_obj.ts.data # get the end date as the last z3d file end_date = z_obj.ts_obj.ts.index[-1] # fill attribute data frame for attr in attr_list: attr_dict[attr].append(getattr(t_obj, attr)) # need to trim the data new_ts.ts = new_ts.ts.data[(new_ts.ts.index >= start_dt) & (new_ts.ts.index <= end_date)].to_frame() # fill gaps with forwards or backwards values, this seems to work # better than interpolation and is faster than regression. # The gaps should be max 13 seconds if everything went well new_ts.ts.data[new_ts.ts.data == 0] = np.nan new_ts.ts.data.fillna(method='ffill', inplace=True) # fill the new MTTS with the appropriate metadata attr_df = pd.DataFrame(attr_dict) for attr in attr_list: try: attr_series = attr_df[attr][attr_df[attr] != 0] try: setattr(new_ts, attr, attr_series.median()) except TypeError: setattr(new_ts, attr, attr_series.mode()[0]) except ValueError: print('Warning: could not set {0}'.format(attr)) ascii_fn = '{0}_combined_{1}.{2}'.format(new_ts.station, int(new_ts.sampling_rate), new_ts.component) sv_fn_ascii = z3d_path.joinpath(ascii_fn) new_ts.write_ascii_file(sv_fn_ascii.absolute()) return_fn_list.append(sv_fn_ascii) et = datetime.datetime.now() compute_time = (et - st).total_seconds() print(" Combining took {0:.2f} seconds".format(compute_time)) return return_fn_list
t_diff = (end_dt - start_dt).total_seconds() # make a new MTTS object that will have a length that is buffered # at the end to make sure there is room for the data, will trimmed new_ts = ts.MTTS() new_ts.ts = np.zeros(int((t_diff + t_buffer) * sampling_rate)) new_ts.sampling_rate = sampling_rate new_ts.start_time_utc = start_dt # make an attribute dictionary that can be used to fill in the new # MTTS object attr_dict = dict([(key, []) for key in attr_list]) # loop over each z3d file for the given component index = 1 for row in comp_df.itertuples(): z_obj = zen.Zen3D(row.fn) print(row) z_obj.read_z3d() t_obj = z_obj.ts_obj # decimate to the required sampling rate t_obj.decimate(int(z_obj.df / sampling_rate)) # fill the new time series with the data at the appropriate times print(f"start = {t_obj.ts.index[0]}, end = {t_obj.ts.index[-1]}") new_ts.ts.data[(new_ts.ts.index >= t_obj.ts.index[0]) & (new_ts.ts.index <= t_obj.ts.index[-1])] = t_obj.ts.data plt.figure(index) plt.plot(new_ts.ts) plt.plot(t_obj.ts) # get the end date as the last z3d file end_date = z_obj.ts_obj.ts.index[-1]
256: datetime.timedelta(hours=7, minutes=45), 4096: datetime.timedelta(minutes=15), 1024: datetime.timedelta(minutes=10), } if not csv_path.exists(): schedule_dict = "" date_dict = [] for station in survey_path.glob("**/*"): station_path = Path.joinpath(survey_path.parent, survey_path.name, station) for z3d_fn in station_path.glob("*.Z3D"): z_obj = zen.Zen3D( Path.joinpath(station_path.parent, station_path.name, z3d_fn) ) try: z_obj.read_all_info() except zen.ZenGPSError: continue entry = { "station": z_obj.station, "start_date": z_obj.zen_schedule.isoformat(), "sampling_rate": z_obj.df, } date_dict.append(entry) df = pd.DataFrame(date_dict) df.drop_duplicates(["station", "start_date"], inplace=True)