def load_ts_obj(self, ts_fn):
"""
load an MT file
"""
if isinstance(ts_fn, str):
ext = os.path.splitext(ts_fn)[-1][1:].lower()
if ext == 'z3d':
self.logger.info('Opening Z3D file {0}'.format(ts_fn))
z3d_obj = zen.Zen3D(ts_fn)
z3d_obj.read_z3d()
ts_obj = z3d_obj.ts_obj
elif ext in ['ex', 'ey', 'hx', 'hy', 'hz']:
self.logger.info('Opening ascii file {0}'.format(ts_fn))
ts_obj = mtts.MTTS()
ts_obj.read_file(ts_fn)
elif ext in ['bnn', 'bin']:
self.logger.info('Opening NIMS file {0}'.format(ts_fn))
nims_obj = nims.NIMS(ts_fn)
ts_obj = [
nims_obj.hx, nims_obj.hy, nims_obj.hz, nims_obj.ex,
nims_obj.ey
]
elif isinstance(ts_fn, mtts.MTTS):
ts_obj = ts_fn
self.logger.info('Loading MT object')
else:
raise mtts.MTTSError("Do not understand {0}".format(type(ts_fn)))
return ts_obj
def get_z3d_info(self, z3d_fn_list, calibration_path=None):
"""
Get general z3d information and put information in a dataframe
:param z3d_fn_list: List of files Paths to z3d files
:type z3d_fn_list: list
:return: Dataframe of z3d information
:rtype: Pandas.DataFrame
:Example: ::
>>> zc_obj = zc.Z3DCollection(r"/home/z3d_files")
>>> z3d_fn_list = zc.get_z3d_fn_list()
>>> z3d_df = zc.get_z3d_info(z3d_fn_list)
>>> # write dataframe to a file to use later
>>> z3d_df.to_csv(r"/home/z3d_files/z3d_info.csv")
"""
if len(z3d_fn_list) < 1:
raise ValueError('No Z3D files found')
cal_dict = self.get_calibrations(calibration_path)
z3d_info_list = []
for z3d_fn in z3d_fn_list:
z3d_obj = zen.Zen3D(z3d_fn)
z3d_obj.read_all_info()
z3d_obj.start = z3d_obj.zen_schedule.isoformat()
# set some attributes to null to fill later
z3d_obj.stop = None
z3d_obj.n_samples = 0
z3d_obj.fn_ascii = None
z3d_obj.block = 0
z3d_obj.remote = False
z3d_obj.zen_num = 'ZEN{0:03.0f}'.format(z3d_obj.header.box_number)
try:
z3d_obj.cal_fn = cal_dict[z3d_obj.coil_num]
except KeyError:
z3d_obj.cal_fn = 0
# make a dictionary of values to put into data frame
entry = dict([(key, getattr(z3d_obj, value)) for key, value in
self._keys_dict.items()])
z3d_info_list.append(entry)
# make pandas dataframe and set data types
z3d_df = pd.DataFrame(z3d_info_list)
z3d_df = z3d_df.astype(self._dtypes)
z3d_df.start = pd.to_datetime(z3d_df.start, errors='coerce')
z3d_df.stop = pd.to_datetime(z3d_df.stop, errors='coerce')
# assign block numbers
for sr in z3d_df.sampling_rate.unique():
starts = sorted(z3d_df[z3d_df.sampling_rate == sr].start.unique())
for block_num, start in enumerate(starts):
z3d_df.loc[(z3d_df.start == start), 'block'] = block_num
return z3d_df
def str_save():
import mtpy.usgs.zen as zen
import numpy as np
z1 = zen.Zen3D(
r"d:\Peacock\MTData\MonoBasin\MB_June2015\mb300\mb300_20150610_070056_4096_EY.Z3D"
)
z1.read_z3d()
l_header = ','.join(
list(('256.0', '1470097901', str(len(z1.time_series)), '.2345',
'.1230', '10.')))
ls = z1.convert_counts() / ((100 / 100.) * (2 * np.pi))
ls = ls.astype('S18')
with open(r"d:\Peacock\test_4.ex", 'w') as fid:
fid.write(l_header + '\n')
fid.write('\n'.join(ls))
def str_save():
import mtpy.usgs.zen as zen
import numpy as np
z1 = zen.Zen3D(
r"d:\Peacock\MTData\MonoBasin\MB_June2015\mb300\mb300_20150610_070056_4096_EY.Z3D"
)
z1.read_z3d()
l_header = ",".join(
list(("256.0", "1470097901", str(len(z1.time_series)), ".2345",
".1230", "10.")))
ls = z1.convert_counts() / ((100 / 100.0) * (2 * np.pi))
ls = ls.astype("S18")
with open(r"d:\Peacock\test_4.ex", "w") as fid:
fid.write(l_header + "\n")
fid.write("\n".join(ls))
def get_z3d_info(z3d_path):
"""
get information on z3d files
"""
if not isinstance(z3d_path, Path):
z3d_path = Path(z3d_path)
# need to get all the files for one channel
fn_dict = dict([(key, []) for key in ['ex', 'ey', 'hx', 'hy', 'hz']])
# get all z3d files within a given folder, will look through recursively
fn_list = [
fn_path for fn_path in z3d_path.rglob('*')
if fn_path.suffix in ['.z3d', '.Z3D']
]
# loop over files, read just the metadata and get important information
for fn in fn_list:
z_obj = zen.Zen3D(fn)
z_obj.read_all_info()
fn_dict[z_obj.component].append({
'start': z_obj.zen_schedule.isoformat(),
'df': z_obj.df,
'fn': z_obj.fn
})
return fn_dict
comps = ["ex", "ey", "hx", "hy"]
colors = [
(28 / 255, 146 / 255, 205 / 255),
(28 / 255, 52 / 255, 205 / 255),
(203 / 255, 158 / 255, 33 / 255),
(203 / 255, 61 / 255, 33 / 255),
]
fig = plt.figure(1, dpi=150)
fig.clf()
ax = fig.add_subplot(1, 1, 1)
p_dict = {"fs": 256, "nperseg": 2 ** 13}
line_list = []
for ii, comp, c in zip(range(len(comps)), comps, colors):
fn = Path(f"{fn_stem}{comp.upper()}.Z3D")
z1 = zen.Zen3D(fn)
z1.read_z3d()
z1.apply_adaptive_notch_filter()
b = z1.plot_spectrogram(plot_type="all")
b.save_figure(save_path.joinpath(f"{fn.stem}.png").as_posix(), fig_dpi=150)
f, p = signal.welch(z1.ts_obj.ts.data, **p_dict)
(l1,) = ax.loglog(f, p, lw=1.75, color=c)
l1.label = comp.capitalize()
line_list.append(l1)
ax.set_xlabel("Frequency (Hz)", fontdict={"size": 10, "weight": "bold"})
ax.set_ylabel("Power (dB)", fontdict={"size": 10, "weight": "bold"})
ax.axis("tight")
# -*- coding: utf-8 -*-
"""
Created on Fri May 29 15:20:12 2015
@author: jpeacock-pr
"""
import mtpy.usgs.zen as zen
import numpy as np
import struct
import os
station_dir = r"d:\Peacock\MTData\Test\mb666"
fn_list = [
os.path.join(station_dir, fn) for fn in os.listdir(station_dir)
if fn.find('165016') > 0
]
zt_list = []
for fn in fn_list:
zt = zen.Zen3D(fn)
zt.read_z3d()
zt_list.append(zt)
# ==============================================================================
fn = r"d:\Peacock\MTData\Umatilla\um102\um102_20170607_070018_4096_HY.Z3D"
# noise period, this is something you will have to find, should make a
# function to do it automatically
noise_per = 5
pad_edge = 0.10
pad_width = 0.03
# ==============================================================================
# Load time series data into TS format
# ==============================================================================
st = datetime.datetime.utcnow()
z1 = zen.Zen3D()
z1.read_z3d(fn)
# number of samples
n = z1.ts_obj.ts.data.size
# relative time array to correspond with data
t_arr = np.arange(0, n / z1.ts_obj.sampling_rate,
1.0 / z1.ts_obj.sampling_rate)
# ==============================================================================
# Window data and average to get the shape of 1 pulse from the pipeline
# ==============================================================================
# set window length, initialize an average window array for statistical
# analysis later
window_len = int(noise_per * z1.ts_obj.sampling_rate)
data_type = np.dtype([(st, dt) for st, dt in
zip(stamp_lst, data_types)])
block_seconds = 8*60*60 #256 for 8 hours
block_df = 256
master_schedule = ['00:00:00', '08:00:00', '16:00:00']
#number of maximum samples per data block after down sampling
block_len = (block_df)*block_seconds
for z_fn in os.listdir(dirpath):
z1 = zen.Zen3D(z_fn)
z1.get_info()
#round the time to the nearest hour
t1 = time.mktime(time.strptime(z1.start_dt, dt_fmt)))
#create file instance
rfid2 = file(z_fn, 'rb')
#--> get header
header2 = rfid2.read(header_len)
#--> find the first gps stamp
gps_find = -1
gt2 = ''
zt = zen.Zen3D()
while gt2 != sdt_lst[0]:
sdate = time.strftime("%Y%m%d", dt3)
stime = time.strftime("%H%M%S", dt3)
sdt = time.strftime("%Y-%m-%d,%H:%M:%S", dt3)
sdth_lst.append(sdt)
t3 += bsec
# create file instance
rfid2 = file(rr_fn2, "rb")
# --> get header
header2 = rfid2.read(header_len)
# --> find the first gps stamp
gps_find = -1
gt2 = ""
zt = zen.Zen3D()
while gt2 != sdt_lst[0]:
test_string = rfid2.read((df0 / 2 + 9) * 4)
zt = zen.Zen3D()
zt._raw_data = test_string
gps_index = zt.get_gps_stamp_location()
if gps_index != -1:
if gps_index >= (df0 / 2 + 9) * 4 - 36:
print "reading extra"
zt._raw_data += rfid2.read(gps_stamp_len)
test_stamp = zt.get_gps_stamp(gps_index)[0]
gt2 = zt.get_date_time(gps_week, test_stamp["time"])
else:
pass
def write_cache_file(self, fn_list, save_fn, station='ZEN', decimate=1):
"""
write a cache file from given filenames
"""
#sort the files so they are in order
fn_sort_list = []
for cs in self.chn_order:
for fn in fn_list:
if cs in fn.lower():
fn_sort_list.append(fn)
fn_list = fn_sort_list
print(fn_list)
n_fn = len(fn_list)
self.zt_list = []
for fn in fn_list:
zt1 = zen.Zen3D(fn=fn)
zt1.verbose = self.verbose
try:
zt1.read_3d()
except ZenGPSError:
zt1._seconds_diff = 59
zt1.read_3d()
self.zt_list.append(zt1)
#fill in meta data from the time series file
self.meta_data['DATA.DATE0'] = ',' + zt1.date_time[0].split(',')[0]
self.meta_data['DATA.TIME0'] = ',' + zt1.date_time[0].split(',')[1]
self.meta_data['TS.ADFREQ'] = ',{0}'.format(int(zt1.df))
self.meta_data['CH.FACTOR'] += ',' + self._ch_factor
self.meta_data['CH.GAIN'] += ',' + self._ch_gain
self.meta_data['CH.CMP'] += ',' + zt1.ch_cmp.upper()
self.meta_data['CH.LENGTH'] += ',' + zt1.ch_length
self.meta_data['CH.EXTGAIN'] += ',1'
self.meta_data['CH.NOTCH'] += ',NONE'
self.meta_data['CH.HIGHPASS'] += ',NONE'
self.meta_data['CH.LOWPASS'] += ','+\
self._ch_lowpass_dict[str(int(zt1.df))]
self.meta_data['CH.ADCARDSN'] += ',' + zt1.ch_adcard_sn
self.meta_data['CH.NUMBER'] += ',{0}'.format(zt1.ch_number)
self.meta_data['RX.STN'] += ',' + zt1.rx_stn
#make sure all files have the same sampling rate
self.check_sampling_rate(self.zt_list)
#make sure the length of time series is the same for all channels
self.ts, ts_len = self.check_time_series(self.zt_list,
decimate=decimate)
self.meta_data['TS.NPNT'] = ',{0}'.format(ts_len)
#get the file name to save to
if save_fn[-4:] == '.cac':
self.save_fn = save_fn
elif save_fn[-4] == '.':
raise ZenInputFileError('File extension needs to be .cac, not'+\
save_fn[-4:])
else:
general_fn = station+'_'+\
self.meta_data['DATA.DATE0'][1:].replace('-','')+\
'_'+self.meta_data['DATA.TIME0'][1:].replace(':','')+\
'_'+self.meta_data['TS.ADFREQ'][1:]+'.cac'
if os.path.basename(save_fn) != 'Merged':
save_fn = os.path.join(save_fn, 'Merged')
if not os.path.exists(save_fn):
os.mkdir(save_fn)
self.save_fn = os.path.join(save_fn, general_fn)
cfid = file(self.save_fn, 'wb+')
#--> write navigation records first
cfid.write(struct.pack('<i', self._nav_len))
cfid.write(struct.pack('<i', self._flag))
cfid.write(struct.pack('<h', self._type_dict['nav']))
for nd in range(self._nav_len - 2):
cfid.write(struct.pack('<b', 0))
cfid.write(struct.pack('<i', self._nav_len))
#--> write meta data
meta_str = ''.join([
key + self.meta_data[key] + '\n'
for key in np.sort(list(self.meta_data.keys()))
])
meta_len = len(meta_str)
cfid.write(struct.pack('<i', meta_len + 2))
cfid.write(struct.pack('<i', self._flag))
cfid.write(struct.pack('<h', self._type_dict['meta']))
cfid.write(meta_str)
cfid.write(struct.pack('<i', meta_len + 2))
#--> write calibrations
cal_data1 = 'HEADER.TYPE,Calibrate\nCAL.VER,019\nCAL.SYS,0000,'+\
''.join([' 0.000000: '+'0.000000 0.000000,'*3]*27)
cal_data2 = '\nCAL.SYS,0000,'+\
''.join([' 0.000000: '+'0.000000 0.000000,'*3]*27)
cal_data = cal_data1 + (cal_data2 * (n_fn - 1))
cal_len = len(cal_data)
cfid.write(struct.pack('<i', cal_len + 2))
cfid.write(struct.pack('<i', self._flag))
cfid.write(struct.pack('<h', self._type_dict['cal']))
cfid.write(cal_data[:-1] + '\n')
cfid.write(struct.pack('<i', cal_len + 2))
#--> write data
ts_block_len = int(ts_len) * n_fn * 4 + 2
#--> Need to scale the time series into counts cause that is apparently
# what MTFT24 expects
self.ts = self.ts.astype(np.int32)
#--> make sure none of the data is above the allowed level
self.ts[np.where(self.ts > 2.14e9)] = 2.14e9
self.ts[np.where(self.ts < -2.14e9)] = -2.14e9
#--> write time series block
cfid.write(struct.pack('<i', ts_block_len))
cfid.write(struct.pack('<i', self._flag))
cfid.write(struct.pack('<h', self._type_dict['ts']))
#--> need to pack the data as signed integers
for zz in range(ts_len):
cfid.write(struct.pack('<' + 'i' * n_fn, *self.ts[zz]))
cfid.write(struct.pack('<i', ts_block_len))
cfid.close()
if self.verbose:
print('Saved File to: ', self.save_fn)
self.log_lines.append('=' * 72 + '\n')
self.log_lines.append('Saved File to: \n')
self.log_lines.append(' ' * 4 + '{0}\n'.format(self.save_fn))
self.log_lines.append('=' * 72 + '\n')
import glob
import json
from mtpy.usgs import zen
survey_dir = r"d:\Peacock\MTData\SCEC"
survey_dict = {}
for station in os.listdir(survey_dir):
station_dir = os.path.join(survey_dir, station)
if os.path.isdir(station_dir):
z3d_list = glob.glob(os.path.join(station_dir, "*.Z3D"))
if len(z3d_list) == 0:
continue
for z3d_fn in z3d_list:
z3d_obj = zen.Zen3D(fn=z3d_fn)
z3d_obj.read_all_info()
dt_key = z3d_obj.zen_schedule.isoformat()
if not dt_key in survey_dict.keys():
survey_dict[dt_key] = {}
if not z3d_obj.station in survey_dict[dt_key].keys():
survey_dict[dt_key][z3d_obj.station] = {"comp": [], "df": [], "azm": []}
survey_dict[dt_key][z3d_obj.station]["comp"].append(z3d_obj.component)
survey_dict[dt_key][z3d_obj.station]["df"].append(z3d_obj.df)
survey_dict[dt_key][z3d_obj.station]["azm"].append(z3d_obj.azimuth)
with open(os.path.join(survey_dir, "survey_summary.txt"), "w") as fid:
json.dump(survey_dict, fid)
lines = []
for d_key in sorted(list(survey_dict.keys())):
def combine_z3d_files(z3d_path,
new_sampling_rate=4,
t_buffer=8 * 3600,
comp_list=['ex', 'ey', 'hx', 'hy', 'hz']):
"""
Combine all z3d files for a given station and given component for
processing and getting the long period estimations.
:param str z3d_path: full path to z3d files
:param str component: component to combine
:param int new_sampling_rate: new sampling rate of the data
:param int t_buffer: buffer for the last time series, should be length
of longest schedule chunk
"""
st = datetime.datetime.now()
attr_list = [
"station",
"channel_number",
"component",
"coordinate_system",
"dipole_length",
"azimuth",
"units",
"lat",
"lon",
"elev",
"datum",
"data_logger",
"instrument_id",
"calibration_fn",
"declination",
"fn",
"conversion",
"gain",
]
fn_df = get_z3d_info(z3d_path)
return_fn_list = []
for comp in comp_list:
if len(fn_df[comp]) == 0:
print('Warning: Skipping {0} because no Z3D files found.'.format(
comp))
continue
comp_df = pd.DataFrame(fn_df[comp])
# sort the data frame by date
comp_df = comp_df.sort_values('start')
# get start date and end at last start date, get time difference
start_dt = datetime.datetime.fromisoformat(comp_df.start.min())
end_dt = datetime.datetime.fromisoformat(comp_df.start.max())
t_diff = (end_dt - start_dt).total_seconds()
### make a new MTTS object that will have a length that is buffered
### at the end to make sure there is room for the data, will trimmed
new_ts = ts.MTTS()
new_ts.ts = np.zeros(int((t_diff + t_buffer) * sampling_rate))
new_ts.sampling_rate = sampling_rate
new_ts.start_time_utc = start_dt
# make an attribute dictionary that can be used to fill in the new
# MTTS object
attr_dict = dict([(key, []) for key in attr_list])
# loop over each z3d file for the given component
for row in comp_df.itertuples():
z_obj = zen.Zen3D(row.fn)
print(row.fn)
z_obj.read_z3d()
t_obj = z_obj.ts_obj
# decimate to the required sampling rate
t_obj.decimate(int(z_obj.df / sampling_rate))
# fill the new time series with the data at the appropriate times
print(f"start = {t_obj.ts.index[0]}, end = {t_obj.ts.index[-1]}")
new_ts.ts.data[(new_ts.ts.index >= t_obj.ts.index[0]) & (
new_ts.ts.index <= t_obj.ts.index[-1])] = t_obj.ts.data
# get the end date as the last z3d file
end_date = z_obj.ts_obj.ts.index[-1]
# fill attribute data frame
for attr in attr_list:
attr_dict[attr].append(getattr(t_obj, attr))
# need to trim the data
new_ts.ts = new_ts.ts.data[(new_ts.ts.index >= start_dt)
& (new_ts.ts.index <= end_date)].to_frame()
# fill gaps with forwards or backwards values, this seems to work
# better than interpolation and is faster than regression.
# The gaps should be max 13 seconds if everything went well
new_ts.ts.data[new_ts.ts.data == 0] = np.nan
new_ts.ts.data.fillna(method='ffill', inplace=True)
# fill the new MTTS with the appropriate metadata
attr_df = pd.DataFrame(attr_dict)
for attr in attr_list:
try:
attr_series = attr_df[attr][attr_df[attr] != 0]
try:
setattr(new_ts, attr, attr_series.median())
except TypeError:
setattr(new_ts, attr, attr_series.mode()[0])
except ValueError:
print('Warning: could not set {0}'.format(attr))
ascii_fn = '{0}_combined_{1}.{2}'.format(new_ts.station,
int(new_ts.sampling_rate),
new_ts.component)
sv_fn_ascii = z3d_path.joinpath(ascii_fn)
new_ts.write_ascii_file(sv_fn_ascii.absolute())
return_fn_list.append(sv_fn_ascii)
et = datetime.datetime.now()
compute_time = (et - st).total_seconds()
print(" Combining took {0:.2f} seconds".format(compute_time))
return return_fn_list
t_diff = (end_dt - start_dt).total_seconds()
# make a new MTTS object that will have a length that is buffered
# at the end to make sure there is room for the data, will trimmed
new_ts = ts.MTTS()
new_ts.ts = np.zeros(int((t_diff + t_buffer) * sampling_rate))
new_ts.sampling_rate = sampling_rate
new_ts.start_time_utc = start_dt
# make an attribute dictionary that can be used to fill in the new
# MTTS object
attr_dict = dict([(key, []) for key in attr_list])
# loop over each z3d file for the given component
index = 1
for row in comp_df.itertuples():
z_obj = zen.Zen3D(row.fn)
print(row)
z_obj.read_z3d()
t_obj = z_obj.ts_obj
# decimate to the required sampling rate
t_obj.decimate(int(z_obj.df / sampling_rate))
# fill the new time series with the data at the appropriate times
print(f"start = {t_obj.ts.index[0]}, end = {t_obj.ts.index[-1]}")
new_ts.ts.data[(new_ts.ts.index >= t_obj.ts.index[0])
& (new_ts.ts.index <= t_obj.ts.index[-1])] = t_obj.ts.data
plt.figure(index)
plt.plot(new_ts.ts)
plt.plot(t_obj.ts)
# get the end date as the last z3d file
end_date = z_obj.ts_obj.ts.index[-1]
256: datetime.timedelta(hours=7, minutes=45),
4096: datetime.timedelta(minutes=15),
1024: datetime.timedelta(minutes=10),
}
if not csv_path.exists():
schedule_dict = ""
date_dict = []
for station in survey_path.glob("**/*"):
station_path = Path.joinpath(survey_path.parent, survey_path.name, station)
for z3d_fn in station_path.glob("*.Z3D"):
z_obj = zen.Zen3D(
Path.joinpath(station_path.parent, station_path.name, z3d_fn)
)
try:
z_obj.read_all_info()
except zen.ZenGPSError:
continue
entry = {
"station": z_obj.station,
"start_date": z_obj.zen_schedule.isoformat(),
"sampling_rate": z_obj.df,
}
date_dict.append(entry)
df = pd.DataFrame(date_dict)
df.drop_duplicates(["station", "start_date"], inplace=True)
