def start(self, start_time):
"""
start time of time series in UTC given in some format or a datetime
object.
Resets epoch seconds if the new value is not equivalent to previous
value.
Resets how the ts data frame is indexed, setting the starting time to
the new start time.
:param start_time: start time of time series, can be string or epoch seconds
"""
if not isinstance(start_time, MTime):
start_time = MTime(start_time)
self.channel_metadata.time_period.start = start_time.iso_str
if self.has_data:
if start_time == MTime(
self._ts.coords.indexes["time"][0].isoformat()):
return
else:
new_dt = make_dt_coordinates(start_time, self.sample_rate,
self.n_samples, self.logger)
self._ts.coords["time"] = new_dt
# make a time series that the data can be indexed by
else:
self.logger.debug("No data, just updating metadata start")
self._update_xarray_metadata()
def get_slice(self, start, end=None, n_samples=None):
"""
Get just a chunk of data from the run
:param start: DESCRIPTION
:type start: TYPE
:param end: DESCRIPTION
:type end: TYPE
:return: DESCRIPTION
:rtype: TYPE
"""
if not isinstance(start, MTime):
start = MTime(start)
if n_samples is not None:
seconds = n_samples / self.sample_rate
end = start + seconds
if end is not None:
if not isinstance(end, MTime):
end = MTime(end)
new_runts = RunTS()
new_runts.station_metadata = self.station_metadata
new_runts.run_metadata = self.run_metadata
new_runts.dataset = self._dataset.sel(
time=slice(start.iso_no_tz, end.iso_no_tz))
return new_runts
def end(self):
"""MTime object"""
if self.has_data:
return MTime(self._ts.coords.indexes["time"][-1].isoformat())
else:
self.logger.debug("Data not set yet, pulling end time from " +
"metadata.time_period.end")
return MTime(self.channel_metadata.time_period.end)
def __init__(self, fn=None, **kwargs):
self.logger = logging.getLogger(
f"{__name__}.{self.__class__.__name__}")
self.fn = fn
self.SurveyID = None
self.RunID = None
self.MissingDataFlag = np.NaN
self.CoordinateSystem = None
self._metadata_len = 30
self.declination = 0.0
self._latitude = None
self._longitude = None
self._start = MTime()
self._end = MTime()
self._station = None
self._key_list = [
"SurveyID",
"SiteID",
"RunID",
"SiteLatitude",
"SiteLongitude",
"SiteElevation",
"AcqStartTime",
"AcqStopTime",
"AcqSmpFreq",
"AcqNumSmp",
"Nchan",
"CoordinateSystem",
"ChnSettings",
"MissingDataFlag",
"DataSet",
]
self._chn_settings = [
"ChnNum",
"ChnID",
"InstrumentID",
"Azimuth",
"Dipole_Length",
]
self._chn_fmt = {
"ChnNum": "<8",
"ChnID": "<6",
"InstrumentID": "<12",
"Azimuth": ">7.1f",
"Dipole_Length": ">14.1f",
}
self.channel_dict = dict([(comp,
dict([(key, None)
for key in self._chn_settings]))
for comp in ["ex", "ey", "hx", "hy", "hz"]])
for key in kwargs.keys():
setattr(self, key, kwargs[key])
def test_initialize(self):
with self.subTest("channels"):
self.assertListEqual(["ex", "ey", "hx", "hy", "hz"],
self.run.channels)
with self.subTest("sample rate"):
self.assertEqual(self.run.sample_rate, self.sample_rate)
with self.subTest("start"):
self.assertEqual(self.run.start, MTime(self.start))
with self.subTest("end"):
self.assertEqual(self.run.end, MTime(self.end))
def test_channels(self):
for comp in ["ex", "ey", "hx", "hy", "hz"]:
ch = getattr(self, comp)
with self.subTest("isinstance channel"):
self.assertIsInstance(ch, ChannelTS)
with self.subTest("sample rate"):
self.assertEqual(ch.sample_rate, self.sample_rate)
with self.subTest("start"):
self.assertEqual(ch.start, MTime(self.start))
with self.subTest("end"):
self.assertEqual(ch.end, MTime(self.end))
with self.subTest("component"):
self.assertEqual(ch.component, comp)
def get_slice(self, start, end=None, n_samples=None):
"""
Get a slice from the time series given a start and end time.
Looks for >= start & <= end
Uses loc to be exact with milliseconds
:param start: DESCRIPTION
:type start: TYPE
:param end: DESCRIPTION
:type end: TYPE
:return: DESCRIPTION
:rtype: TYPE
"""
if n_samples is None and end is None:
msg = "Must input either end_time or n_samples."
self.logger.error(msg)
raise ValueError(msg)
if n_samples is not None and end is not None:
msg = "Must input either end_time or n_samples, not both."
self.logger.error(msg)
raise ValueError(msg)
if not isinstance(start, MTime):
start = MTime(start)
if n_samples is not None:
n_samples = int(n_samples)
end = start + n_samples / self.sample_rate
if end is not None:
if not isinstance(end, MTime):
end = MTime(end)
new_ts = self._ts.loc[(self._ts.indexes["time"] >= start.iso_no_tz)
& (self._ts.indexes["time"] < end.iso_no_tz)]
new_ch_ts = ChannelTS(
channel_type=self.channel_type,
data=new_ts,
channel_metadata=self.channel_metadata,
run_metadata=self.run_metadata,
station_metadata=self.station_metadata,
)
return new_ch_ts
def test_get_slice(self):
start = "2015-01-08T19:49:30+00:00"
npts = 256
r_slice = self.run.get_slice(start, n_samples=npts)
with self.subTest("isinstance runts"):
self.assertIsInstance(r_slice, RunTS)
with self.subTest("sample rate"):
self.assertEqual(r_slice.sample_rate, self.sample_rate)
with self.subTest("start"):
self.assertEqual(r_slice.start, MTime(start))
def test_from_run_ts(self):
ts_list = []
for comp in ["ex", "ey", "hx", "hy", "hz"]:
if comp[0] in ["e"]:
ch_type = "electric"
elif comp[1] in ["h", "b"]:
ch_type = "magnetic"
else:
ch_type = "auxiliary"
meta_dict = {
ch_type: {
"component": comp,
"dipole_length": 49.0,
"measurement_azimuth": 12.0,
"type": ch_type,
"units": "counts",
"time_period.start": "2020-01-01T12:00:00",
"sample_rate": 1,
}
}
channel_ts = ChannelTS(ch_type,
data=np.random.rand(4096),
channel_metadata=meta_dict)
ts_list.append(channel_ts)
run_ts = RunTS(ts_list, {"id": "MT002a"})
station = self.mth5_obj.add_station("MT002", survey="test")
run = station.add_run("MT002a")
channel_groups = run.from_runts(run_ts)
self.assertListEqual(["ex", "ey", "hx", "hy", "hz"], run.groups_list)
# check to make sure the metadata was transfered
for cg in channel_groups:
with self.subTest(name=cg.metadata.component):
self.assertEqual(MTime("2020-01-01T12:00:00"), cg.start)
self.assertEqual(1, cg.sample_rate)
self.assertEqual(4096, cg.n_samples)
# slicing
with self.subTest("get slice"):
r_slice = run.to_runts(start="2020-01-01T12:00:00", n_samples=256)
self.assertEqual(r_slice.end, "2020-01-01T12:04:16+00:00")
def make_dt_coordinates(start_time, sample_rate, n_samples, logger):
"""
get the date time index from the data
:param string start_time: start time in time format
:param float sample_rate: sample rate in samples per seconds
:param int n_samples: number of samples in time series
:param :class:`logging.logger` logger: logger class object
:return: date-time index
"""
if sample_rate in [0, None]:
msg = (f"Need to input a valid sample rate. Not {sample_rate}, " +
"returning a time index assuming a sample rate of 1")
logger.warning(msg)
sample_rate = 1
if start_time is None:
msg = (f"Need to input a start time. Not {start_time}, " +
"returning a time index with start time of " +
"1980-01-01T00:00:00")
logger.warning(msg)
start_time = "1980-01-01T00:00:00"
if n_samples < 1:
msg = f"Need to input a valid n_samples. Not {n_samples}"
logger.error(msg)
raise ValueError(msg)
if not isinstance(start_time, MTime):
start_time = MTime(start_time)
dt_freq = "{0:.0f}N".format(1.0e9 / (sample_rate))
dt_index = pd.date_range(
start=start_time.iso_str.split("+", 1)[0],
periods=n_samples,
freq=dt_freq,
closed=None,
)
return dt_index
class AsciiMetadata:
"""
Container for all the important metadata in a USGS ascii file.
========================= =================================================
Attributes Description
========================= =================================================
SurveyID Survey name
SiteID Site name
RunID Run number
SiteLatitude Site latitude in decimal degrees WGS84
SiteLongitude Site longitude in decimal degrees WGS84
SiteElevation Site elevation according to national map meters
AcqStartTime Start time of station YYYY-MM-DDThh:mm:ss UTC
AcqStopTime Stop time of station YYYY-MM-DDThh:mm:ss UTC
AcqSmpFreq Sampling rate samples/second
AcqNumSmp Number of samples
Nchan Number of channels
CoordinateSystem [ Geographic North | Geomagnetic North ]
ChnSettings Channel settings, see below
MissingDataFlag Missing data value
========================= =================================================
:ChnSettings:
========================= =================================================
Keys Description
========================= =================================================
ChnNum SiteID+channel number
ChnID Component [ ex | ey | hx | hy | hz ]
InstrumentID Data logger + sensor number
Azimuth Setup angle of componet in degrees relative to
CoordinateSystem
Dipole_Length Dipole length in meters
========================= =================================================
"""
def __init__(self, fn=None, **kwargs):
self.logger = logging.getLogger(
f"{__name__}.{self.__class__.__name__}")
self.fn = fn
self.SurveyID = None
self.RunID = None
self.MissingDataFlag = np.NaN
self.CoordinateSystem = None
self._metadata_len = 30
self.declination = 0.0
self._latitude = None
self._longitude = None
self._start = MTime()
self._end = MTime()
self._station = None
self._key_list = [
"SurveyID",
"SiteID",
"RunID",
"SiteLatitude",
"SiteLongitude",
"SiteElevation",
"AcqStartTime",
"AcqStopTime",
"AcqSmpFreq",
"AcqNumSmp",
"Nchan",
"CoordinateSystem",
"ChnSettings",
"MissingDataFlag",
"DataSet",
]
self._chn_settings = [
"ChnNum",
"ChnID",
"InstrumentID",
"Azimuth",
"Dipole_Length",
]
self._chn_fmt = {
"ChnNum": "<8",
"ChnID": "<6",
"InstrumentID": "<12",
"Azimuth": ">7.1f",
"Dipole_Length": ">14.1f",
}
self.channel_dict = dict([(comp,
dict([(key, None)
for key in self._chn_settings]))
for comp in ["ex", "ey", "hx", "hy", "hz"]])
for key in kwargs.keys():
setattr(self, key, kwargs[key])
@property
def SiteID(self):
return self._station
@SiteID.setter
def SiteID(self, station):
self._station = station
@property
def SiteLatitude(self):
return self._latitude
# return gis_tools.convert_position_float2str(self._latitude)
@SiteLatitude.setter
def SiteLatitude(self, lat):
self._latitude = lat
@property
def SiteLongitude(self):
return self._longitude
# return gis_tools.convert_position_float2str(self._longitude)
@SiteLongitude.setter
def SiteLongitude(self, lon):
self._longitude = lon
@property
def SiteElevation(self):
"""
get elevation from national map
"""
# the url for national map elevation query
nm_url = r"https://nationalmap.gov/epqs/pqs.php?x={0:.5f}&y={1:.5f}&units=Meters&output=xml"
# call the url and get the response
try:
response = url.request.urlopen(
nm_url.format(self._longitude, self._latitude))
except url.error.HTTPError:
self.logger.error(
"could not connect to get elevation from national map.")
self.logger.debug(nm_url.format(self._longitude, self._latitude))
return -666
# read the xml response and convert to a float
info = ET.ElementTree(ET.fromstring(response.read()))
info = info.getroot()
for elev in info.iter("Elevation"):
nm_elev = float(elev.text)
return nm_elev
@property
def AcqStartTime(self):
return self._start.iso_str
@AcqStartTime.setter
def AcqStartTime(self, time_string):
self._start.from_str(time_string)
@property
def AcqStopTime(self):
return self._end.iso_str
@AcqStopTime.setter
def AcqStopTime(self, time_string):
self._end.from_str(time_string)
@property
def Nchan(self):
return self._chn_num
@Nchan.setter
def Nchan(self, n_channel):
try:
self._chn_num = int(n_channel)
except ValueError:
self.logger.warning(
f"{n_channel} is not a number, setting Nchan to 0")
@property
def AcqSmpFreq(self):
return self._sampling_rate
@AcqSmpFreq.setter
def AcqSmpFreq(self, df):
self._sampling_rate = float(df)
@property
def AcqNumSmp(self):
return self._n_samples
@AcqNumSmp.setter
def AcqNumSmp(self, n_samples):
self._n_samples = int(n_samples)
def get_component_info(self, comp):
"""
:param comp: DESCRIPTION
:type comp: TYPE
:return: DESCRIPTION
:rtype: TYPE
"""
for key, kdict in self.channel_dict.items():
if kdict["ChnID"].lower() == comp.lower():
return kdict
return None
def read_metadata(self, fn=None, meta_lines=None):
"""
Read in a meta from the raw string or file. Populate all metadata
as attributes.
:param fn: full path to USGS ascii file
:type fn: string
:param meta_lines: lines of metadata to read
:type meta_lines: list
"""
chn_find = False
comp = 0
self.channel_dict = {}
if fn is not None:
self.fn = fn
if self.fn is not None:
with open(self.fn, "r") as fid:
meta_lines = [
fid.readline() for ii in range(self._metadata_len)
]
for ii, line in enumerate(meta_lines):
if line.find(":") > 0:
key, value = line.strip().split(":", 1)
value = value.strip()
if len(value) < 1 and key == "DataSet":
chn_find = False
# return the line that the data starts on that way can
# read in as a numpy object or pandas
return ii + 1
elif len(value) < 1:
chn_find = True
if "elev" in key.lower():
pass
else:
setattr(self, key, value)
elif "coordinate" in line:
self.CoordinateSystem = " ".join(line.strip().split()[-2:])
else:
if chn_find is True:
if "chnnum" in line.lower():
ch_key = line.strip().split()
else:
line_list = line.strip().split()
if len(line_list) == 5:
comp += 1
self.channel_dict[comp] = {}
for key, value in zip(ch_key, line_list):
if key.lower() in ["azimuth", "dipole_length"]:
value = float(value)
self.channel_dict[comp][key] = value
else:
self.logger.warning("Not sure what line this is")
def write_metadata(self, chn_list=["Ex", "Ey", "Hx", "Hy", "Hz"]):
"""
Write out metadata in the format of USGS ascii.
:return: list of metadate lines.
.. note:: meant to use '\n'.join(lines) to write out in a file.
"""
lines = []
for key in self._key_list:
if key in ["ChnSettings"]:
lines.append("{0}:".format(key))
lines.append(" ".join(self._chn_settings))
for chn_key in chn_list:
chn_line = []
try:
for comp_key in self._chn_settings:
chn_line.append("{0:{1}}".format(
self.channel_dict[chn_key][comp_key],
self._chn_fmt[comp_key],
))
lines.append("".join(chn_line))
except KeyError:
pass
elif key in ["DataSet"]:
lines.append("{0}:".format(key))
return lines
else:
if key in ["SiteLatitude", "SiteLongitude"]:
lines.append("{0}: {1:.5f}".format(key, getattr(self,
key)))
else:
lines.append("{0}: {1}".format(key, getattr(self, key)))
return lines
EMCAY10LFN,
EMCAY10LFZ,
EMCAY10LQE,
EMCAY10LQN,
ZUCAS04LQ1,
ZUCAS04LQ2,
ZUCAS04BF1,
ZUCAS04BF2,
ZUCAS04BF3,
ZUNRV08LQ1,
ZUNRV08LQ2,
ZUNRV08BF1,
ZUNRV08BF2,
ZUNRV08BF3,
]
st = MTime(get_now_utc())
make_mth5 = MakeMTH5()
make_mth5.mth5_version = "0.2.0"
# Turn list into dataframe
metadata_df = pd.DataFrame(request_list, columns=make_mth5.column_names)
# # get only metadata
# inventory, streams = make_mth5.get_inventory_from_df(metadata_df, "iris", data=False)
# df = make_mth5.get_df_from_inventory(station_xml)
mth5_file = make_mth5.make_mth5_from_fdsnclient(metadata_df, interact=True)
et = MTime(get_now_utc())
print(f"Took {(int(et - st) // 60)}:{(et - st) % 60:05.2f} minutes")
# =============================================================================
# set to true if you want to interact with the mth5 object in the console
interact = False
nims_dir = "path/to/nims/file.bin"
h5_fn = "from_nims.mth5"
if h5_fn.exists():
h5_fn.unlink()
print(f"INFO: Removed existing file {h5_fn}")
# need to unzip the data
with zipfile.ZipFile(nims_dir.joinpath("nims.zip"), "r") as zip_ref:
zip_ref.extractall(nims_dir)
processing_start = MTime()
processing_start.now()
# write some simple metadata for the survey
survey = metadata.Survey()
survey.acquired_by.author = "MT Master"
survey.archive_id = "TST01"
survey.archive_network = "MT"
survey.name = "test"
m = mth5.MTH5()
m.open_mth5(h5_fn, "w")
# add survey metadata
survey_group = m.survey_group
survey_group.metadata.from_dict(survey.to_dict())
:license: MIT
"""
# =============================================================================
# Imports
# =============================================================================
import zipfile
from mth5 import read_file
from mth5 import mth5
from mt_metadata import timeseries as metadata
from mt_metadata.utils.mttime import MTime
start = MTime()
start.now()
# =============================================================================
#
# =============================================================================
# set to true if you want to interact with the mth5 object in the console
interact = True
zen_dir = "path/to/s3d/files"
h5_fn = "from_zen.h5"
if h5_fn.exists():
h5_fn.unlink()
print(f"INFO: Removed existing file {h5_fn}")
# need to unzip the data
with zipfile.ZipFile(zen_dir.joinpath("zen.zip"), "r") as zip_ref:
def end(self):
if self.has_data:
return MTime(
self.dataset.coords["time"].to_index()[-1].isoformat())
return self.run_metadata.time_period.end
