def start(self, start_time): """ start time of time series in UTC given in some format or a datetime object. Resets epoch seconds if the new value is not equivalent to previous value. Resets how the ts data frame is indexed, setting the starting time to the new start time. :param start_time: start time of time series, can be string or epoch seconds """ if not isinstance(start_time, MTime): start_time = MTime(start_time) self.channel_metadata.time_period.start = start_time.iso_str if self.has_data: if start_time == MTime( self._ts.coords.indexes["time"][0].isoformat()): return else: new_dt = make_dt_coordinates(start_time, self.sample_rate, self.n_samples, self.logger) self._ts.coords["time"] = new_dt # make a time series that the data can be indexed by else: self.logger.debug("No data, just updating metadata start") self._update_xarray_metadata()
def get_slice(self, start, end=None, n_samples=None): """ Get just a chunk of data from the run :param start: DESCRIPTION :type start: TYPE :param end: DESCRIPTION :type end: TYPE :return: DESCRIPTION :rtype: TYPE """ if not isinstance(start, MTime): start = MTime(start) if n_samples is not None: seconds = n_samples / self.sample_rate end = start + seconds if end is not None: if not isinstance(end, MTime): end = MTime(end) new_runts = RunTS() new_runts.station_metadata = self.station_metadata new_runts.run_metadata = self.run_metadata new_runts.dataset = self._dataset.sel( time=slice(start.iso_no_tz, end.iso_no_tz)) return new_runts
def end(self): """MTime object""" if self.has_data: return MTime(self._ts.coords.indexes["time"][-1].isoformat()) else: self.logger.debug("Data not set yet, pulling end time from " + "metadata.time_period.end") return MTime(self.channel_metadata.time_period.end)
def __init__(self, fn=None, **kwargs): self.logger = logging.getLogger( f"{__name__}.{self.__class__.__name__}") self.fn = fn self.SurveyID = None self.RunID = None self.MissingDataFlag = np.NaN self.CoordinateSystem = None self._metadata_len = 30 self.declination = 0.0 self._latitude = None self._longitude = None self._start = MTime() self._end = MTime() self._station = None self._key_list = [ "SurveyID", "SiteID", "RunID", "SiteLatitude", "SiteLongitude", "SiteElevation", "AcqStartTime", "AcqStopTime", "AcqSmpFreq", "AcqNumSmp", "Nchan", "CoordinateSystem", "ChnSettings", "MissingDataFlag", "DataSet", ] self._chn_settings = [ "ChnNum", "ChnID", "InstrumentID", "Azimuth", "Dipole_Length", ] self._chn_fmt = { "ChnNum": "<8", "ChnID": "<6", "InstrumentID": "<12", "Azimuth": ">7.1f", "Dipole_Length": ">14.1f", } self.channel_dict = dict([(comp, dict([(key, None) for key in self._chn_settings])) for comp in ["ex", "ey", "hx", "hy", "hz"]]) for key in kwargs.keys(): setattr(self, key, kwargs[key])
def test_initialize(self): with self.subTest("channels"): self.assertListEqual(["ex", "ey", "hx", "hy", "hz"], self.run.channels) with self.subTest("sample rate"): self.assertEqual(self.run.sample_rate, self.sample_rate) with self.subTest("start"): self.assertEqual(self.run.start, MTime(self.start)) with self.subTest("end"): self.assertEqual(self.run.end, MTime(self.end))
def test_channels(self): for comp in ["ex", "ey", "hx", "hy", "hz"]: ch = getattr(self, comp) with self.subTest("isinstance channel"): self.assertIsInstance(ch, ChannelTS) with self.subTest("sample rate"): self.assertEqual(ch.sample_rate, self.sample_rate) with self.subTest("start"): self.assertEqual(ch.start, MTime(self.start)) with self.subTest("end"): self.assertEqual(ch.end, MTime(self.end)) with self.subTest("component"): self.assertEqual(ch.component, comp)
def get_slice(self, start, end=None, n_samples=None): """ Get a slice from the time series given a start and end time. Looks for >= start & <= end Uses loc to be exact with milliseconds :param start: DESCRIPTION :type start: TYPE :param end: DESCRIPTION :type end: TYPE :return: DESCRIPTION :rtype: TYPE """ if n_samples is None and end is None: msg = "Must input either end_time or n_samples." self.logger.error(msg) raise ValueError(msg) if n_samples is not None and end is not None: msg = "Must input either end_time or n_samples, not both." self.logger.error(msg) raise ValueError(msg) if not isinstance(start, MTime): start = MTime(start) if n_samples is not None: n_samples = int(n_samples) end = start + n_samples / self.sample_rate if end is not None: if not isinstance(end, MTime): end = MTime(end) new_ts = self._ts.loc[(self._ts.indexes["time"] >= start.iso_no_tz) & (self._ts.indexes["time"] < end.iso_no_tz)] new_ch_ts = ChannelTS( channel_type=self.channel_type, data=new_ts, channel_metadata=self.channel_metadata, run_metadata=self.run_metadata, station_metadata=self.station_metadata, ) return new_ch_ts
def test_get_slice(self): start = "2015-01-08T19:49:30+00:00" npts = 256 r_slice = self.run.get_slice(start, n_samples=npts) with self.subTest("isinstance runts"): self.assertIsInstance(r_slice, RunTS) with self.subTest("sample rate"): self.assertEqual(r_slice.sample_rate, self.sample_rate) with self.subTest("start"): self.assertEqual(r_slice.start, MTime(start))
def test_from_run_ts(self): ts_list = [] for comp in ["ex", "ey", "hx", "hy", "hz"]: if comp[0] in ["e"]: ch_type = "electric" elif comp[1] in ["h", "b"]: ch_type = "magnetic" else: ch_type = "auxiliary" meta_dict = { ch_type: { "component": comp, "dipole_length": 49.0, "measurement_azimuth": 12.0, "type": ch_type, "units": "counts", "time_period.start": "2020-01-01T12:00:00", "sample_rate": 1, } } channel_ts = ChannelTS(ch_type, data=np.random.rand(4096), channel_metadata=meta_dict) ts_list.append(channel_ts) run_ts = RunTS(ts_list, {"id": "MT002a"}) station = self.mth5_obj.add_station("MT002", survey="test") run = station.add_run("MT002a") channel_groups = run.from_runts(run_ts) self.assertListEqual(["ex", "ey", "hx", "hy", "hz"], run.groups_list) # check to make sure the metadata was transfered for cg in channel_groups: with self.subTest(name=cg.metadata.component): self.assertEqual(MTime("2020-01-01T12:00:00"), cg.start) self.assertEqual(1, cg.sample_rate) self.assertEqual(4096, cg.n_samples) # slicing with self.subTest("get slice"): r_slice = run.to_runts(start="2020-01-01T12:00:00", n_samples=256) self.assertEqual(r_slice.end, "2020-01-01T12:04:16+00:00")
def make_dt_coordinates(start_time, sample_rate, n_samples, logger): """ get the date time index from the data :param string start_time: start time in time format :param float sample_rate: sample rate in samples per seconds :param int n_samples: number of samples in time series :param :class:`logging.logger` logger: logger class object :return: date-time index """ if sample_rate in [0, None]: msg = (f"Need to input a valid sample rate. Not {sample_rate}, " + "returning a time index assuming a sample rate of 1") logger.warning(msg) sample_rate = 1 if start_time is None: msg = (f"Need to input a start time. Not {start_time}, " + "returning a time index with start time of " + "1980-01-01T00:00:00") logger.warning(msg) start_time = "1980-01-01T00:00:00" if n_samples < 1: msg = f"Need to input a valid n_samples. Not {n_samples}" logger.error(msg) raise ValueError(msg) if not isinstance(start_time, MTime): start_time = MTime(start_time) dt_freq = "{0:.0f}N".format(1.0e9 / (sample_rate)) dt_index = pd.date_range( start=start_time.iso_str.split("+", 1)[0], periods=n_samples, freq=dt_freq, closed=None, ) return dt_index
class AsciiMetadata: """ Container for all the important metadata in a USGS ascii file. ========================= ================================================= Attributes Description ========================= ================================================= SurveyID Survey name SiteID Site name RunID Run number SiteLatitude Site latitude in decimal degrees WGS84 SiteLongitude Site longitude in decimal degrees WGS84 SiteElevation Site elevation according to national map meters AcqStartTime Start time of station YYYY-MM-DDThh:mm:ss UTC AcqStopTime Stop time of station YYYY-MM-DDThh:mm:ss UTC AcqSmpFreq Sampling rate samples/second AcqNumSmp Number of samples Nchan Number of channels CoordinateSystem [ Geographic North | Geomagnetic North ] ChnSettings Channel settings, see below MissingDataFlag Missing data value ========================= ================================================= :ChnSettings: ========================= ================================================= Keys Description ========================= ================================================= ChnNum SiteID+channel number ChnID Component [ ex | ey | hx | hy | hz ] InstrumentID Data logger + sensor number Azimuth Setup angle of componet in degrees relative to CoordinateSystem Dipole_Length Dipole length in meters ========================= ================================================= """ def __init__(self, fn=None, **kwargs): self.logger = logging.getLogger( f"{__name__}.{self.__class__.__name__}") self.fn = fn self.SurveyID = None self.RunID = None self.MissingDataFlag = np.NaN self.CoordinateSystem = None self._metadata_len = 30 self.declination = 0.0 self._latitude = None self._longitude = None self._start = MTime() self._end = MTime() self._station = None self._key_list = [ "SurveyID", "SiteID", "RunID", "SiteLatitude", "SiteLongitude", "SiteElevation", "AcqStartTime", "AcqStopTime", "AcqSmpFreq", "AcqNumSmp", "Nchan", "CoordinateSystem", "ChnSettings", "MissingDataFlag", "DataSet", ] self._chn_settings = [ "ChnNum", "ChnID", "InstrumentID", "Azimuth", "Dipole_Length", ] self._chn_fmt = { "ChnNum": "<8", "ChnID": "<6", "InstrumentID": "<12", "Azimuth": ">7.1f", "Dipole_Length": ">14.1f", } self.channel_dict = dict([(comp, dict([(key, None) for key in self._chn_settings])) for comp in ["ex", "ey", "hx", "hy", "hz"]]) for key in kwargs.keys(): setattr(self, key, kwargs[key]) @property def SiteID(self): return self._station @SiteID.setter def SiteID(self, station): self._station = station @property def SiteLatitude(self): return self._latitude # return gis_tools.convert_position_float2str(self._latitude) @SiteLatitude.setter def SiteLatitude(self, lat): self._latitude = lat @property def SiteLongitude(self): return self._longitude # return gis_tools.convert_position_float2str(self._longitude) @SiteLongitude.setter def SiteLongitude(self, lon): self._longitude = lon @property def SiteElevation(self): """ get elevation from national map """ # the url for national map elevation query nm_url = r"https://nationalmap.gov/epqs/pqs.php?x={0:.5f}&y={1:.5f}&units=Meters&output=xml" # call the url and get the response try: response = url.request.urlopen( nm_url.format(self._longitude, self._latitude)) except url.error.HTTPError: self.logger.error( "could not connect to get elevation from national map.") self.logger.debug(nm_url.format(self._longitude, self._latitude)) return -666 # read the xml response and convert to a float info = ET.ElementTree(ET.fromstring(response.read())) info = info.getroot() for elev in info.iter("Elevation"): nm_elev = float(elev.text) return nm_elev @property def AcqStartTime(self): return self._start.iso_str @AcqStartTime.setter def AcqStartTime(self, time_string): self._start.from_str(time_string) @property def AcqStopTime(self): return self._end.iso_str @AcqStopTime.setter def AcqStopTime(self, time_string): self._end.from_str(time_string) @property def Nchan(self): return self._chn_num @Nchan.setter def Nchan(self, n_channel): try: self._chn_num = int(n_channel) except ValueError: self.logger.warning( f"{n_channel} is not a number, setting Nchan to 0") @property def AcqSmpFreq(self): return self._sampling_rate @AcqSmpFreq.setter def AcqSmpFreq(self, df): self._sampling_rate = float(df) @property def AcqNumSmp(self): return self._n_samples @AcqNumSmp.setter def AcqNumSmp(self, n_samples): self._n_samples = int(n_samples) def get_component_info(self, comp): """ :param comp: DESCRIPTION :type comp: TYPE :return: DESCRIPTION :rtype: TYPE """ for key, kdict in self.channel_dict.items(): if kdict["ChnID"].lower() == comp.lower(): return kdict return None def read_metadata(self, fn=None, meta_lines=None): """ Read in a meta from the raw string or file. Populate all metadata as attributes. :param fn: full path to USGS ascii file :type fn: string :param meta_lines: lines of metadata to read :type meta_lines: list """ chn_find = False comp = 0 self.channel_dict = {} if fn is not None: self.fn = fn if self.fn is not None: with open(self.fn, "r") as fid: meta_lines = [ fid.readline() for ii in range(self._metadata_len) ] for ii, line in enumerate(meta_lines): if line.find(":") > 0: key, value = line.strip().split(":", 1) value = value.strip() if len(value) < 1 and key == "DataSet": chn_find = False # return the line that the data starts on that way can # read in as a numpy object or pandas return ii + 1 elif len(value) < 1: chn_find = True if "elev" in key.lower(): pass else: setattr(self, key, value) elif "coordinate" in line: self.CoordinateSystem = " ".join(line.strip().split()[-2:]) else: if chn_find is True: if "chnnum" in line.lower(): ch_key = line.strip().split() else: line_list = line.strip().split() if len(line_list) == 5: comp += 1 self.channel_dict[comp] = {} for key, value in zip(ch_key, line_list): if key.lower() in ["azimuth", "dipole_length"]: value = float(value) self.channel_dict[comp][key] = value else: self.logger.warning("Not sure what line this is") def write_metadata(self, chn_list=["Ex", "Ey", "Hx", "Hy", "Hz"]): """ Write out metadata in the format of USGS ascii. :return: list of metadate lines. .. note:: meant to use '\n'.join(lines) to write out in a file. """ lines = [] for key in self._key_list: if key in ["ChnSettings"]: lines.append("{0}:".format(key)) lines.append(" ".join(self._chn_settings)) for chn_key in chn_list: chn_line = [] try: for comp_key in self._chn_settings: chn_line.append("{0:{1}}".format( self.channel_dict[chn_key][comp_key], self._chn_fmt[comp_key], )) lines.append("".join(chn_line)) except KeyError: pass elif key in ["DataSet"]: lines.append("{0}:".format(key)) return lines else: if key in ["SiteLatitude", "SiteLongitude"]: lines.append("{0}: {1:.5f}".format(key, getattr(self, key))) else: lines.append("{0}: {1}".format(key, getattr(self, key))) return lines
EMCAY10LFN, EMCAY10LFZ, EMCAY10LQE, EMCAY10LQN, ZUCAS04LQ1, ZUCAS04LQ2, ZUCAS04BF1, ZUCAS04BF2, ZUCAS04BF3, ZUNRV08LQ1, ZUNRV08LQ2, ZUNRV08BF1, ZUNRV08BF2, ZUNRV08BF3, ] st = MTime(get_now_utc()) make_mth5 = MakeMTH5() make_mth5.mth5_version = "0.2.0" # Turn list into dataframe metadata_df = pd.DataFrame(request_list, columns=make_mth5.column_names) # # get only metadata # inventory, streams = make_mth5.get_inventory_from_df(metadata_df, "iris", data=False) # df = make_mth5.get_df_from_inventory(station_xml) mth5_file = make_mth5.make_mth5_from_fdsnclient(metadata_df, interact=True) et = MTime(get_now_utc()) print(f"Took {(int(et - st) // 60)}:{(et - st) % 60:05.2f} minutes")
# ============================================================================= # set to true if you want to interact with the mth5 object in the console interact = False nims_dir = "path/to/nims/file.bin" h5_fn = "from_nims.mth5" if h5_fn.exists(): h5_fn.unlink() print(f"INFO: Removed existing file {h5_fn}") # need to unzip the data with zipfile.ZipFile(nims_dir.joinpath("nims.zip"), "r") as zip_ref: zip_ref.extractall(nims_dir) processing_start = MTime() processing_start.now() # write some simple metadata for the survey survey = metadata.Survey() survey.acquired_by.author = "MT Master" survey.archive_id = "TST01" survey.archive_network = "MT" survey.name = "test" m = mth5.MTH5() m.open_mth5(h5_fn, "w") # add survey metadata survey_group = m.survey_group survey_group.metadata.from_dict(survey.to_dict())
:license: MIT """ # ============================================================================= # Imports # ============================================================================= import zipfile from mth5 import read_file from mth5 import mth5 from mt_metadata import timeseries as metadata from mt_metadata.utils.mttime import MTime start = MTime() start.now() # ============================================================================= # # ============================================================================= # set to true if you want to interact with the mth5 object in the console interact = True zen_dir = "path/to/s3d/files" h5_fn = "from_zen.h5" if h5_fn.exists(): h5_fn.unlink() print(f"INFO: Removed existing file {h5_fn}") # need to unzip the data with zipfile.ZipFile(zen_dir.joinpath("zen.zip"), "r") as zip_ref:
def end(self): if self.has_data: return MTime( self.dataset.coords["time"].to_index()[-1].isoformat()) return self.run_metadata.time_period.end