def read_event_json_files(eventfiles): """Read event.json file and return ScalarEvent object. Args: eventfiles (list): Event.json files to be read. Returns: list: ScalarEvent objects. """ events = [] for eventfile in eventfiles: with open(eventfile, "rt", encoding="utf-8") as f: event = json.load(f) try: origintime = datetime.fromtimestamp( event["properties"]["time"] / 1000.0, pytz.utc) evdict = { "id": event["id"], "time": origintime.strftime("%Y-%m-%dT%H:%M:%S.%f"), "lat": event["geometry"]["coordinates"][1], "lon": event["geometry"]["coordinates"][0], "depth": event["geometry"]["coordinates"][2], "magnitude": event["properties"]["mag"], "magnitude_type": event["properties"]["magType"], } event = get_event_object(evdict) except BaseException: event = get_event_object(event) events.append(event) return events
def parse_event_file(eventfile): """Parse text file containing basic event information. Files can contain: - one column, in which case that column contains ComCat event IDs. - Seven columns, in which case those columns should be: - id: any string (no spaces) - time: Any ISO standard for date/time. - lat: Earthquake latitude in decimal degrees. - lon: Earthquake longitude in decimal degrees. - depth: Earthquake longitude in kilometers. - magnitude: Earthquake magnitude. - magnitude_type: Earthquake magnitude type. NB: THERE SHOULD NOT BE ANY HEADERS ON THIS FILE! Args: eventfile (str): Path to event text file Returns: list: ScalarEvent objects constructed from list of event information. """ df = pd.read_csv(eventfile, sep=",", header=None) nrows, ncols = df.shape events = [] if ncols == 1: df.columns = ["eventid"] for idx, row in df.iterrows(): event = get_event_object(row["eventid"]) events.append(event) elif ncols == 7: df.columns = [ "id", "time", "lat", "lon", "depth", "magnitude", "magnitude_type", ] df["time"] = pd.to_datetime(df["time"]) for idx, row in df.iterrows(): rowdict = row.to_dict() event = get_event_object(rowdict) events.append(event) else: return None return events
def test_event(): subdir = os.path.join('data', 'testdata', 'vcr_event_test.yaml') tape_file = pkg_resources.resource_filename('gmprocess', subdir) with vcr.use_cassette(tape_file): eid = 'us1000j96d' # M7.0 Peru Mar 1 2019 edict = get_event_dict(eid) tdict = { 'id': 'us1000j96d', 'time': UTCDateTime(2019, 3, 1, 8, 50, 42, 615000), 'lat': -14.7007, 'lon': -70.1516, 'depth': 267, 'magnitude': 7.0, 'magnitude_type': 'mww' } for key, value in tdict.items(): v1 = edict[key] assert value == v1 event = get_event_object(eid) assert event.id == eid assert event.magnitude == tdict['magnitude'] assert event.magnitude_type == tdict['magnitude_type'] assert event.time == tdict['time'] assert event.latitude == tdict['lat'] assert event.longitude == tdict['lon'] assert event.depth == tdict['depth'] * 1000 assert event.depth_km == tdict['depth']
def test_event(): subdir = os.path.join("data", "testdata", "vcr_event_test.yaml") tape_file = pkg_resources.resource_filename("gmprocess", subdir) with vcr.use_cassette(tape_file): eid = "us1000j96d" # M7.0 Peru Mar 1 2019 edict = get_event_dict(eid) tdict = { "id": "us1000j96d", "time": UTCDateTime(2019, 3, 1, 8, 50, 42, 615000), "lat": -14.7007, "lon": -70.1516, "depth": 267, "magnitude": 7.0, "magnitude_type": "mww", } for key, value in tdict.items(): v1 = edict[key] assert value == v1 event = get_event_object(eid) assert event.id == eid assert event.magnitude == tdict["magnitude"] assert event.magnitude_type == tdict["magnitude_type"] assert event.time == tdict["time"] assert event.latitude == tdict["lat"] assert event.longitude == tdict["lon"] assert event.depth == tdict["depth"] * 1000 assert event.depth_km == tdict["depth"]
def test_zero_crossings(): datapath = os.path.join("data", "testdata", "zero_crossings") datadir = pkg_resources.resource_filename("gmprocess", datapath) sc = StreamCollection.from_directory(datadir) sc.describe() conf = get_config() update = { "processing": [ {"detrend": {"detrending_method": "demean"}}, {"check_zero_crossings": {"min_crossings": 1}}, ] } update_dict(conf, update) edict = { "id": "ak20419010", "time": UTCDateTime("2018-11-30T17:29:29"), "lat": 61.346, "lon": -149.955, "depth": 46.7, "magnitude": 7.1, } event = get_event_object(edict) test = process_streams(sc, event, conf) for st in test: for tr in st: assert tr.hasParameter("ZeroCrossingRate") np.testing.assert_allclose( test[0][0].getParameter("ZeroCrossingRate")["crossing_rate"], 0.008888888888888889, atol=1e-5, )
def test_fit_spectra(): config = get_config() datapath = os.path.join('data', 'testdata', 'demo', 'ci38457511', 'raw') datadir = pkg_resources.resource_filename('gmprocess', datapath) event = get_event_object('ci38457511') sc = StreamCollection.from_directory(datadir) for st in sc: st = signal_split(st, event) end_conf = config['windows']['signal_end'] st = signal_end(st, event_time=event.time, event_lon=event.longitude, event_lat=event.latitude, event_mag=event.magnitude, **end_conf) st = compute_snr(st, 30) st = get_corner_frequencies(st, method='constant', constant={ 'highpass': 0.08, 'lowpass': 20.0 }) for st in sc: spectrum.fit_spectra(st, event)
def test_fit_spectra(): config = get_config() datapath = os.path.join("data", "testdata", "demo", "ci38457511", "raw") datadir = pkg_resources.resource_filename("gmprocess", datapath) event = get_event_object("ci38457511") sc = StreamCollection.from_directory(datadir) for st in sc: st = signal_split(st, event) end_conf = config["windows"]["signal_end"] st = signal_end(st, event_time=event.time, event_lon=event.longitude, event_lat=event.latitude, event_mag=event.magnitude, **end_conf) st = compute_snr(st, 30) st = get_corner_frequencies(st, event, method="constant", constant={ "highpass": 0.08, "lowpass": 20.0 }) for st in sc: spectrum.fit_spectra(st, event)
def events_from_directory(dir): events = [] eventfiles = get_event_files(dir) if len(eventfiles): events = read_event_json_files(eventfiles) else: eventids = [f for f in os.listdir(dir) if not f.startswith('.')] for eventid in eventids: try: event = get_event_object(eventid) events.append(event) # If the event ID has been updated, make sure to rename # the source folder and issue a warning to the user if event.id != eventid: old_dir = os.path.join(dir, eventid) new_dir = os.path.join(dir, event.id) os.rename(old_dir, new_dir) logging.warn('Directory %s has been renamed to %s.' % (old_dir, new_dir)) except BaseException: logging.warning('Could not get info for event id: %s' % eventid) return events
def update_eq_options(wdir, proj, label, imc, eqid): if any([val is None for val in locals().values()]): return [], None, [], None, [], [] df_eq_imc = get_eq_imc_df(wdir, proj, label, eqid, imc) imt_options = get_options(df_eq_imc, IMT_REGEX) imt_options.append({'label': 'Pass/Fail', 'value': 'Pass/Fail'}) imt = imt_options[0]['value'] dist_options = get_options(df_eq_imc, DIST_REGEX) dist = dist_options[0]['value'] model_options = get_model_options(MODELS_DICT, imc, imt) rep = get_event_object(eqid).__repr__() return imt_options, imt, dist_options, dist, model_options, rep
def read_event_json_files(eventfiles): """Read event.json file and return ScalarEvent object. Args: eventfiles (list): Event.json files to be read. Returns: list: ScalarEvent objects. """ events = [] for eventfile in eventfiles: with open(eventfile, 'rt', encoding='utf-8') as f: eventdict = json.load(f) event = get_event_object(eventdict) events.append(event) return events
def _test_trim_multiple_events(): datapath = os.path.join("data", "testdata", "multiple_events") datadir = pkg_resources.resource_filename("gmprocess", datapath) sc = StreamCollection.from_directory(os.path.join(datadir, "ci38457511")) origin = get_event_object("ci38457511") df, catalog = create_travel_time_dataframe( sc, os.path.join(datadir, "catalog.csv"), 5, 0.1, "iasp91" ) for st in sc: st.detrend("demean") remove_response(st, None, None) st = corner_frequencies.from_constant(st) lowpass_filter(st) highpass_filter(st) signal_split(st, origin) signal_end( st, origin.time, origin.longitude, origin.latitude, origin.magnitude, method="model", model="AS16", ) cut(st, 2) trim_multiple_events( st, origin, catalog, df, 0.2, 0.7, "B14", {"vs30": 760}, {"rake": 0} ) num_failures = sum([1 if not st.passed else 0 for st in sc]) assert num_failures == 2 failure = sc.select(station="WRV2")[0][0].getParameter("failure") assert failure["module"] == "trim_multiple_events" assert failure["reason"] == ( "A significant arrival from another event " "occurs within the first 70.0 percent of the " "signal window" ) for tr in sc.select(station="JRC2")[0]: np.testing.assert_almost_equal( tr.stats.endtime, UTCDateTime("2019-07-06T03:20:56.368300Z") )
def test_check_clipping(): data_files, _ = read_data_dir("clipping_samples", "hv70907436", "*.mseed") data_files.sort() origin = get_event_object("hv70907436") streams = [] for f in data_files: streams += read_data(f) codes = ["HV.TOUO", "HV.MOKD", "HV.MLOD", "HV.HOVE", "HV.HUAD", "HV.HSSD"] passed = [] for code in codes: traces = [] for ss in streams: tcode = f"{ss[0].stats.network}.{ss[0].stats.station}" if tcode == code: traces.append(ss[0]) st = StationStream(traces) check_clipping(st, origin) passed.append(st.passed) assert np.all(~np.array(passed))
def test_zero_crossings(): datapath = os.path.join('data', 'testdata', 'zero_crossings') datadir = pkg_resources.resource_filename('gmprocess', datapath) sc = StreamCollection.from_directory(datadir) sc.describe() conf = get_config() update = { 'processing': [{ 'detrend': { 'detrending_method': 'demean' } }, { 'check_zero_crossings': { 'min_crossings': 1 } }] } update_dict(conf, update) edict = { 'id': 'ak20419010', 'time': UTCDateTime('2018-11-30T17:29:29'), 'lat': 61.346, 'lon': -149.955, 'depth': 46.7, 'magnitude': 7.1 } event = get_event_object(edict) test = process_streams(sc, event, conf) for st in test: for tr in st: assert tr.hasParameter('ZeroCrossingRate') np.testing.assert_allclose( test[0][0].getParameter('ZeroCrossingRate')['crossing_rate'], 0.008888888888888889, atol=1e-5)
def test_trim_multiple_events(): datapath = os.path.join('data', 'testdata', 'multiple_events') datadir = pkg_resources.resource_filename('gmprocess', datapath) sc = StreamCollection.from_directory(os.path.join(datadir, 'ci38457511')) origin = get_event_object('ci38457511') df, catalog = create_travel_time_dataframe( sc, os.path.join(datadir, 'catalog.csv'), 5, 0.1, 'iasp91') for st in sc: st.detrend('demean') remove_response(st, None, None) st = corner_frequencies.get_constant(st) lowpass_filter(st) highpass_filter(st) signal_split(st, origin) signal_end(st, origin.time, origin.longitude, origin.latitude, origin.magnitude, method='model', model='AS16') cut(st, 2) trim_multiple_events(st, origin, catalog, df, 0.2, 0.7, 'B14', {'vs30': 760}, {'rake': 0}) num_failures = sum([1 if not st.passed else 0 for st in sc]) assert num_failures == 1 failure = sc.select(station='WRV2')[0][0].getParameter('failure') assert failure['module'] == 'trim_multiple_events' assert failure['reason'] == ('A significant arrival from another event ' 'occurs within the first 70.0 percent of the ' 'signal window') for tr in sc.select(station='JRC2')[0]: np.testing.assert_almost_equal( tr.stats.endtime, UTCDateTime('2019-07-06T03:20:38.7983Z'))
def get_events(eventids, textfile, eventinfo, directory, outdir=None): """Find the list of events. Args: eventids (list or None): List of ComCat event IDs. textfile (str or None): Path to text file containing event IDs or info. eventinfo (list or None): List containing: - id Any string, no spaces. - time Any ISO-compatible date/time string. - latitude Latitude in decimal degrees. - longitude Longitude in decimal degrees. - depth Depth in kilometers. - magnitude Earthquake magnitude. - magnitude_type Earthquake magnitude type. directory (str): Path to a directory containing event subdirectories, each containing an event.json file, where the ID in the json file matches the subdirectory containing it. outdir (str): Output directory. Returns: list: ScalarEvent objects. """ events = [] if eventids is not None: # Get list of events from directory if it has been provided tevents = [] if directory is not None: tevents = events_from_directory(directory) elif outdir is not None: tevents = events_from_directory(outdir) eventidlist = [event.id for event in tevents] for eventid in eventids: try: idx = eventidlist.index(eventid) event = tevents[idx] except ValueError: # This connects to comcat to get event, does not check for a # local json file event = get_event_object(eventid) events.append(event) elif textfile is not None: events = parse_event_file(textfile) elif eventinfo is not None: eid = eventinfo[0] time = eventinfo[1] lat = float(eventinfo[2]) lon = float(eventinfo[3]) dep = float(eventinfo[4]) mag = float(eventinfo[5]) mag_type = str(eventinfo[6]) event = ScalarEvent() event.fromParams(eid, time, lat, lon, dep, mag, mag_type) events = [event] elif directory is not None: events = events_from_directory(directory) elif outdir is not None: events = events_from_directory(outdir) # "events" elements are None if an error occurred, e.g., bad event id is specified. events = [e for e in events if e is not None] return events
def main(): desc = """Convert CWB data from web site into form ingestible by gmprocess. To obtain CWB strong motion data, create an account on the CWB GDMS-2020 website: https://gdmsn.cwb.gov.tw/index.php To retrieve strong motion response files: Click on the "Data" icon, then click on "Instrument Response". - For "Output Format", choose "RESP file". - For "Network", choose "CWBSN". - For "Station", check "All Stations". - For "Location", choose "*" for all locations. - For "Channel", choose "HN?" for all strong motion stations. - For "Start Time (UTC)", enter a date before the event of interest. - For "End Time (UTC)", enter a date after the event of interest. - For "Label", enter any string that is descriptive to you. Click the "Submit" button, and you should see a "Success!" pop up. Next will be a screen showing a list of all of your downloads. Response data links should appear fairly quickly - click on the name of the generated gzipped "tarball" file to download it. To retrieve a strong motion data file: Click on the "Data" icon, then click on "Multi-Station Waveform Data". - For "Output Format", choose "MiniSEED". - For "Network", choose "CWBSN". - For "Station", check "All Stations". - For "Location", choose "*" for all locations. - For "Channel", choose "HN?" for all strong motion stations. - For "Start Time (UTC)", enter a time 30 seconds before the origin time of interest. - For "End Time (UTC)", enter a time 7 minutes after the origin time of interest. - For "Label", enter any string that is descriptive to you. Click the "Submit" button, and you should see a "Success!" pop up. Next will be a screen showing a list of all of your downloads. Data links will take a few minutes to process - click on the name of the generated miniseed file to download it. Pass these files along with the ComCat ID as described below. """ parser = argparse.ArgumentParser( description=desc, formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument( "event", help="ComCat Event ID", ) parser.add_argument( "seedfile", help="Miniseed file containing all relevant data", ) parser.add_argument( "tarball", help="Tarball containing response files", ) if len(sys.argv) == 1: parser.print_help(sys.stderr) sys.exit(1) args = parser.parse_args() stations = read_stations() CONF_PATH = pathlib.Path.home() / ".gmprocess" PROJECTS_FILE = CONF_PATH / "projects.conf" projects_conf = configobj.ConfigObj(str(PROJECTS_FILE), encoding="utf-8") project = projects_conf["project"] current_project = projects_conf["projects"][project] data_parts = pathlib.PurePath(current_project["data_path"]).parts data_path = CONF_PATH.joinpath(*data_parts).resolve() event_path = data_path / args.event raw_path = event_path / "raw" if not event_path.exists(): event_path.mkdir() raw_path.mkdir() raw_stream = read(args.seedfile) # some of these files are returned from CWB web site in time chunks # using this merge method joins up all of the traces with the same # NSCL. Thanks Obspy! stream = raw_stream.merge(fill_value="interpolate", interpolation_samples=-1) for trace in stream: network = trace.stats.network station = trace.stats.station channel = trace.stats.channel location = trace.stats.location starttime_str = trace.stats.starttime.strftime("%Y%m%dT%H%M%SZ") endtime_str = trace.stats.endtime.strftime("%Y%m%dT%H%M%SZ") fname = (f"{network}.{station}.{location}.{channel}__" f"{starttime_str}__{endtime_str}.mseed") filename = raw_path / fname trace.write(str(filename), format="MSEED") print(f"{len(stream)} channels written to {raw_path}.") responses = {} with tarfile.open(args.tarball, "r") as tarball: for member in tarball.getmembers(): if member.isdir(): continue with tarball.extractfile(member) as fh: inventory = read_inventory(fh) network = inventory.networks[0] netcode = network.code station = network.stations[0] stacode = station.code resp_name = f"{netcode}.{stacode}" if resp_name not in stations: print( f"No station coordinates available for station {resp_name}. Skipping." ) continue latitude = stations[resp_name]["latitude"] longitude = stations[resp_name]["longitude"] elevation = stations[resp_name]["elevation"] if resp_name in responses: old_inventory = responses[resp_name] old_station = old_inventory.networks[0].stations[0] new_station = inventory.networks[0].stations[0] new_channel = new_station.channels[0] new_channel.latitude = latitude new_channel.longitude = longitude new_channel.elevation = elevation old_station.channels.append(new_channel) else: for station in inventory.networks[0].stations: station.latitude = latitude station.longitude = longitude station.elevation = elevation for channel in station.channels: channel.latitude = latitude channel.longitude = longitude channel.elevation = elevation responses[resp_name] = inventory for resp_name, response in responses.items(): fname = resp_name + ".xml" filename = raw_path / fname response.write(str(filename), format="stationxml") print(f"{len(responses)} station responses written to {raw_path}.") scalar_event = get_event_object(args.event) create_event_file(scalar_event, str(event_path)) event_file = event_path / "event.json" msg = f"Created event file at {event_file}." if not event_file.exists(): msg = f"Error: Failed to create {event_file}." print(msg) download_rupture_file(args.event, str(event_path)) rupture_file = event_path / "rupture.json" msg = f"Created rupture file at {rupture_file}." if not rupture_file.exists(): msg = f"Error: Failed to create {rupture_file}." print(msg)
def test_lowpass_max(): datapath = os.path.join('data', 'testdata', 'lowpass_max') datadir = pkg_resources.resource_filename('gmprocess', datapath) sc = StreamCollection.from_directory(datadir) sc.describe() conf = get_config() update = { 'processing': [ { 'detrend': { 'detrending_method': 'demean' } }, { 'remove_response': { 'f1': 0.001, 'f2': 0.005, 'f3': None, 'f4': None, 'output': 'ACC', 'water_level': 60 } }, # {'detrend': {'detrending_method': 'linear'}}, # {'detrend': {'detrending_method': 'demean'}}, { 'get_corner_frequencies': { 'constant': { 'highpass': 0.08, 'lowpass': 20.0 }, 'method': 'constant', 'snr': { 'same_horiz': True } } }, { 'lowpass_max_frequency': { 'fn_fac': 0.9 } } ] } update_dict(conf, update) update = { 'windows': { 'signal_end': { 'method': 'model', 'vmin': 1.0, 'floor': 120, 'model': 'AS16', 'epsilon': 2.0 }, 'window_checks': { 'do_check': False, 'min_noise_duration': 1.0, 'min_signal_duration': 1.0 } } } update_dict(conf, update) edict = { 'id': 'ci38038071', 'time': UTCDateTime('2018-08-30 02:35:36'), 'lat': 34.136, 'lon': -117.775, 'depth': 5.5, 'magnitude': 4.4 } event = get_event_object(edict) test = process_streams(sc, event, conf) for st in test: for tr in st: freq_dict = tr.getParameter('corner_frequencies') np.testing.assert_allclose(freq_dict['lowpass'], 18.0)
def test_lowpass_max(): datapath = os.path.join("data", "testdata", "lowpass_max") datadir = pkg_resources.resource_filename("gmprocess", datapath) sc = StreamCollection.from_directory(datadir) sc.describe() conf = get_config() update = { "processing": [ { "detrend": { "detrending_method": "demean" } }, { "remove_response": { "f1": 0.001, "f2": 0.005, "f3": None, "f4": None, "water_level": 60, } }, # {'detrend': {'detrending_method': 'linear'}}, # {'detrend': {'detrending_method': 'demean'}}, { "get_corner_frequencies": { "constant": { "highpass": 0.08, "lowpass": 20.0 }, "method": "constant", "snr": { "same_horiz": True }, } }, { "lowpass_max_frequency": { "fn_fac": 0.9 } }, ] } update_dict(conf, update) update = { "windows": { "signal_end": { "method": "model", "vmin": 1.0, "floor": 120, "model": "AS16", "epsilon": 2.0, }, "window_checks": { "do_check": False, "min_noise_duration": 1.0, "min_signal_duration": 1.0, }, } } update_dict(conf, update) edict = { "id": "ci38038071", "time": UTCDateTime("2018-08-30 02:35:36"), "lat": 34.136, "lon": -117.775, "depth": 5.5, "magnitude": 4.4, } event = get_event_object(edict) test = process_streams(sc, event, conf) for st in test: for tr in st: freq_dict = tr.getParameter("corner_frequencies") np.testing.assert_allclose(freq_dict["lowpass"], 18.0)