def test_metrics(): eventid = 'usb000syza' datafiles, event = read_data_dir('knet', eventid, '*') datadir = os.path.split(datafiles[0])[0] raw_streams = StreamCollection.from_directory(datadir) config = get_config() # turn off sta/lta check and snr checks newconfig = drop_processing(config, ['check_sta_lta', 'compute_snr']) processed_streams = process_streams(raw_streams, event, config=newconfig) tdir = tempfile.mkdtemp() try: tfile = os.path.join(tdir, 'test.hdf') workspace = StreamWorkspace(tfile) workspace.addEvent(event) workspace.addStreams(event, processed_streams, label='processed') stream1 = processed_streams[0] stream2 = processed_streams[1] summary1 = StationSummary.from_config(stream1) summary2 = StationSummary.from_config(stream2) workspace.setStreamMetrics(event.id, 'processed', summary1) workspace.setStreamMetrics(event.id, 'processed', summary2) workspace.calcStationMetrics(event.id, labels=['processed']) summary1_a = workspace.getStreamMetrics(event.id, stream1[0].stats.station, 'processed') s1_df_in = summary1.pgms.sort_values(['IMT', 'IMC']) s1_df_out = summary1_a.pgms.sort_values(['IMT', 'IMC']) array1 = s1_df_in['Result'].as_matrix() array2 = s1_df_out['Result'].as_matrix() np.testing.assert_almost_equal(array1, array2, decimal=4) df = workspace.getMetricsTable(event.id) cmp_series = { 'GREATER_OF_TWO_HORIZONTALS': 0.6787, 'H1': 0.3869, 'H2': 0.6787, 'Z': 0.7663 } pga_dict = df.iloc[0]['PGA'].to_dict() for key, value in pga_dict.items(): value2 = cmp_series[key] np.testing.assert_almost_equal(value, value2, decimal=4) workspace.close() except Exception as e: raise(e) finally: shutil.rmtree(tdir)
def test_metrics(): eventid = 'usb000syza' datafiles, event = read_data_dir('knet', eventid, '*') datadir = os.path.split(datafiles[0])[0] raw_streams = StreamCollection.from_directory(datadir) config = get_config() # turn off sta/lta check and snr checks newconfig = drop_processing(config, ['check_sta_lta', 'compute_snr']) processed_streams = process_streams(raw_streams, event, config=newconfig) tdir = tempfile.mkdtemp() try: tfile = os.path.join(tdir, 'test.hdf') workspace = StreamWorkspace(tfile) workspace.addEvent(event) workspace.addStreams(event, processed_streams, label='processed') stream1 = processed_streams[0] stream2 = processed_streams[1] summary1 = StationSummary.from_config(stream1) summary2 = StationSummary.from_config(stream2) workspace.setStreamMetrics(event.id, 'processed', summary1) workspace.setStreamMetrics(event.id, 'processed', summary2) summary1_a = workspace.getStreamMetrics(event.id, stream1[0].stats.station, 'processed') s1_df_in = summary1.pgms.sort_values(['IMT', 'IMC']) s1_df_out = summary1_a.pgms.sort_values(['IMT', 'IMC']) array1 = s1_df_in['Result'].as_matrix() array2 = s1_df_out['Result'].as_matrix() np.testing.assert_almost_equal(array1, array2, decimal=4) df = workspace.getMetricsTable(event.id) cmp_series = {'GREATER_OF_TWO_HORIZONTALS': 0.6787, 'HN1': 0.3869, 'HN2': 0.6787, 'HNZ': 0.7663} pga_dict = df.iloc[0]['PGA'].to_dict() for key, value in pga_dict.items(): value2 = cmp_series[key] np.testing.assert_almost_equal(value, value2, decimal=4) workspace.close() except Exception as e: raise(e) finally: shutil.rmtree(tdir)
def test_workspace(): eventid = 'us1000778i' datafiles, origin = read_data_dir('geonet', eventid, '*.V1A') event = get_event_object(origin) tdir = tempfile.mkdtemp() try: with warnings.catch_warnings(): warnings.filterwarnings("ignore", category=H5pyDeprecationWarning) warnings.filterwarnings("ignore", category=YAMLLoadWarning) warnings.filterwarnings("ignore", category=FutureWarning) config = get_config() tfile = os.path.join(tdir, 'test.hdf') raw_streams = [] for dfile in datafiles: raw_streams += read_data(dfile) workspace = StreamWorkspace(tfile) t1 = time.time() workspace.addStreams(event, raw_streams, label='raw') t2 = time.time() print('Adding %i streams took %.2f seconds' % (len(raw_streams), (t2 - t1))) str_repr = workspace.__repr__() assert str_repr == 'Events: 1 Stations: 3 Streams: 3' eventobj = workspace.getEvent(eventid) assert eventobj.origins[0].latitude == event.origins[0].latitude assert eventobj.magnitudes[0].mag == event.magnitudes[0].mag stations = workspace.getStations() assert sorted(stations) == ['hses', 'thz', 'wtmc'] stations = workspace.getStations(eventid=eventid) assert sorted(stations) == ['hses', 'thz', 'wtmc'] # test retrieving tags for an event that doesn't exist try: workspace.getStreamTags('foo') except KeyError: assert 1 == 1 # test retrieving event that doesn't exist try: workspace.getEvent('foo') except KeyError: assert 1 == 1 instream = None for stream in raw_streams: if stream[0].stats.station.lower() == 'hses': instream = stream break if instream is None: assert 1 == 2 outstream = workspace.getStreams(eventid, stations=['hses'], labels=['raw'])[0] compare_streams(instream, outstream) label_summary = workspace.summarizeLabels() assert label_summary.iloc[0]['Label'] == 'raw' assert label_summary.iloc[0]['Software'] == 'gmprocess' sc = StreamCollection(raw_streams) processed_streams = process_streams(sc, origin, config=config) workspace.addStreams(event, processed_streams, 'processed') idlist = workspace.getEventIds() assert idlist[0] == eventid event_tags = workspace.getStreamTags(eventid) assert sorted(event_tags) == [ 'hses_processed', 'hses_raw', 'thz_processed', 'thz_raw', 'wtmc_processed', 'wtmc_raw' ] outstream = workspace.getStreams(eventid, stations=['hses'], labels=['processed'])[0] provenance = workspace.getProvenance(eventid, labels=['processed']) first_row = pd.Series({ 'Record': 'NZ.HSES.HN1', 'Processing Step': 'Remove Response', 'Step Attribute': 'input_units', 'Attribute Value': 'counts' }) last_row = pd.Series({ 'Record': 'NZ.WTMC.HNZ', 'Processing Step': 'Detrend', 'Step Attribute': 'detrending_method', 'Attribute Value': 'baseline_sixth_order' }) assert provenance.iloc[0].equals(first_row) assert provenance.iloc[-1].equals(last_row) # compare the parameters from the input processed stream # to it's output equivalent instream = None for stream in processed_streams: if stream[0].stats.station.lower() == 'hses': instream = stream break if instream is None: assert 1 == 2 compare_streams(instream, outstream) workspace.close() # read in data from a second event and stash it in the workspace eventid = 'nz2018p115908' datafiles, origin = read_data_dir('geonet', eventid, '*.V2A') raw_streams = [] for dfile in datafiles: raw_streams += read_data(dfile) event = get_event_object(origin) workspace = StreamWorkspace.open(tfile) workspace.addStreams(event, raw_streams, label='foo') stations = workspace.getStations(eventid) eventids = workspace.getEventIds() assert eventids == ['us1000778i', 'nz2018p115908'] instation = raw_streams[0][0].stats.station this_stream = workspace.getStreams(eventid, stations=[instation], labels=['foo'])[0] assert instation == this_stream[0].stats.station # set and retrieve waveform metrics in the file imclist = [ 'greater_of_two_horizontals', 'channels', 'rotd50', 'rotd100' ] imtlist = ['sa1.0', 'PGA', 'pgv', 'fas2.0', 'arias'] usid = 'us1000778i' tags = workspace.getStreamTags(usid) workspace.setStreamMetrics(eventid, labels=['foo'], imclist=imclist, imtlist=imtlist) summary = workspace.getStreamMetrics(eventid, instation, 'foo') summary_series = summary.toSeries()['ARIAS'] cmpseries = pd.Series({ 'GEOMETRIC_MEAN': np.NaN, 'GREATER_OF_TWO_HORIZONTALS': 0.0005, 'HN1': 0.0001, 'HN2': 0.0005, 'HNZ': 0.0000, 'ROTD100.0': 0.0005, 'ROTD50.0': 0.0003 }) assert cmpseries.equals(summary_series) workspace.setStreamMetrics(usid, labels=['processed']) df = workspace.getMetricsTable(usid, labels=['processed']) cmpdict = { 'GREATER_OF_TWO_HORIZONTALS': [26.8906, 4.9415, 94.6646], 'HN1': [24.5105, 4.9415, 94.6646], 'HN2': [26.8906, 4.0758, 86.7877], 'HNZ': [16.0941, 2.5401, 136.7054] } cmpframe = pd.DataFrame(cmpdict) assert df['PGA'].equals(cmpframe) inventory = workspace.getInventory(usid) codes = [ station.code for station in inventory.networks[0].stations ] assert sorted(codes) == ['HSES', 'THZ', 'WPWS', 'WTMC'] except Exception as e: raise (e) finally: shutil.rmtree(tdir)