def build_anxcor(tau,interpolation_method='nearest'): broadband_data_dir = basedir+'test_data/correlation_integration_testing/Broadband' broadband_station_location_file = basedir+'test_data/correlation_integration_testing/broadband_stationlist.txt' nodal_data_dir = basedir+'test_data/correlation_integration_testing/Nodal' nodal_station_location_file = basedir+'test_data/correlation_integration_testing/nodal_stationlist.txt' broadband_database = DWellsDecimatedReader(broadband_data_dir, broadband_station_location_file) nodal_database = DWellsDecimatedReader(nodal_data_dir, nodal_station_location_file,extension='d') window_length = 10*60.0 #include_stations = ['Nodal.{}'.format(x) for x in range(1,10)] include_stations = ['UU.FORU','DV.1'] taper_ratio = 0.05 target_rate = 50.0 correlate_kwargs= dict(max_tau_shift=tau,taper=taper_ratio) resample_kwargs = dict(target_rate=target_rate,lowpass=False) anxcor_main = Anxcor(interp_method=interpolation_method) anxcor_main.set_window_length(window_length) anxcor_main.set_must_only_include_station_pairs(include_stations) anxcor_main.add_dataset(broadband_database,'BB') anxcor_main.add_dataset(nodal_database, 'Nodal') anxcor_main.add_process(XArrayTaper(taper=taper_ratio)) anxcor_main.add_process(XArrayRemoveMeanTrend()) anxcor_main.add_process(XArrayTaper(taper=taper_ratio)) anxcor_main.set_task_kwargs('crosscorrelate',correlate_kwargs) # anxcor_main.set_task('post-correlate',XArrayCustomComponentNormalizer()) return anxcor_main
def test_dask_read_combine_instastack(self): from distributed import Client, LocalCluster from dask.distributed import wait cluster = LocalCluster(n_workers=1, threads_per_worker=1) c = Client(cluster) anxcor = Anxcor() anxcor.set_window_length(120.0) times = anxcor.get_starttimes(starttime_stamp, endtime_stamp, 0.5) bank = WavebankWrapper(source_dir) anxcor.add_dataset(bank, 'nodals') anxcor.save_at_task(target_dir, 'combine') result = anxcor.process(times,dask_client=c,stack=True) anxcor = Anxcor() anxcor.set_window_length(120.0) bank = WavebankWrapper(source_dir) anxcor.add_dataset(bank, 'nodals') anxcor.load_at_task(target_dir, 'combine') result = anxcor.process(times,dask_client=c,stack=True) how_many_nc = _how_many_fmt(target_dir, format='.nc') _clean_files_in_dir(target_dir) c.close() cluster.close() assert 48 == how_many_nc
def test_filter_exclude(self): anxcor = Anxcor() anxcor.set_window_length(120.0) bank = WavebankWrapper(source_dir) anxcor.add_dataset(bank, 'nodals') anxcor.set_must_exclude_single_stations('AX.1') stations = anxcor.get_station_combinations() assert len(stations.index) == 3, 'too many stations retained'
def test_pair_preservation(self): anxcor = Anxcor() anxcor.set_window_length(120.0) times = anxcor.get_starttimes(starttime_stamp,endtime_stamp, 0.5) bank = WavebankWrapper(source_dir) anxcor.add_dataset(bank, 'nodals') result = anxcor.process(times) pairs = list(result.coords['rec'].values) + list(result.coords['src'].values) assert 6 ==len(pairs)
def test_metadata_with_latlons(self): anxcor = Anxcor() anxcor.set_window_length(100) times = anxcor.get_starttimes(starttime_stamp, starttime_stamp + 2 * 100, 0.5) bank = WavebankWrapperWLatLons(source_dir) anxcor.add_dataset(bank, 'nodals') result = anxcor.process(times) pair_dict = result.attrs['df'] assert 'src_latitude' in pair_dict.columns
def test_load_result(self): anxcor = Anxcor() anxcor.set_window_length(120.0) times = anxcor.get_starttimes(starttime_stamp, endtime_stamp, 0.5) bank = WavebankWrapper(source_dir) anxcor.add_dataset(bank, 'nodals') original_result = anxcor.process(times) anxcor.save_result(original_result, target_dir) final_result = anxcor.load_result(target_dir) assert len(final_result.to_array().squeeze().data.shape) == 5
def test_bandpass_result(self): bp = XArrayBandpass(freqmin=0.1, freqmax=10.0) anxcor = Anxcor() anxcor.set_window_length(120.0) times = anxcor.get_starttimes(starttime_stamp, endtime_stamp, 0.5) bank = WavebankWrapper(source_dir) anxcor.add_dataset(bank, 'nodals') result = anxcor.process(times) bp_result = bp(result) assert isinstance(bp_result, xr.Dataset)
def test_save_result(self): anxcor = Anxcor() anxcor.set_window_length(120.0) times = anxcor.get_starttimes(starttime_stamp, endtime_stamp, 0.5) bank = WavebankWrapper(source_dir) anxcor.add_dataset(bank, 'nodals') result = anxcor.process(times) anxcor.save_result(result, target_dir) num_nc = _how_many_fmt(target_dir, format='.nc') assert num_nc == 1
def test_write_combine(self): anxcor = Anxcor() anxcor.set_window_length(120.0) times = anxcor.get_starttimes(starttime_stamp,endtime_stamp, 0.5) bank = WavebankWrapper(source_dir) anxcor.add_dataset(bank, 'nodals') anxcor.save_at_task(target_dir, 'combine') result = anxcor.process(times) how_many_nc = _how_many_fmt(target_dir, format='.nc') _clean_files_in_dir(target_dir) assert 48 == how_many_nc
def test_stacking_preserves_pair_key(self): anxcor = Anxcor() anxcor.set_window_length(100) times = anxcor.get_starttimes(starttime_stamp, starttime_stamp + 2 * 100, 0.5) bank = WavebankWrapper(source_dir) anxcor.add_dataset(bank, 'nodals') result = anxcor.process(times) pairs = result.coords['rec'].values assert 'AX.1' in list(pairs) and 'AX.2' in list(pairs)
def test_output_dataset_format(self): anxcor = Anxcor() anxcor.set_window_length(100) times = anxcor.get_starttimes(starttime_stamp, starttime_stamp + 2 * 100, 0.5) bank = WavebankWrapperWLatLons(source_dir) anxcor.add_dataset(bank, 'nodals') result = anxcor.process(times) len_variables = len(list(result.data_vars)) # assert len_variables == 1, 'too many variables added to dataset'
def test_single_execution(self): # stations 21, & 22 # 3 windows say # anxcor = Anxcor() anxcor.set_window_length(120.0) bank = WavebankWrapper(source_dir) anxcor.add_dataset(bank, 'nodals') result = anxcor.process([starttime_stamp]) streams = anxcor.xarray_to_obspy(result) assert len(streams) == 25, 'not enough traces retained!'
def test_write_resample(self): anxcor = Anxcor() anxcor.set_window_length(120.0) times = anxcor.get_starttimes(starttime_stamp,endtime_stamp, 0.5) bank = WavebankWrapper(source_dir) anxcor.add_dataset(bank, 'nodals') anxcor.add_process(XArrayResample(target_rate=10.0)) anxcor.save_at_process(target_dir, 'resample:0') result = anxcor.process(times) how_many_nc = _how_many_fmt(target_dir, format='.nc') _clean_files_in_dir(target_dir) assert 20 == how_many_nc
def test_single_execution(self): # stations 21, & 22 # 3 windows say # anxcor = Anxcor() anxcor.set_window_length(120.0) times = anxcor.get_starttimes(starttime_stamp,endtime_stamp, 0.5) bank = WavebankWrapper(source_dir) anxcor.add_dataset(bank, 'nodals') result = anxcor.process(times) pairs = list(result.coords['rec'].values) + list(result.coords['src'].values) assert len(pairs) == 6
def test_write_tempnorm(self): anxcor = Anxcor() anxcor.set_window_length(120.0) times = anxcor.get_starttimes(starttime_stamp,endtime_stamp, 0.5) bank = WavebankWrapper(source_dir) anxcor.add_dataset(bank, 'nodals') anxcor.add_process(XArrayTemporalNorm(time_window=5.0, freqmin=0.02)) anxcor.save_at_process(target_dir,'temp_norm:0') result = anxcor.process(times) how_many_nc = _how_many_fmt(target_dir, format='.nc') _clean_files_in_dir(target_dir) assert 20 == how_many_nc
def test_dask_execution(self): from distributed import Client, LocalCluster cluster = LocalCluster(n_workers=1, threads_per_worker=1) c = Client(cluster) anxcor = Anxcor() anxcor.set_window_length(120.0) times = anxcor.get_starttimes(starttime_stamp,endtime_stamp, 0.5) bank = WavebankWrapper(source_dir) anxcor.add_dataset(bank, 'nodals') result = anxcor.process(times,dask_client=c) pairs = list(result.coords['rec'].values)+ list(result.coords['src'].values) c.close() cluster.close() assert 6 ==len(pairs)
def test_load_config_with_routine(self): time_mean_target = 15 anxcor = Anxcor() anxcor.set_window_length(120.0) anxcor.add_process( XArrayTemporalNorm(window=time_mean_target, rolling_metric='max')) anxcor.save_config(save_dir + os.sep + 'config.ini') anxcor2 = Anxcor() anxcor.set_window_length(120.0) anxcor2.add_process(XArrayTemporalNorm()) anxcor2.load_config(save_dir + os.sep + 'config.ini') tnorm = anxcor2._get_task('process')['temp_norm:0'] time_mean_src = tnorm._kwargs['window'] assert time_mean_target == time_mean_src
def test_dask_execution_exclude_with_stack_number(self): from distributed import Client, LocalCluster cluster = LocalCluster(n_workers=1, threads_per_worker=1) c = Client(cluster) anxcor = Anxcor() anxcor.set_window_length(120) anxcor.set_task_kwargs('crosscorrelate', dict(max_tau_shift=20.0)) times = anxcor.get_starttimes(starttime_stamp, endtime_stamp, 0.5) bank = WavebankWrapper(source_dir) anxcor.set_must_exclude_single_stations('AX.1') anxcor.add_dataset(bank, 'nodals') result = anxcor.process(times, dask_client=c, stack=10) pairs = list(result.coords['rec'].values) + list(result.coords['src'].values) c.close() cluster.close() assert 4 == len(pairs)
def test_read_xconvert(self): anxcor = Anxcor() anxcor.set_window_length(120.0) times = anxcor.get_starttimes(starttime_stamp,endtime_stamp, 0.5) bank = WavebankWrapper(source_dir) anxcor.add_dataset(bank, 'nodals') anxcor.save_at_task(target_dir,'xconvert') result = anxcor.process(times) anxcor = Anxcor() anxcor.set_window_length(120.0) bank = WavebankWrapper(source_dir) anxcor.add_dataset(bank, 'nodals') anxcor.load_at_task(target_dir, 'xconvert') result = anxcor.process(times) _clean_files_in_dir(target_dir) assert isinstance(result,xr.Dataset)
def test_output_dimension_lengths(self): anxcor = Anxcor() anxcor.set_window_length(100) times = anxcor.get_starttimes(starttime_stamp, starttime_stamp + 2 * 100, 0.5) bank = WavebankWrapperWLatLons(source_dir) anxcor.add_dataset(bank, 'nodals') result = anxcor.process(times) len_src_chan = len(list(result.coords['src_chan'].values)) len_rec_chan = len(list(result.coords['rec_chan'].values)) len_src = len(list(result.coords['src'].values)) len_rec = len(list(result.coords['rec'].values)) # assert len_src_chan == 3, 'not enough src chan retained' assert len_rec_chan == 3, 'not enough rec chan retained' assert len_src == 3, 'not enough sources retained' assert len_rec == 3, 'not enough receivers retained'
def test_dask_execution(self): # created test after observing incorrect time array conversion. Will test again on cluster from distributed import Client, LocalCluster cluster = LocalCluster() c = Client(cluster) anxcor = Anxcor() anxcor.set_window_length(150) starttime = 0.0 times = [starttime, starttime + 60] bank = WavebankWrapper(source_dir) anxcor.add_dataset(bank, 'nodals') result = anxcor.process(times, dask_client=c) if isinstance(result, Future): print('future') result = result.result() rec_chan = list(result.coords['rec_chan'].values) assert 3 == len(rec_chan)
def test_write_tempnorm_dask(self): from distributed import Client, LocalCluster from dask.distributed import wait cluster = LocalCluster(n_workers=1, threads_per_worker=1) c = Client(cluster) anxcor = Anxcor() anxcor.set_window_length(120.0) times = anxcor.get_starttimes(starttime_stamp,endtime_stamp, 0.5) bank = WavebankWrapper(source_dir) anxcor.add_dataset(bank, 'nodals') anxcor.add_process(XArrayTemporalNorm(time_window=5.0, freqmin=0.02)) anxcor.save_at_process(target_dir,'temp_norm:0') result = anxcor.process(times,dask_client=c) print(result) how_many_nc = _how_many_fmt(target_dir, format='.nc') _clean_files_in_dir(target_dir) assert 20 == how_many_nc
def test_read_every_stack(self): anxcor = Anxcor() anxcor.set_window_length(120.0) times = anxcor.get_starttimes(starttime_stamp,endtime_stamp, 0.5) bank = WavebankWrapper(source_dir) anxcor.add_dataset(bank, 'nodals') anxcor.save_at_task(target_dir, 'stack') result = anxcor.process(times) anxcor = Anxcor() anxcor.set_window_length(120.0) times = anxcor.get_starttimes(starttime_stamp,endtime_stamp, 0.5) bank = WavebankWrapper(source_dir) anxcor.add_dataset(bank, 'nodals') anxcor.load_at_task(target_dir,'stack') result = anxcor.process(times,stack=4) how_many_nc = _how_many_fmt(target_dir, format='.nc') _clean_files_in_dir(target_dir) assert 2 == how_many_nc
def test_read_post_combine(self): anxcor = Anxcor() anxcor.set_window_length(120.0) times = anxcor.get_starttimes(starttime_stamp,endtime_stamp, 0.5) bank = WavebankWrapper(source_dir) anxcor.add_dataset(bank, 'nodals') anxcor.save_at_task(target_dir,'post-combine') result = anxcor.process(times) anxcor = Anxcor() anxcor.set_window_length(120.0) bank = WavebankWrapper(source_dir) anxcor.add_dataset(bank, 'nodals') anxcor.load_at_task(target_dir,'post-combine') result = anxcor.process(times) _clean_files_in_dir(target_dir) print(result) assert len(result.coords['src_chan'].values)==3
def test_single_execution(self): anxcorWavebank = WavebankWrapper(source_dir) min_starttime = anxcorWavebank.bank.get_availability_df( )['starttime'].min() endtime = anxcorWavebank.bank.get_availability_df()['endtime'].max() starttime = int(min_starttime + 1) endtime = int(endtime) overlap = 0.5 window_length = 3 * 60 resample_kwargs = dict(taper=0.05, target_rate=20.0) correlate_kwargs = dict(taper=0.01, max_tau_shift=60) anxcor_main = Anxcor() anxcor_main.set_window_length(window_length) anxcor_main.add_dataset(anxcorWavebank, 'test') anxcor_main.set_task_kwargs('resample', resample_kwargs) anxcor_main.set_task_kwargs('crosscorrelate', correlate_kwargs) starttime_list = anxcor_main.get_starttimes(starttime, starttime + window_length, overlap) result = anxcor_main.process(starttime_list) assert len(result['src:test rec:test'].coords['rec_chan'].values) == 3 assert len(result['src:test rec:test'].coords['src_chan'].values) == 3 assert not np.isnan(result['src:test rec:test']).any()