def build_anxcor(tau,interpolation_method='nearest'):
broadband_data_dir = basedir+'test_data/correlation_integration_testing/Broadband'
broadband_station_location_file = basedir+'test_data/correlation_integration_testing/broadband_stationlist.txt'
nodal_data_dir = basedir+'test_data/correlation_integration_testing/Nodal'
nodal_station_location_file = basedir+'test_data/correlation_integration_testing/nodal_stationlist.txt'
broadband_database = DWellsDecimatedReader(broadband_data_dir, broadband_station_location_file)
nodal_database = DWellsDecimatedReader(nodal_data_dir, nodal_station_location_file,extension='d')
window_length = 10*60.0
#include_stations = ['Nodal.{}'.format(x) for x in range(1,10)]
include_stations = ['UU.FORU','DV.1']
taper_ratio = 0.05
target_rate = 50.0
correlate_kwargs= dict(max_tau_shift=tau,taper=taper_ratio)
resample_kwargs = dict(target_rate=target_rate,lowpass=False)
anxcor_main = Anxcor(interp_method=interpolation_method)
anxcor_main.set_window_length(window_length)
anxcor_main.set_must_only_include_station_pairs(include_stations)
anxcor_main.add_dataset(broadband_database,'BB')
anxcor_main.add_dataset(nodal_database, 'Nodal')
anxcor_main.add_process(XArrayTaper(taper=taper_ratio))
anxcor_main.add_process(XArrayRemoveMeanTrend())
anxcor_main.add_process(XArrayTaper(taper=taper_ratio))
anxcor_main.set_task_kwargs('crosscorrelate',correlate_kwargs)
# anxcor_main.set_task('post-correlate',XArrayCustomComponentNormalizer())
return anxcor_main
def test_dask_read_combine_instastack(self):
from distributed import Client, LocalCluster
from dask.distributed import wait
cluster = LocalCluster(n_workers=1, threads_per_worker=1)
c = Client(cluster)
anxcor = Anxcor()
anxcor.set_window_length(120.0)
times = anxcor.get_starttimes(starttime_stamp, endtime_stamp, 0.5)
bank = WavebankWrapper(source_dir)
anxcor.add_dataset(bank, 'nodals')
anxcor.save_at_task(target_dir, 'combine')
result = anxcor.process(times,dask_client=c,stack=True)
anxcor = Anxcor()
anxcor.set_window_length(120.0)
bank = WavebankWrapper(source_dir)
anxcor.add_dataset(bank, 'nodals')
anxcor.load_at_task(target_dir, 'combine')
result = anxcor.process(times,dask_client=c,stack=True)
how_many_nc = _how_many_fmt(target_dir, format='.nc')
_clean_files_in_dir(target_dir)
c.close()
cluster.close()
assert 48 == how_many_nc
def test_filter_exclude(self):
anxcor = Anxcor()
anxcor.set_window_length(120.0)
bank = WavebankWrapper(source_dir)
anxcor.add_dataset(bank, 'nodals')
anxcor.set_must_exclude_single_stations('AX.1')
stations = anxcor.get_station_combinations()
assert len(stations.index) == 3, 'too many stations retained'
def test_pair_preservation(self):
anxcor = Anxcor()
anxcor.set_window_length(120.0)
times = anxcor.get_starttimes(starttime_stamp,endtime_stamp, 0.5)
bank = WavebankWrapper(source_dir)
anxcor.add_dataset(bank, 'nodals')
result = anxcor.process(times)
pairs = list(result.coords['rec'].values) + list(result.coords['src'].values)
assert 6 ==len(pairs)
def test_load_result(self):
anxcor = Anxcor()
anxcor.set_window_length(120.0)
times = anxcor.get_starttimes(starttime_stamp, endtime_stamp, 0.5)
bank = WavebankWrapper(source_dir)
anxcor.add_dataset(bank, 'nodals')
original_result = anxcor.process(times)
anxcor.save_result(original_result, target_dir)
final_result = anxcor.load_result(target_dir)
assert len(final_result.to_array().squeeze().data.shape) == 5
def test_save_result(self):
anxcor = Anxcor()
anxcor.set_window_length(120.0)
times = anxcor.get_starttimes(starttime_stamp, endtime_stamp, 0.5)
bank = WavebankWrapper(source_dir)
anxcor.add_dataset(bank, 'nodals')
result = anxcor.process(times)
anxcor.save_result(result, target_dir)
num_nc = _how_many_fmt(target_dir, format='.nc')
assert num_nc == 1
def test_bandpass_result(self):
bp = XArrayBandpass(freqmin=0.1, freqmax=10.0)
anxcor = Anxcor()
anxcor.set_window_length(120.0)
times = anxcor.get_starttimes(starttime_stamp, endtime_stamp, 0.5)
bank = WavebankWrapper(source_dir)
anxcor.add_dataset(bank, 'nodals')
result = anxcor.process(times)
bp_result = bp(result)
assert isinstance(bp_result, xr.Dataset)
def test_metadata_with_latlons(self):
anxcor = Anxcor()
anxcor.set_window_length(100)
times = anxcor.get_starttimes(starttime_stamp,
starttime_stamp + 2 * 100, 0.5)
bank = WavebankWrapperWLatLons(source_dir)
anxcor.add_dataset(bank, 'nodals')
result = anxcor.process(times)
pair_dict = result.attrs['df']
assert 'src_latitude' in pair_dict.columns
def test_output_dataset_format(self):
anxcor = Anxcor()
anxcor.set_window_length(100)
times = anxcor.get_starttimes(starttime_stamp,
starttime_stamp + 2 * 100, 0.5)
bank = WavebankWrapperWLatLons(source_dir)
anxcor.add_dataset(bank, 'nodals')
result = anxcor.process(times)
len_variables = len(list(result.data_vars))
#
assert len_variables == 1, 'too many variables added to dataset'
def test_write_combine(self):
anxcor = Anxcor()
anxcor.set_window_length(120.0)
times = anxcor.get_starttimes(starttime_stamp,endtime_stamp, 0.5)
bank = WavebankWrapper(source_dir)
anxcor.add_dataset(bank, 'nodals')
anxcor.save_at_task(target_dir, 'combine')
result = anxcor.process(times)
how_many_nc = _how_many_fmt(target_dir, format='.nc')
_clean_files_in_dir(target_dir)
assert 48 == how_many_nc
def test_stacking_preserves_pair_key(self):
anxcor = Anxcor()
anxcor.set_window_length(100)
times = anxcor.get_starttimes(starttime_stamp,
starttime_stamp + 2 * 100, 0.5)
bank = WavebankWrapper(source_dir)
anxcor.add_dataset(bank, 'nodals')
result = anxcor.process(times)
pairs = result.coords['rec'].values
assert 'AX.1' in list(pairs) and 'AX.2' in list(pairs)
def test_single_execution(self):
# stations 21, & 22
# 3 windows say
#
anxcor = Anxcor()
anxcor.set_window_length(120.0)
bank = WavebankWrapper(source_dir)
anxcor.add_dataset(bank, 'nodals')
result = anxcor.process([starttime_stamp])
streams = anxcor.xarray_to_obspy(result)
assert len(streams) == 25, 'not enough traces retained!'
def test_rotation(self):
# stations 21, & 22
# 3 windows say
#
anxcor = Anxcor(3600)
bank = WavebankWrapper(source_dir)
anxcor.add_dataset(bank, 'nodals')
anxcor.set_task_kwargs('crosscorrelate', dict(dummy_task=True))
result = anxcor.process([starttime_stamp])
rotated_result = anxcor.align_station_pairs(result)
streams = anxcor.xarray_to_obspy(result)
assert len(streams) == 9, 'not enough traces retained!'
def test_single_execution(self):
# stations 21, & 22
# 3 windows say
#
anxcor = Anxcor()
anxcor.set_window_length(120.0)
times = anxcor.get_starttimes(starttime_stamp,endtime_stamp, 0.5)
bank = WavebankWrapper(source_dir)
anxcor.add_dataset(bank, 'nodals')
result = anxcor.process(times)
pairs = list(result.coords['rec'].values) + list(result.coords['src'].values)
assert len(pairs) == 6
def test_write_resample(self):
anxcor = Anxcor()
anxcor.set_window_length(120.0)
times = anxcor.get_starttimes(starttime_stamp,endtime_stamp, 0.5)
bank = WavebankWrapper(source_dir)
anxcor.add_dataset(bank, 'nodals')
anxcor.add_process(XArrayResample(target_rate=10.0))
anxcor.save_at_process(target_dir, 'resample:0')
result = anxcor.process(times)
how_many_nc = _how_many_fmt(target_dir, format='.nc')
_clean_files_in_dir(target_dir)
assert 20 == how_many_nc
def test_write_tempnorm(self):
anxcor = Anxcor()
anxcor.set_window_length(120.0)
times = anxcor.get_starttimes(starttime_stamp,endtime_stamp, 0.5)
bank = WavebankWrapper(source_dir)
anxcor.add_dataset(bank, 'nodals')
anxcor.add_process(XArrayTemporalNorm(time_window=5.0, freqmin=0.02))
anxcor.save_at_process(target_dir,'temp_norm:0')
result = anxcor.process(times)
how_many_nc = _how_many_fmt(target_dir, format='.nc')
_clean_files_in_dir(target_dir)
assert 20 == how_many_nc
def test_expected_pair_amount(self):
anxcor_main = Anxcor(60 * 10.0)
anxcor_main.add_dataset(IRISWrapper(), 'IMUSH_ST_HELLENS_DATA')
correlate_kwargs = dict(taper=0.1, max_tau_shift=50.0)
anxcor_main.set_task_kwargs('crosscorrelate', correlate_kwargs)
anxcor_main.add_process(XArrayRemoveMeanTrend())
anxcor_main.add_process(XArrayOneBit())
starttime = UTCDateTime("2005-6-22 12:00:00").timestamp
starttimes = []
for window_number in range(0, 4):
starttimes.append(starttime + 60 * 10.0 * window_number)
xarray_dataset = anxcor_main.process(starttimes)
assert len(list(xarray_dataset.coords['rec'].values)) == 2
def test_dask_execution(self):
from distributed import Client, LocalCluster
cluster = LocalCluster(n_workers=1, threads_per_worker=1)
c = Client(cluster)
anxcor = Anxcor()
anxcor.set_window_length(120.0)
times = anxcor.get_starttimes(starttime_stamp,endtime_stamp, 0.5)
bank = WavebankWrapper(source_dir)
anxcor.add_dataset(bank, 'nodals')
result = anxcor.process(times,dask_client=c)
pairs = list(result.coords['rec'].values)+ list(result.coords['src'].values)
c.close()
cluster.close()
assert 6 ==len(pairs)
def test_set_must_include(self):
anxcorWavebank = WavebankWrapper(source_dir)
min_starttime = anxcorWavebank.bank.get_availability_df(
)['starttime'].min()
endtime = anxcorWavebank.bank.get_availability_df()['endtime'].max()
starttime = int(min_starttime + 1)
endtime = int(endtime)
overlap = 0.5
window_length = 3 * 60
resample_kwargs = dict(taper=0.05, target_rate=20.0)
correlate_kwargs = dict(taper=0.01, max_tau_shift=60)
anxcor_main = Anxcor()
anxcor_main.add_dataset(anxcorWavebank, 'test')
anxcor_main.set_must_only_include_station_pairs(['AX.1', 'AX.2'])
correlations = anxcor_main.get_station_combinations()
assert len(correlations) == 3
def test_read_xconvert(self):
anxcor = Anxcor()
anxcor.set_window_length(120.0)
times = anxcor.get_starttimes(starttime_stamp,endtime_stamp, 0.5)
bank = WavebankWrapper(source_dir)
anxcor.add_dataset(bank, 'nodals')
anxcor.save_at_task(target_dir,'xconvert')
result = anxcor.process(times)
anxcor = Anxcor()
anxcor.set_window_length(120.0)
bank = WavebankWrapper(source_dir)
anxcor.add_dataset(bank, 'nodals')
anxcor.load_at_task(target_dir, 'xconvert')
result = anxcor.process(times)
_clean_files_in_dir(target_dir)
assert isinstance(result,xr.Dataset)
def test_dask_execution_exclude_with_stack_number(self):
from distributed import Client, LocalCluster
cluster = LocalCluster(n_workers=1, threads_per_worker=1)
c = Client(cluster)
anxcor = Anxcor()
anxcor.set_window_length(120)
anxcor.set_task_kwargs('crosscorrelate', dict(max_tau_shift=20.0))
times = anxcor.get_starttimes(starttime_stamp, endtime_stamp, 0.5)
bank = WavebankWrapper(source_dir)
anxcor.set_must_exclude_single_stations('AX.1')
anxcor.add_dataset(bank, 'nodals')
result = anxcor.process(times, dask_client=c, stack=10)
pairs = list(result.coords['rec'].values) + list(result.coords['src'].values)
c.close()
cluster.close()
assert 4 == len(pairs)
def test_write_tempnorm_dask(self):
from distributed import Client, LocalCluster
from dask.distributed import wait
cluster = LocalCluster(n_workers=1, threads_per_worker=1)
c = Client(cluster)
anxcor = Anxcor()
anxcor.set_window_length(120.0)
times = anxcor.get_starttimes(starttime_stamp,endtime_stamp, 0.5)
bank = WavebankWrapper(source_dir)
anxcor.add_dataset(bank, 'nodals')
anxcor.add_process(XArrayTemporalNorm(time_window=5.0, freqmin=0.02))
anxcor.save_at_process(target_dir,'temp_norm:0')
result = anxcor.process(times,dask_client=c)
print(result)
how_many_nc = _how_many_fmt(target_dir, format='.nc')
_clean_files_in_dir(target_dir)
assert 20 == how_many_nc
def test_output_dimension_lengths(self):
anxcor = Anxcor()
anxcor.set_window_length(100)
times = anxcor.get_starttimes(starttime_stamp,
starttime_stamp + 2 * 100, 0.5)
bank = WavebankWrapperWLatLons(source_dir)
anxcor.add_dataset(bank, 'nodals')
result = anxcor.process(times)
len_src_chan = len(list(result.coords['src_chan'].values))
len_rec_chan = len(list(result.coords['rec_chan'].values))
len_src = len(list(result.coords['src'].values))
len_rec = len(list(result.coords['rec'].values))
#
assert len_src_chan == 3, 'not enough src chan retained'
assert len_rec_chan == 3, 'not enough rec chan retained'
assert len_src == 3, 'not enough sources retained'
assert len_rec == 3, 'not enough receivers retained'
def test_dask_execution(self):
# created test after observing incorrect time array conversion. Will test again on cluster
from distributed import Client, LocalCluster
cluster = LocalCluster()
c = Client(cluster)
anxcor = Anxcor()
anxcor.set_window_length(150)
starttime = 0.0
times = [starttime, starttime + 60]
bank = WavebankWrapper(source_dir)
anxcor.add_dataset(bank, 'nodals')
result = anxcor.process(times, dask_client=c)
if isinstance(result, Future):
print('future')
result = result.result()
rec_chan = list(result.coords['rec_chan'].values)
assert 3 == len(rec_chan)
def test_read_every_stack(self):
anxcor = Anxcor()
anxcor.set_window_length(120.0)
times = anxcor.get_starttimes(starttime_stamp,endtime_stamp, 0.5)
bank = WavebankWrapper(source_dir)
anxcor.add_dataset(bank, 'nodals')
anxcor.save_at_task(target_dir, 'stack')
result = anxcor.process(times)
anxcor = Anxcor()
anxcor.set_window_length(120.0)
times = anxcor.get_starttimes(starttime_stamp,endtime_stamp, 0.5)
bank = WavebankWrapper(source_dir)
anxcor.add_dataset(bank, 'nodals')
anxcor.load_at_task(target_dir,'stack')
result = anxcor.process(times,stack=4)
how_many_nc = _how_many_fmt(target_dir, format='.nc')
_clean_files_in_dir(target_dir)
assert 2 == how_many_nc
def test_read_post_combine(self):
anxcor = Anxcor()
anxcor.set_window_length(120.0)
times = anxcor.get_starttimes(starttime_stamp,endtime_stamp, 0.5)
bank = WavebankWrapper(source_dir)
anxcor.add_dataset(bank, 'nodals')
anxcor.save_at_task(target_dir,'post-combine')
result = anxcor.process(times)
anxcor = Anxcor()
anxcor.set_window_length(120.0)
bank = WavebankWrapper(source_dir)
anxcor.add_dataset(bank, 'nodals')
anxcor.load_at_task(target_dir,'post-combine')
result = anxcor.process(times)
_clean_files_in_dir(target_dir)
print(result)
assert len(result.coords['src_chan'].values)==3
def test_single_execution(self):
anxcorWavebank = WavebankWrapper(source_dir)
min_starttime = anxcorWavebank.bank.get_availability_df(
)['starttime'].min()
endtime = anxcorWavebank.bank.get_availability_df()['endtime'].max()
starttime = int(min_starttime + 1)
endtime = int(endtime)
overlap = 0.5
window_length = 3 * 60
resample_kwargs = dict(taper=0.05, target_rate=20.0)
correlate_kwargs = dict(taper=0.01, max_tau_shift=60)
anxcor_main = Anxcor()
anxcor_main.set_window_length(window_length)
anxcor_main.add_dataset(anxcorWavebank, 'test')
anxcor_main.set_task_kwargs('resample', resample_kwargs)
anxcor_main.set_task_kwargs('crosscorrelate', correlate_kwargs)
starttime_list = anxcor_main.get_starttimes(starttime,
starttime + window_length,
overlap)
result = anxcor_main.process(starttime_list)
assert len(result['src:test rec:test'].coords['rec_chan'].values) == 3
assert len(result['src:test rec:test'].coords['src_chan'].values) == 3
assert not np.isnan(result['src:test rec:test']).any()
