def test_RFdecon(): seis1 = get_live_seismogram(71, 2.0) seis1.t0 = -5 seis2 = get_live_seismogram(71, 2.0) seis2.t0 = -5 for i in range(3): for j in range(seis2.npts): seis2.data[i, j] = seis1.data[i, j] processor = RFdeconProcessor() processor.loaddata(seis1) processor.loadnoise(seis1, window=True) processor.loadwavelet(seis1, window=True) result1 = WindowData(seis1, processor.dwin.start, processor.dwin.end) for k in range(3): processor.loaddata(result1, component=k) x = processor.apply() for i in range(seis1.npts): result1.data[k, i] = x[i] result2 = RFdecon(seis2) for k in range(3): assert all( abs(a - b) < 1e-6 for a, b in zip(result1.data[k], result2.data[k]))
def test_mspass_func_wrapper_multi(): with pytest.raises(TypeError) as err: dummy_func_multi(1, 2) assert (str( err.value ) == "mspass_func_wrapper_multi only accepts mspass object as data input") with pytest.raises(ValueError) as err: seis1 = get_live_seismogram() seis2 = get_live_seismogram() dummy_func_multi(seis1, seis2, object_history=True) assert (str(err.value) == "dummy_func_multi: object_history was true but alg_id not defined") assert "OK" == dummy_func_multi(seis1, seis2, dryrun=True) assert seis1.number_of_stages() == 0 assert seis2.number_of_stages() == 0 dummy_func_multi(seis1, seis2, object_history=True, alg_id="0") assert seis1.number_of_stages() == 1 assert seis2.number_of_stages() == 1 seis_e = get_live_seismogram_ensemble(3) for i in range(3): assert seis_e.member[i].number_of_stages() == 0 dummy_func_multi(seis1, seis_e, object_history=True, alg_id="0") assert seis1.number_of_stages() == 2 for i in range(3): assert seis_e.member[i].number_of_stages() == 1 # dead object will return immediately seis1.kill() seis2.kill() data = dummy_func_multi(seis1, seis2) assert data is None
def test_reduce_stack(): seis1 = get_live_seismogram() seis2 = get_live_seismogram() seis_cp = np.array(seis1.data) stack(seis1, seis2) res = np.add(np.array(seis_cp), np.array(seis2.data)) for i in range(3): assert np.isclose(seis1.data[i], res[i]).all() # fixme ts1 = get_live_timeseries() ts2 = get_live_timeseries() ts1_cp = np.array(ts1.data) stack(ts1, ts2) assert np.isclose(ts1.data, (np.array(ts1_cp) + np.array(ts2.data))).all() tse1 = get_live_timeseries_ensemble(2) tse2 = get_live_timeseries_ensemble(2) tse1_cp = TimeSeriesEnsemble(tse1) stack(tse1, tse2) for i in range(2): assert np.isclose( tse1.member[i].data, np.add(np.array(tse1_cp.member[i].data), np.array(tse2.member[i].data)), ).all() seis_e1 = get_live_seismogram_ensemble(2) seis_e2 = get_live_seismogram_ensemble(2) seis_e1_cp = SeismogramEnsemble(seis_e1) stack(seis_e1, seis_e2) for i in range(2): res = np.add(np.array(seis_e1_cp.member[i].data), np.array(seis_e2.member[i].data)) for j in range(3): assert np.isclose(seis_e1.member[i].data[j], res[j]).all() # fixme
def test_RFdeconProcessor(): decon_processor = RFdeconProcessor(alg="MultiTaperXcor") seis_data = get_live_seismogram() seis_wavelet = get_live_seismogram() seis_noise = get_live_seismogram() decon_processor.loaddata(seis_data) decon_processor.loadwavelet(seis_wavelet) decon_processor.loadnoise(seis_noise) # decon_processor_copy = pickle.loads(pickle.dumps(decon_processor)) data = pickle.dumps(decon_processor) decon_processor_copy = pickle.loads(data) assert (decon_processor.dvector == decon_processor_copy.dvector).all() assert (decon_processor.wvector == decon_processor_copy.wvector).all() assert (decon_processor.nvector == decon_processor_copy.nvector).all() seis_data.npts = 8 seis_data.data[2] = [1, -1, 0, 0, 0, 0, 0, 0] seis_data.data[1] = [0, 1, -1, 0, 0, 0, 0, 0] seis_data.data[0] = [0, 0, -1, 1, 0, 0, 0, 0] decon_processor = RFdeconProcessor() decon_processor.loaddata(seis_data) decon_processor.loadwavelet(seis_data) decon_processor_copy = pickle.loads(pickle.dumps(decon_processor)) result1 = np.array(decon_processor.apply()) result2 = np.array(decon_processor_copy.apply()) assert all(abs(a - b) < 1e-6 for a, b in zip(result1, result2))
def test_xcorr_3c(): seis1 = get_live_seismogram() seis2 = get_live_seismogram() st1 = seis1.toStream() st2 = seis2.toStream() res1 = xcorr_3c(seis1, seis2, 1) res2 = obspy.signal.cross_correlation.xcorr_3c(st1, st2, 1) assert res1 == res2
def test_all_decorators(): # test mspass_func_wrapper with pytest.raises(TypeError) as err: dummy_func_2(1) assert (str(err.value) == "mspass_func_wrapper only accepts mspass object as data input") with pytest.raises(ValueError) as err: seis = get_live_seismogram() dummy_func_2(seis, object_history=True) assert (str(err.value) == "dummy_func_2: object_history was true but alg_id not defined") assert "OK" == dummy_func_2(seis, dryrun=True) assert seis.number_of_stages() == 0 dummy_func_2(seis, object_history=True, alg_id="0") assert seis.number_of_stages() == 1 # test timeseries_as_trace ts = get_live_timeseries() cp = np.array(ts.data) dummy_func_2(ts, object_history=True, alg_id="0") assert len(cp) != len(ts.data) np.isclose([0, 1, 2], ts.data).all() assert ts.number_of_stages() == 1 # test seismogram_as_stream seis1 = get_live_seismogram() cp1 = np.array(seis1.data[0]) dummy_func_2(seis1, object_history=True, alg_id="0") assert cp1[0] != seis1.data[0, 0] assert seis1.data[0, 0] == -1 assert seis1.number_of_stages() == 1 # test timeseries_ensemble_as_stream tse = get_live_timeseries_ensemble(2) cp = TimeSeriesEnsemble(tse) dummy_func_2(tse, object_history=True, alg_id="0") assert tse.member[0].data[0] == -1 assert tse.member[0].data[0] != cp.member[0].data[0] assert tse.member[0].number_of_stages() == 1 # test seismogram_ensemble_as_stream seis_e = get_live_seismogram_ensemble(2) cp = SeismogramEnsemble(seis_e) dummy_func_2(seis_e, object_history=True, alg_id="0") assert seis_e.member[0].data[0, 0] == -1 assert seis_e.member[0].data[0, 0] != cp.member[0].data[0, 0] assert seis_e.member[0].number_of_stages() == 1 # test inplace return seis1 = get_live_seismogram() # upgrade of decorator -> should explicitly pass the positional arguments ret = dummy_func_2(seis1, object_history=True, alg_id="0") assert seis1 == ret
def test_seismogram_as_trace(): seis1 = get_live_seismogram() seis2 = get_live_seismogram() cp1 = np.array(seis1.data[0]) cp2 = np.array(seis2.data[0]) dummy_func_seismogram_as_stream(seis1, seis2) assert cp1[0] != seis1.data[0, 0] assert cp2[0] != seis2.data[0, 0] assert seis1.data[0, 0] == -1 assert seis2.data[0, 0] == -1 assert seis1["test"] == "test"
def test_correlate_stream_template(): tse1 = get_live_seismogram() tse2 = get_live_seismogram() st1 = tse1.toStream() st2 = tse2.toStream() res1 = correlate_stream_template(tse1, tse2, object_history=True, alg_id="0") res2 = obspy.signal.cross_correlation.correlate_stream_template(st1, st2) for i in range(3): assert all( abs(a - b) < 0.001 for a, b in zip(res1.data[i, :], res2[i].data))
def test_copy_helpers(): ts1 = get_live_timeseries() assert ts1.dt != 1 / 255 ts2 = get_live_timeseries() ts2.dt = 1 / 255 timeseries_copy_helper(ts1, ts2) assert ts1.dt == 1 / 255 seis1 = get_live_seismogram() assert seis1.dt != 1 / 255 seis2 = get_live_seismogram() seis2.dt = 1 / 255 seismogram_copy_helper(seis1, seis2) assert seis1.dt == 1 / 255
def test_mspass_reduce_func_wrapper(): ts1 = get_live_timeseries() ts1.data[0] = 1 ts2 = get_live_timeseries() logging_helper.info(ts2, "dummy_func", "1") logging_helper.info(ts2, "dummy_func_2", "2") assert len(ts1.get_nodes()) == 0 dummy_reduce_func(ts1, ts2, object_history=True, alg_id="3") assert ts1.data[0] == -1 assert len(ts1.get_nodes()) == 3 with pytest.raises(TypeError) as err: dummy_reduce_func([0], [1], object_history=True, alg_id="3") assert (str(err.value) == "only mspass objects are supported in reduce wrapped methods") with pytest.raises(TypeError) as err: dummy_reduce_func(ts1, get_live_seismogram(), object_history=True, alg_id="3") assert str(err.value) == "data2 has a different type as data1" with pytest.raises(ValueError) as err: seis1 = get_live_seismogram() seis2 = get_live_seismogram() dummy_reduce_func(seis1, seis2, object_history=True) assert (str( err.value ) == "dummy_reduce_func: object_history was true but alg_id not defined") assert "OK" == dummy_reduce_func(seis1, seis2, dryrun=True) ts1 = get_live_timeseries() ts2 = get_live_timeseries() assert len(ts1.elog.get_error_log()) == 0 dummy_reduce_func_runtime(ts1, ts2, object_history=True, alg_id="3") assert len(ts1.elog.get_error_log()) == 1 assert len(ts2.elog.get_error_log()) == 1 ts1 = get_live_timeseries() ts2 = get_live_timeseries() assert len(ts1.elog.get_error_log()) == 0 with pytest.raises(MsPASSError) as err: dummy_reduce_func_mspasserror(ts1, ts2, object_history=True, alg_id="3") assert str(err.value) == "test"
def test_info_new_map(): # Seismogram and TimeSeries seis = get_live_seismogram() assert seis.number_of_stages() == 0 logging_helper.info(seis, 'dummy_func', '1') assert seis.number_of_stages() == 1 ts = get_live_timeseries() assert ts.number_of_stages() == 0 logging_helper.info(ts, 'dummy_func', '1') assert ts.number_of_stages() == 1 # ensemble seis_e = get_live_seismogram_ensemble(3) logging_helper.info(seis_e, 'dummy_func', '0') for i in range(3): assert seis_e.member[i].number_of_stages() == 1 seis_e = get_live_seismogram_ensemble(3) logging_helper.info(seis_e, 'dummy_func', '0', 0) assert seis_e.member[0].number_of_stages() == 1 tse = get_live_timeseries_ensemble(3) logging_helper.info(tse, 'dummy_func', '0', 0) assert tse.member[0].number_of_stages() == 1
def test_detrend(): ts = get_live_timeseries() seis = get_live_seismogram() tse = get_live_timeseries_ensemble(3) seis_e = get_live_seismogram_ensemble(3) detrend(ts, object_history=True, alg_id="0") detrend(seis, object_history=True, alg_id="0") detrend(tse, object_history=True, alg_id="0") detrend(seis_e, object_history=True, alg_id="0") detrend(ts, type="linear", object_history=True, alg_id="0") detrend(ts, type="constant", object_history=True, alg_id="0") detrend(ts, type="polynomial", order=2, object_history=True, alg_id="0") detrend(ts, type="spline", order=2, dspline=1000, object_history=True, alg_id="0") # functionality verification testing ts = get_live_timeseries() tr = obspy.Trace() tr.data = np.array(ts.data) copy = np.array(ts.data) tr.stats.sampling_rate = 20 tr.detrend(type="simple") detrend(ts, "simple", object_history=True, alg_id="0") assert all(abs(a - b) < 0.001 for a, b in zip(ts.data, tr.data)) assert not all(abs(a - b) < 0.001 for a, b in zip(ts.data, copy))
def test_mspass_func_wrapper(): with pytest.raises(TypeError) as err: dummy_func(1) assert (str(err.value) == "mspass_func_wrapper only accepts mspass object as data input") with pytest.raises(ValueError) as err: seis = get_live_seismogram() dummy_func(seis, object_history=True) assert (str(err.value) == "dummy_func: object_history was true but alg_id not defined") assert "OK" == dummy_func(seis, dryrun=True) # default behavior assert "dummy" == dummy_func(seis) assert seis.number_of_stages() == 0 # object_history is true dummy_func(seis, object_history=True, alg_id="0") assert seis.number_of_stages() == 1 assert len(seis.get_nodes()) == 1 assert seis.current_nodedata().algorithm == "dummy_func" assert seis.current_nodedata().algid == "0" # inplace return data = dummy_func(seis, inplace_return=True) assert isinstance(data, Seismogram) # valid function_return_key data = dummy_func(seis, inplace_return=True, function_return_key="test_key") assert isinstance(data, Seismogram) assert "test_key" in data and data["test_key"] == "dummy" # invalid function_return_key and not inplace_return data = dummy_func(seis, inplace_return=False, function_return_key=dict()) assert isinstance(data, Seismogram) errs = seis.elog.get_error_log() assert len(errs) == 2 assert errs[-1].algorithm == "dummy_func" assert ( errs[-1].message == "Inconsistent arguments; inplace_return was set False and function_return_key was not None.\nAssuming inplace_return == True is correct" ) assert errs[-2].algorithm == "dummy_func" assert ( errs[-2].message == "Illegal type received for function_return_key argument=<class 'dict'>\nReturn value not saved in Metadata" ) # dead object will return immediately seis.kill() data = dummy_func(seis) assert not data.live data = dummy_func(seis, inplace_return=True) assert not data.live
def test_reduce_dead_silent(): seis = get_live_seismogram() assert seis.number_of_stages() == 0 logging_helper.info(seis, 'dummy_func', '1') logging_helper.info(seis, 'dummy_func_2', '2') assert seis.number_of_stages() == 2 seis.kill() seis2 = get_live_seismogram() assert seis2.number_of_stages() == 0 logging_helper.reduce(seis2, seis, 'reduce', '3') assert len(seis2.get_nodes()) == 3 seis = get_live_seismogram() seis2 = get_live_seismogram() logging_helper.info(seis, 'dummy_func', '1') logging_helper.info(seis, 'dummy_func_2', '2') seis2.kill() logging_helper.reduce(seis2, seis, 'reduce', '3') assert len(seis2.get_nodes()) == 0
def test_reduce_dead_silent(): seis = get_live_seismogram() assert seis.number_of_stages() == 0 logging_helper.info(seis, "1", "dummy_func") logging_helper.info(seis, "2", "dummy_func_2") assert seis.number_of_stages() == 2 seis.kill() seis2 = get_live_seismogram() assert seis2.number_of_stages() == 0 logging_helper.reduce(seis2, seis, "3", "reduce") assert len(seis2.get_nodes()) == 3 seis = get_live_seismogram() seis2 = get_live_seismogram() logging_helper.info(seis, "1", "dummy_func") logging_helper.info(seis, "2", "dummy_func_2") seis2.kill() logging_helper.reduce(seis2, seis, "3", "reduce") assert len(seis2.get_nodes()) == 0
def test_filter(): ts = get_live_timeseries() seis = get_live_seismogram() tse = get_live_timeseries_ensemble(3) seis_e = get_live_seismogram_ensemble(3) filter(ts, "bandpass", freqmin=1, freqmax=5, object_history=True, alg_id="0") filter(seis, "bandpass", freqmin=1, freqmax=5, object_history=True, alg_id="0") filter(tse, "bandpass", freqmin=1, freqmax=5, object_history=True, alg_id="0") filter(seis_e, "bandpass", freqmin=1, freqmax=5, object_history=True, alg_id="0") filter(ts, "bandstop", freqmin=1, freqmax=5) filter(ts, "lowpass", freq=1) filter(ts, "highpass", freq=1) filter(ts, "lowpass_cheby_2", freq=1) # fixme fix testing warning # filter(ts, "lowpass_fir", freq=10) these two types are not supported # filter(ts, "remez_fir", freqmin=10, freqmax=20) # functionality verification testing ts = get_live_timeseries() tr = obspy.Trace() tr.data = np.array(ts.data) copy = np.array(ts.data) tr.stats.sampling_rate = 20 tr.filter("bandpass", freqmin=1, freqmax=5) filter(ts, "bandpass", freqmin=1, freqmax=5, object_history=True, alg_id="0") assert all(abs(a - b) < 0.001 for a, b in zip(ts.data, tr.data)) assert not all(abs(a - b) < 0.001 for a, b in zip(ts.data, copy))
def test_reduce_functionality(): # Seismogram and TimeSeries seis = get_live_seismogram() assert seis.number_of_stages() == 0 logging_helper.info(seis, 'dummy_func', '1') logging_helper.info(seis, 'dummy_func_2', '2') assert seis.number_of_stages() == 2 seis2 = get_live_seismogram() assert seis2.number_of_stages() == 0 logging_helper.reduce(seis2, seis, 'reduce', '3') assert len(seis2.get_nodes()) == 3 ts = get_live_timeseries() ts2 = get_live_timeseries() assert ts.number_of_stages() == 0 logging_helper.info(ts, 'dummy_func', '1') logging_helper.info(ts, 'dummy_func', '2') assert ts.number_of_stages() == 2 logging_helper.reduce(ts2, ts, 'reduce', '3') assert len(ts2.get_nodes()) == 3 # ensemble seis_e = get_live_seismogram_ensemble(3) seis_e2 = get_live_seismogram_ensemble(3) logging_helper.info(seis_e, 'dummy_func', '0') logging_helper.info(seis_e, 'dummy_func', '1') logging_helper.info(seis_e, 'dummy_func', '2') logging_helper.reduce(seis_e2, seis_e, "reduce", "3") for i in range(3): assert len(seis_e2.member[i].get_nodes()) == 4 tse = get_live_timeseries_ensemble(3) tse2 = get_live_timeseries_ensemble(3) logging_helper.info(tse, 'dummy_func', '0') logging_helper.info(tse, 'dummy_func', '1') logging_helper.info(tse, 'dummy_func', '2') logging_helper.reduce(tse2, tse, "reduce", "3") for i in range(3): assert len(tse2.member[i].get_nodes()) == 4
def test_info_not_live(): # Seismogram and TimeSeries seis = get_live_seismogram() seis.kill() assert seis.number_of_stages() == 0 logging_helper.info(seis, 'dummy_func', '1') assert seis.number_of_stages() == 0 # ensemble seis_e = get_live_seismogram_ensemble(3) assert seis_e.member[0].number_of_stages() == 0 seis_e.member[0].kill() logging_helper.info(seis_e, 'dummy_func', '0', 0) assert seis_e.member[0].number_of_stages() == 0
def test_interpolate(): ts = get_live_timeseries() seis = get_live_seismogram() tse = get_live_timeseries_ensemble(3) seis_e = get_live_seismogram_ensemble(3) interpolate(ts, 255, object_history=True, alg_id="0") interpolate(seis, 255, object_history=True, alg_id="0") interpolate(tse, 255, object_history=True, alg_id="0") interpolate(seis_e, 255, object_history=True, alg_id="0") interpolate(ts, 255, method="lanczos", a=20, object_history=True, alg_id="0") ts = get_live_timeseries() interpolate(ts, 25, method="slinear", object_history=True, alg_id="0") ts = get_live_timeseries() interpolate(ts, 255, method="linear", object_history=True, alg_id="0") ts = get_live_timeseries() interpolate(ts, 255, method="nearest", object_history=True, alg_id="0") ts = get_live_timeseries() interpolate(ts, 255, method="zero", object_history=True, alg_id="0") # functionality verification testing ts = get_sin_timeseries() tr = obspy.Trace() tr.data = np.array(ts.data) copy = np.array(ts.data) tr.stats.sampling_rate = 20 tr.interpolate(40, method="linear", npts=500) interpolate(ts, 40, method="linear", npts=500, object_history=True, alg_id="0") assert len(ts.data) == len(tr.data) assert all(abs(a - b) < 0.001 for a, b in zip(ts.data, tr.data)) assert not all(abs(a - b) < 0.001 for a, b in zip(ts.data, copy)) assert ts.dt == 1 / 40
def test_interpolate(): ts = get_live_timeseries() seis = get_live_seismogram() tse = get_live_timeseries_ensemble(3) seis_e = get_live_seismogram_ensemble(3) interpolate(ts, 255, preserve_history=True, instance='0') interpolate(seis, 255, preserve_history=True, instance='0') interpolate(tse, 255, preserve_history=True, instance='0') interpolate(seis_e, 255, preserve_history=True, instance='0') interpolate(ts, 255, method='lanczos', a=20, preserve_history=True, instance='0') ts = get_live_timeseries() interpolate(ts, 25, method='slinear', preserve_history=True, instance='0') ts = get_live_timeseries() interpolate(ts, 255, method='linear', preserve_history=True, instance='0') ts = get_live_timeseries() interpolate(ts, 255, method='nearest', preserve_history=True, instance='0') ts = get_live_timeseries() interpolate(ts, 255, method='zero', preserve_history=True, instance='0') # functionality verification testing ts = get_sin_timeseries() tr = obspy.Trace() tr.data = np.array(ts.data) copy = np.array(ts.data) tr.stats.sampling_rate = 20 tr.interpolate(40, method="linear", npts=500) interpolate(ts, 40, method='linear', npts=500, preserve_history=True, instance='0') assert len(ts.data) == len(tr.data) assert all(abs(a - b) < 0.001 for a, b in zip(ts.data, tr.data)) assert not all(abs(a - b) < 0.001 for a, b in zip(ts.data, copy)) assert ts.dt == 1 / 40
def test_is_input_dead(): seis = get_live_seismogram() assert False == is_input_dead(seis) assert False == is_input_dead(any=seis) seis.kill() assert True == is_input_dead(seis) assert True == is_input_dead(any=seis) ts = get_live_timeseries() assert False == is_input_dead(ts) assert False == is_input_dead(any=ts) ts.kill() assert True == is_input_dead(ts) assert True == is_input_dead(any=ts) seis_e = get_live_seismogram_ensemble(3) assert False == is_input_dead(seis_e) assert False == is_input_dead(any=seis_e) seis_e.member[0].kill() assert False == is_input_dead(seis_e) assert False == is_input_dead(any=seis_e) seis_e.member[1].kill() seis_e.member[2].kill() assert True == is_input_dead(seis_e) assert True == is_input_dead(any=seis_e) tse = get_live_timeseries_ensemble(3) assert False == is_input_dead(tse) assert False == is_input_dead(any=tse) tse.member[0].kill() assert False == is_input_dead(tse) assert False == is_input_dead(any=tse) tse.member[1].kill() tse.member[2].kill() assert True == is_input_dead(tse) assert True == is_input_dead(any=tse)
# zero = get_live_timeseries() # zero.data = DoubleVector(np.zeros(255)) res = data.reduce( lambda a, b: stack(a, b, object_history=True, alg_id="3")) return res def test_reduce_dask_spark(spark_context): l = [get_live_timeseries() for i in range(5)] res = np.zeros(255) for i in range(5): for j in range(255): res[j] = res[j] + l[i].data[j] spark_res = spark_reduce(l, spark_context) dask_res = dask_reduce(l) assert np.isclose(res, dask_res.data).all() assert np.isclose(res, spark_res.data).all() assert len(res) == len(spark_res.data) if __name__ == "__main__": # test_reduce_stack() a1 = get_live_seismogram() a2 = get_live_seismogram() print(a1.data[0, 0]) print(a2.data[0, 0]) print(a1.t0, a1.endtime()) print(a2.t0, a2.endtime()) a1 += a2 print(a1.data[0, 0])
def test_mspass_map_with_filePath(self, spark_context): # test mapass_map for spark (file input) # data input of RFdecon, needed for parallelization d = [get_live_seismogram(71, 2.0) for i in range(5)] for i in range(5): d[i].t0 = -5 # parameters string pfPath = "python/mspasspy/data/pf/RFdeconProcessor.pf" pf = AntelopePf(pfPath) pf_dict = AntelopePf2dict(pf) parameter_dict = collections.OrderedDict() parameter_dict["alg"] = "LeastSquares" parameter_dict["pf"] = pf_dict parameter_dict["object_history"] = "True" gTree = ParameterGTree(parameter_dict) json_params = json.dumps(gTree.asdict()) data = spark_context.parallelize(d) spark_res = data.mspass_map( RFdecon, alg="LeastSquares", pf=pfPath, object_history=True, global_history=self.manager, alg_name=None, parameters=None, ).collect() manager_db = Database(self.client, "test_manager") assert (manager_db["history_global"].count_documents( {"job_name": self.manager.job_name}) == 8) res = manager_db["history_global"].find_one({"alg_name": "RFdecon"}) assert res["job_id"] == self.manager.job_id assert res["job_name"] == self.manager.job_name assert res["alg_name"] == "RFdecon" assert res["parameters"] == json_params spark_alg_id = res["alg_id"] # test mspass_map for dask ddb = daskbag.from_sequence(d) dask_res = ddb.mspass_map( RFdecon, alg="LeastSquares", pf=pfPath, object_history=True, global_history=self.manager, alg_name=None, parameters=None, ).compute() assert (manager_db["history_global"].count_documents( {"job_name": self.manager.job_name}) == 9) assert (manager_db["history_global"].count_documents( {"alg_id": spark_alg_id}) == 2) docs = manager_db["history_global"].find({"alg_id": spark_alg_id}) assert docs[0]["job_id"] == docs[1]["job_id"] == self.manager.job_id assert docs[0]["job_name"] == docs[1][ "job_name"] == self.manager.job_name assert docs[0]["alg_name"] == docs[1]["alg_name"] == "RFdecon" assert docs[0]["parameters"] == docs[1]["parameters"] == json_params assert not docs[0]["time"] == docs[1]["time"] # same alg + parameters combination -> same alg_id ddb = daskbag.from_sequence(d) dask_res = ddb.mspass_map( RFdecon, alg="LeastSquares", pf=pfPath, object_history=True, global_history=self.manager, alg_name=None, parameters=None, ).compute() assert (manager_db["history_global"].count_documents( {"job_name": self.manager.job_name}) == 10) assert (manager_db["history_global"].count_documents( {"alg_id": spark_alg_id}) == 3) # SPARK test user provided alg_name and parameter(exist) spark_alg_name = "RFdecon" spark_alg_parameters = ( "alg=LeastSquares, pf={pfPath}, object_history=True".format( pfPath=pfPath)) data = spark_context.parallelize(d) spark_res = data.mspass_map( RFdecon, alg="LeastSquares", pf=pfPath, object_history=True, global_history=self.manager, alg_name=spark_alg_name, parameters=spark_alg_parameters, ).collect() assert (manager_db["history_global"].count_documents( {"job_name": self.manager.job_name}) == 11) assert (manager_db["history_global"].count_documents( {"alg_id": spark_alg_id}) == 4) # SPARK test user provided alg_name and parameter(new) spark_alg_name = "RFdecon_2" spark_alg_parameters = ( "alg=LeastSquares, pf={pfPath}, object_history=True".format( pfPath=pfPath)) data = spark_context.parallelize(d) spark_res = data.mspass_map( RFdecon, alg="LeastSquares", pf=pfPath, object_history=True, global_history=self.manager, alg_name=spark_alg_name, parameters=spark_alg_parameters, ).collect() assert (manager_db["history_global"].count_documents( {"job_name": self.manager.job_name}) == 12) assert (manager_db["history_global"].count_documents( {"alg_name": "RFdecon_2"}) == 1) res = manager_db["history_global"].find_one({"alg_name": "RFdecon_2"}) assert res["job_id"] == self.manager.job_id assert res["job_name"] == self.manager.job_name assert res["alg_name"] == "RFdecon_2" assert res["parameters"] == json_params new_spark_alg_id = res["alg_id"] assert (manager_db["history_global"].count_documents( {"alg_id": new_spark_alg_id}) == 1) # DASK test user provided alg_name and parameter(exist) dask_alg_name = "RFdecon" dask_alg_parameters = ( "alg=LeastSquares, pf={pfPath}, object_history=True".format( pfPath=pfPath)) ddb = daskbag.from_sequence(d) dask_res = ddb.mspass_map( RFdecon, alg="LeastSquares", pf=pfPath, object_history=True, global_history=self.manager, alg_name=dask_alg_name, parameters=dask_alg_parameters, ).compute() assert (manager_db["history_global"].count_documents( {"job_name": self.manager.job_name}) == 13) assert (manager_db["history_global"].count_documents( {"alg_id": spark_alg_id}) == 5) # DASK test user provided alg_name and parameter(new) dask_alg_name = "RFdecon_3" dask_alg_parameters = ( "alg=LeastSquares, pf={pfPath}, object_history=True".format( pfPath=pfPath)) ddb = daskbag.from_sequence(d) dask_res = ddb.mspass_map( RFdecon, alg="LeastSquares", pf=pfPath, object_history=True, global_history=self.manager, alg_name=dask_alg_name, parameters=dask_alg_parameters, ).compute() assert (manager_db["history_global"].count_documents( {"job_name": self.manager.job_name}) == 14) assert (manager_db["history_global"].count_documents( {"alg_name": "RFdecon_3"}) == 1) res = manager_db["history_global"].find_one({"alg_name": "RFdecon_3"}) assert res["job_id"] == self.manager.job_id assert res["job_name"] == self.manager.job_name assert res["alg_name"] == "RFdecon_3" assert res["parameters"] == json_params new_dask_alg_id = res["alg_id"] assert (manager_db["history_global"].count_documents( {"alg_id": new_dask_alg_id}) == 1)