def test_reduce_stack():
seis1 = get_live_seismogram()
seis2 = get_live_seismogram()
seis_cp = np.array(seis1.data)
stack(seis1, seis2)
res = np.add(np.array(seis_cp), np.array(seis2.data))
for i in range(3):
assert np.isclose(seis1.data[i], res[i]).all() # fixme
ts1 = get_live_timeseries()
ts2 = get_live_timeseries()
ts1_cp = np.array(ts1.data)
stack(ts1, ts2)
assert np.isclose(ts1.data, (np.array(ts1_cp) + np.array(ts2.data))).all()
tse1 = get_live_timeseries_ensemble(2)
tse2 = get_live_timeseries_ensemble(2)
tse1_cp = TimeSeriesEnsemble(tse1)
stack(tse1, tse2)
for i in range(2):
assert np.isclose(
tse1.member[i].data,
np.add(np.array(tse1_cp.member[i].data),
np.array(tse2.member[i].data)),
).all()
seis_e1 = get_live_seismogram_ensemble(2)
seis_e2 = get_live_seismogram_ensemble(2)
seis_e1_cp = SeismogramEnsemble(seis_e1)
stack(seis_e1, seis_e2)
for i in range(2):
res = np.add(np.array(seis_e1_cp.member[i].data),
np.array(seis_e2.member[i].data))
for j in range(3):
assert np.isclose(seis_e1.member[i].data[j], res[j]).all() # fixme
def bury_the_dead(self, d, save_history=True):
"""
Clear the contents of an ensemble and optionally save the history and
error log of the dead. Return the cleaned ensmble.
"""
if not (isinstance(d, TimeSeriesEnsemble) or isinstance(d, SeismogramEnsemble)):
raise MsPASSError(
"Undertaker.bury_the_dead",
"Illegal input type - only works with ensemble objects",
ErrorSeverity.Invalid,
)
# This is a pybind11 wrapper not defined in C++ but useful here
ensmd = d._get_ensemble_md()
nlive = 0
for x in d.member:
if x.live:
nlive += 1
if isinstance(d, TimeSeriesEnsemble):
newens = TimeSeriesEnsemble(ensmd, nlive)
elif isinstance(d, SeismogramEnsemble):
newens = SeismogramEnsemble(ensmd, nlive)
else:
raise MsPASSError(
"Undertaker.bury_the_dead",
"Coding error - newens constructor section has invalid type\nThat cannot happen unless the original code was incorrectly changed",
ErrorSeverity.Invalid,
)
for x in d.member:
if x.live:
newens.member.append(x)
else:
if save_history:
self._save_elog(d.id, d.elog)
return newens
def test_timeseries_ensemble_as_stream():
tse = get_live_timeseries_ensemble(2)
assert len(tse.member) == 2
cp = TimeSeriesEnsemble(tse)
dummy_func_timeseries_ensemble_as_stream(tse)
assert len(tse.member) == 5
np.isclose(cp.member[0].data, tse.member[0].data).all()
np.isclose(cp.member[0].data, tse.member[1].data).all()
tse = get_live_timeseries_ensemble(2)
assert len(tse.member) == 2
cp = TimeSeriesEnsemble(tse)
dummy_func_timeseries_ensemble_as_stream_2(data=tse)
assert len(tse.member) == 5
np.isclose(cp.member[0].data, tse.member[0].data).all()
np.isclose(cp.member[0].data, tse.member[1].data).all()
def Stream2TimeSeriesEnsemble(stream):
"""
Convert a stream to timeseries ensemble.
:param stream: stream input
:return: converted timeseries ensemble
"""
size = len(stream)
tse = TimeSeriesEnsemble()
for i in range(size):
tse.member.append(Trace2TimeSeries(stream[i]))
# potential dead loss problem is resolved by saving the info in converted objects
# Handle the ensemble metadata. The little helper we call here
# get the list set with CONVERTER_ENSEMBLE_KEYS.
enskeys = _converter_get_ensemble_keys(tse)
if len(enskeys) > 0:
post_ensemble_metadata(tse, enskeys)
# By default the above leaves copies of the ensemble md in each member
# Treat that a ok, but we do need to clear the temporary we posted
for d in tse.member:
# Depend on the temp being set in all members - watch out in
# maintenance if any of the related code changes that may be wrong
d.erase("CONVERTER_ENSEMBLE_KEYS")
return tse
def _wtva_Seismogram(self, d, fill):
# this could be implemented by converting d to an ensemble
ens = TimeSeriesEnsemble()
for k in range(3):
dcomp = ExtractComponent(d, k)
ens.member.append(dcomp)
self._wtva_TimeSeriesEnsemble(ens, fill)
def bring_out_your_dead(self, d, bury=False):
"""
Seperate an ensemble into live and dead members.
:param d: must be either a TimeSeriesEnsemble or SeismogramEnsemble of
data to be processed.
:param bury: if true the bury_the_dead method will be called on the
ensemble of dead data before returning
:return: python list with two elements. 0 is ensemble with live data
and 1 is ensemble with dead data.
:rtype: python list with two components
"""
if not (isinstance(d, TimeSeriesEnsemble) or isinstance(d, SeismogramEnsemble)):
raise MsPASSError(
"Undertaker.bring_out_your_dead",
"Illegal input type - only works with ensemble objects",
ErrorSeverity.Invalid,
)
# This is a pybind11 wrapper not defined in C++ but useful here
ensmd = d._get_ensemble_md()
nlive = 0
for x in d.member:
if x.live:
nlive += 1
ndead = len(d.member) - nlive
if isinstance(d, TimeSeriesEnsemble):
newens = TimeSeriesEnsemble(ensmd, nlive)
bodies = TimeSeriesEnsemble(ensmd, ndead)
elif isinstance(d, SeismogramEnsemble):
newens = SeismogramEnsemble(ensmd, nlive)
bodies = SeismogramEnsemble(ensmd, ndead)
else:
raise MsPASSError(
"Undertaker.bring_out_your_dead",
"Coding error - newens constructor section has invalid type\nThat cannot happen unless the original code was incorrectly changed",
ErrorSeverity.Invalid,
)
for x in d.member:
if x.live:
newens.member.append(x)
else:
bodies.member.append(x)
if bury:
self._save_elog(d.id, d.elog)
return [newens, bodies]
def test_all_decorators():
# test mspass_func_wrapper
with pytest.raises(TypeError) as err:
dummy_func_2(1)
assert (str(err.value) ==
"mspass_func_wrapper only accepts mspass object as data input")
with pytest.raises(ValueError) as err:
seis = get_live_seismogram()
dummy_func_2(seis, object_history=True)
assert (str(err.value) ==
"dummy_func_2: object_history was true but alg_id not defined")
assert "OK" == dummy_func_2(seis, dryrun=True)
assert seis.number_of_stages() == 0
dummy_func_2(seis, object_history=True, alg_id="0")
assert seis.number_of_stages() == 1
# test timeseries_as_trace
ts = get_live_timeseries()
cp = np.array(ts.data)
dummy_func_2(ts, object_history=True, alg_id="0")
assert len(cp) != len(ts.data)
np.isclose([0, 1, 2], ts.data).all()
assert ts.number_of_stages() == 1
# test seismogram_as_stream
seis1 = get_live_seismogram()
cp1 = np.array(seis1.data[0])
dummy_func_2(seis1, object_history=True, alg_id="0")
assert cp1[0] != seis1.data[0, 0]
assert seis1.data[0, 0] == -1
assert seis1.number_of_stages() == 1
# test timeseries_ensemble_as_stream
tse = get_live_timeseries_ensemble(2)
cp = TimeSeriesEnsemble(tse)
dummy_func_2(tse, object_history=True, alg_id="0")
assert tse.member[0].data[0] == -1
assert tse.member[0].data[0] != cp.member[0].data[0]
assert tse.member[0].number_of_stages() == 1
# test seismogram_ensemble_as_stream
seis_e = get_live_seismogram_ensemble(2)
cp = SeismogramEnsemble(seis_e)
dummy_func_2(seis_e, object_history=True, alg_id="0")
assert seis_e.member[0].data[0, 0] == -1
assert seis_e.member[0].data[0, 0] != cp.member[0].data[0, 0]
assert seis_e.member[0].number_of_stages() == 1
# test inplace return
seis1 = get_live_seismogram()
# upgrade of decorator -> should explicitly pass the positional arguments
ret = dummy_func_2(seis1, object_history=True, alg_id="0")
assert seis1 == ret
def maketsens(d, n=20, moveout=True, moveout_dt=0.05):
"""
Makes a TimeSeries ensemble as copies of d. If moveout is true
applies a linear moveout to members using moveout_dt times
count of member in ensemble.
"""
# If python had templates this would be one because this and the
# function below are identical except for types
result = TimeSeriesEnsemble()
for i in range(n):
y = TimeSeries(d) # this makes a required deep copy
if (moveout):
y.t0 += float(i) * moveout_dt
result.member.append(y)
return result
def _deepcopy(self, d):
"""
Private helper method for immediately above. Necessary because
copy.deepcopy doesn't work with our pybind11 wrappers. There may be a
fix, but for now we have to use copy constructors specific to each
object type.
"""
if (isinstance(d, TimeSeries)):
return TimeSeries(d)
elif (isinstance(d, Seismogram)):
return Seismogram(d)
elif (isinstance(d, TimeSeriesEnsemble)):
return TimeSeriesEnsemble(d)
elif (isinstance(d, SeismogramEnsemble)):
return SeismogramEnsemble(d)
else:
raise RuntimeError(
"SeismicPlotter._deepcopy: received and unsupported data type=",
type(d))
def test_Ensemble(Ensemble):
md = Metadata()
md["double"] = 3.14
md["bool"] = True
md["long"] = 7
es = Ensemble(md, 3)
if isinstance(es, TimeSeriesEnsemble):
d = TimeSeries(10)
d = make_constant_data_ts(d)
es.member.append(d)
es.member.append(d)
es.member.append(d)
else:
d = Seismogram(10)
d = make_constant_data_seis(d)
es.member.append(d)
es.member.append(d)
es.member.append(d)
es.set_live(
) # new method for LoggingEnsemble needed because default is dead
es.sync_metadata(["double", "long"])
assert es.member[0].is_defined("bool")
assert es.member[0]["bool"] == True
assert not es.member[0].is_defined("double")
assert not es.member[0].is_defined("long")
es.sync_metadata()
assert es.member[1].is_defined("double")
assert es.member[1].is_defined("long")
assert es.member[1]["double"] == 3.14
assert es.member[1]["long"] == 7
es.update_metadata(Metadata({"k": "v"}))
assert es["k"] == "v"
# From here on we test features not in CoreEnsemble but only in
# LoggingEnsemble. Note that we use pybind11 aliasing to
# define TimeSeriesEnsemble == LoggingEnsemble<TimeSeries> and
# SeismogramEnsemble == LoggingEnsemble<Seismogram>.
# Should be initially marked live
assert es.live()
es.elog.log_error("test_ensemble", "test complaint",
ErrorSeverity.Complaint)
es.elog.log_error("test_ensemble", "test invalid", ErrorSeverity.Invalid)
assert es.elog.size() == 2
assert es.live()
es.kill()
assert es.dead()
# resurrect es
es.set_live()
assert es.live()
# validate checks for for any live members - this tests that feature
assert es.validate()
# need this temporary copy for the next test_
if isinstance(es, TimeSeriesEnsemble):
escopy = TimeSeriesEnsemble(es)
else:
escopy = SeismogramEnsemble(es)
for d in escopy.member:
d.kill()
assert not escopy.validate()
# Reuse escopy for pickle test
escopy = pickle.loads(pickle.dumps(es))
assert escopy.is_defined("bool")
assert escopy["bool"] == True
assert escopy.is_defined("double")
assert escopy.is_defined("long")
assert escopy["double"] == 3.14
assert escopy["long"] == 7
assert escopy.live()
assert escopy.elog.size() == 2
assert escopy.member[0].is_defined("bool")
assert escopy.member[0]["bool"] == True
assert escopy.member[0].is_defined("double")
assert escopy.member[0].is_defined("long")
assert es.member[1].is_defined("double")
assert es.member[1].is_defined("long")
assert es.member[1]["double"] == 3.14
assert es.member[1]["long"] == 7
if isinstance(es, TimeSeriesEnsemble):
assert es.member[1].data == escopy.member[1].data
else:
assert (es.member[1].data[:] == escopy.member[1].data[:]).all()
def load_one_ensemble(
doc,
create_history=False,
jobname="Default job",
jobid="99999",
algid="99999",
ensemble_mdkeys=[], # default is to load nothing for ensemble
apply_calib=False,
verbose=False,
):
"""
This function can be used to load a full ensemble indexed in the
collection import_miniseed_ensemble. It uses a large memory model
that eat up the entire file using obspy's miniseed reader. It contains
some relics of early ideas of potentially having the function
utilize the history mechanism. Those may not work, but were retained.
:param doc: is one record in the import_miniseed_ensemble collection
:param create_history: if true each member of the ensemble will be
defined in the history chain as an origin and jobname and jobid will be
be used to construct the ProcessingHistory object.
:param jobname: as used in ProcessingHistory (default "Default job")
:param jobid: as used in processingHistory
:param algid: as used in processingHistory
:param ensemble_mdkeys: list of keys to copy from first member to ensemble
Metadata (no type checking is done)
:param apply_calib: if True tells obspy's reader to apply the calibration
factor to convert the data to ground motion units. Default is false.
:param verbose: write informational messages while processing
"""
try:
ensemblemd = Metadata()
if create_history:
his = ProcessingHistory(jobname, jobid)
form = doc["format"]
mover = doc["mover"]
if form != "mseed":
raise MsPASSError(
"Cannot handle this ensemble - ensemble format="
+ form
+ "\nCan only be mseed for this reader"
)
if mover != "obspy_seed_ensemble_reader":
raise MsPASSError(
"Cannot handle this ensemble - ensemble mover parameter="
+ mover
+ " which is not supported"
)
dir = doc["dir"]
dfile = doc["dfile"]
fname = dir + "/" + dfile
# Note this algorithm actually should work with any format
# supported by obspy's read function - should generalize it for release
dseis = read(fname, format="mseed", apply_calib=apply_calib)
if len(ensemble_mdkeys) > 0:
ensemblemd = _load_md(doc, ensemble_mdkeys)
else:
# default is to load everything != members
members_key = "members"
for k in doc:
if k != members_key:
x = doc[k]
ensemblemd[k] = x
# There is a Stream2TimeSeriesEnsemble function
# but we don't use it here because we need some functionality
# not found in that simple function
nseis = len(dseis)
result = TimeSeriesEnsemble(ensemblemd, nseis)
# Secondary files get handled almost the same except for
# a warning. The warning message (hopefully) explains the
# problem but our documentation must warn about his if this
# prototype algorithm becomes the release version
count = 0
for d in dseis:
# print('debug - working on data object number',count)
count += 1
dts = Trace2TimeSeries(d)
if create_history:
# This should just define jobname and jobid
dts.load_history(his)
seedid = d["seed_file_id"]
dts.set_as_origin(
"load_ensemble", algid, seedid, AtomicType.TIMESERIES, True
)
result.member.append(dts)
return result
except:
print("something threw an exception - needs more complete error handlers")
def get_live_timeseries_ensemble(n):
tse = TimeSeriesEnsemble()
for i in range(n):
ts = get_live_timeseries()
tse.member.append(ts)
return tse
def test_scale():
dts=_CoreTimeSeries(9)
dir=setbasics(dts,9)
d3c=_CoreSeismogram(5)
setbasics(d3c,5)
dts.data[0]=3.0
dts.data[1]=2.0
dts.data[2]=-4.0
dts.data[3]=1.0
dts.data[4]=-100.0
dts.data[5]=-1.0
dts.data[6]=5.0
dts.data[7]=1.0
dts.data[8]=-6.0
# MAD o=f above should be 2
# perf of 0.8 should be 4
# rms should be just over 10=10.010993957
print('Starting tests for time series data of amplitude functions')
ampmad=MADAmplitude(dts)
print('MAD amplitude estimate=',ampmad)
assert(ampmad==3.0)
amprms=RMSAmplitude(dts)
print('RMS amplitude estimate=',amprms)
assert(round(amprms,2)==100.46)
amppeak=PeakAmplitude(dts)
ampperf80=PerfAmplitude(dts,0.8)
print('Peak amplitude=',amppeak)
print('80% clip level amplitude=',ampperf80)
assert(amppeak==100.0)
assert(ampperf80==6.0)
print('Starting comparable tests for 3c data')
d3c.data[0,0]=3.0
d3c.data[0,1]=2.0
d3c.data[1,2]=-4.0
d3c.data[2,3]=1.0
d3c.data[0,4]=np.sqrt(2)*(100.0)
d3c.data[1,4]=-np.sqrt(2)*(100.0)
ampmad=MADAmplitude(d3c)
print('MAD amplitude estimate=',ampmad)
amprms=RMSAmplitude(d3c)
print('RMS amplitude estimate=',amprms)
amppeak=PeakAmplitude(d3c)
ampperf60=PerfAmplitude(d3c,0.6)
print('Peak amplitude=',amppeak)
print('60% clip level amplitude=',ampperf60)
assert(amppeak==200.0)
assert(ampperf60==4.0)
assert(ampmad==3.0)
amptest=round(amprms,2)
assert(amptest==89.48)
print('Trying scaling functions for TimeSeries')
# we need a deep copy here since scaling changes the data
d2=TimeSeries(dts)
amp=_scale(d2,ScalingMethod.Peak,1.0)
print('Computed peak amplitude=',amp)
print(d2.data)
d2=TimeSeries(dts)
amp=_scale(d2,ScalingMethod.Peak,10.0)
print('Computed peak amplitude with peak set to 10=',amp)
print(d2.data)
assert(amp==100.0)
assert(d2.data[4]==-10.0)
print('verifying scale has modified and set calib correctly')
calib=d2.get_double('calib')
assert(calib==10.0)
d2=TimeSeries(dts)
d2.put('calib',6.0)
print('test 2 with MAD metric and initial calib of 6')
amp=_scale(d2,ScalingMethod.MAD,1.0)
calib=d2.get_double('calib')
print('New calib value set=',calib)
assert(calib==18.0)
print('Testing 3C scale functions')
d=Seismogram(d3c)
amp=_scale(d,ScalingMethod.Peak,1.0)
print('Peak amplitude returned by scale funtion=',amp)
calib=d.get_double('calib')
print('Calib value retrieved (assumed inital 1.0)=',calib)
print('Testing python scale function wrapper - first on a TimeSeries with defaults')
d2=TimeSeries(dts)
amp=scale(d2)
print('peak amplitude returned =',amp[0])
assert(amp[0]==100.0)
d=Seismogram(d3c)
amp=scale(d)
print('peak amplitude returned test Seismogram=',amp[0])
assert(amp[0]==200.0)
print('starting tests of scale on ensembles')
print('first test TimeSeriesEnemble with 5 scaled copies of same vector used earlier in this test')
ens=TimeSeriesEnsemble()
scls=[2.0,4.0,1.0,10.0,5.0] # note 4 s the median of this vector
npts=dts.npts
for i in range(5):
d=TimeSeries(dts)
for k in range(npts):
d.data[k]*=scls[i]
d.put('calib',1.0)
ens.member.append(d)
# work on a copy because scaling alters data in place
enscpy=TimeSeriesEnsemble(ens)
amps=scale(enscpy)
print('returned amplitudes for members scaled individually')
for i in range(5):
print(amps[i])
assert(amps[i]==100.0*scls[i])
enscpy=TimeSeriesEnsemble(ens)
amp=scale(enscpy,scale_by_section=True)
print('average amplitude=',amp[0])
#assert(amp[0]==4.0)
avgamp=amp[0]
for i in range(5):
calib=enscpy.member[i].get_double("calib")
print('member number ',i,' calib is ',calib)
assert(round(calib)==400.0)
#print(enscpy.member[i].data)
# similar test for SeismogramEnsemble
npts=d3c.npts
ens=SeismogramEnsemble()
for i in range(5):
d=Seismogram(d3c)
for k in range(3):
for j in range(npts):
d.data[k,j]*=scls[i]
d.put('calib',1.0)
ens.member.append(d)
print('Running comparable tests on SeismogramEnsemble')
enscpy=SeismogramEnsemble(ens)
amps=scale(enscpy)
print('returned amplitudes for members scaled individually')
for i in range(5):
print(amps[i])
assert(round(amps[i])==round(200.0*scls[i]))
print('Trying section scaling of same data')
enscpy=SeismogramEnsemble(ens)
amp=scale(enscpy,scale_by_section=True)
print('average amplitude=',amp[0])
assert(round(amp[0])==800.0)
avgamp=amp[0]
for i in range(5):
calib=enscpy.member[i].get_double("calib")
print('member number ',i,' calib is ',calib)
assert(round(calib)==800.0)