def test_inv_cache(data):
class Segment(object):
def __init__(self, id_, staid):
if staid is not None:
self.station = Segment(staid, None)
else:
self.station = None
self.id = id_
def_size_limit = _default_size_limits()[1]
inventory = data.read_inv("GE.FLT1.xml")
@contextmanager
def setup(inv_cache_instance):
with patch.object(
InventoryCache,
'_size_limit_popped',
wraps=inv_cache_instance._size_limit_popped) as _mock_popitem:
yield inv_cache_instance, _mock_popitem
# test LimitedSizeDict with no size_limit arg (no size limit):
with setup(InventoryCache()) as (inv, mock_popitem):
for i in range(def_size_limit):
inv[Segment(i, 1)] = inventory
assert not mock_popitem.called
# test supplying always the same inventory
with setup(InventoryCache()) as (inv, mock_popitem):
# same station id (1) for all segments:
# it does not matter: keys are removed if they have the same value
# (via 'is' keyword), thus all keys will be removed
for i in range(def_size_limit + 1):
inv[Segment(i, 1)] = inventory
assert mock_popitem.call_count == 1
assert len(inv) == 0
with setup(InventoryCache()) as (inv, mock_popitem):
# different station ids (0,1,2,..) for all segments:
# it does not matter: keys are removed if they have the same
# value (via 'is' keyword), thus all keys will be removed
mock_popitem.reset_mock()
for i in range(def_size_limit + 1):
inv[Segment(i, i)] = inventory
assert mock_popitem.call_count == 1
assert len(inv) == 0
with setup(InventoryCache()) as (inv, mock_popitem):
# now provide different objects, we remove only one element
mock_popitem.reset_mock()
for i in range(def_size_limit + 1):
inv[Segment(i, i)] = ValueError('a')
assert mock_popitem.call_count == 1
assert len(inv) == def_size_limit
with setup(InventoryCache()) as (inv, mock_popitem):
# now provide same object again, we are again with 0 items
v = ValueError('a')
mock_popitem.reset_mock()
for i in range(def_size_limit + 1):
inv[Segment(i, i)] = v
assert mock_popitem.call_count == 1
assert len(inv) == 0
def test_query4gui(self, db):
s = db.session.query(Station).first()
e = db.session.query(Event).first()
dc = db.session.query(DataCenter).first()
run = db.session.query(Download).first()
channels = [
Channel(location='00', channel='HHE', sample_rate=6),
Channel(location='00', channel='HHN', sample_rate=6),
Channel(location='00', channel='HHZ', sample_rate=6),
Channel(location='00', channel='HHW', sample_rate=6),
Channel(location='10', channel='HHE', sample_rate=6),
Channel(location='10', channel='HHN', sample_rate=6),
Channel(location='10', channel='HHZ', sample_rate=6),
Channel(location='', channel='HHE', sample_rate=6),
Channel(location='', channel='HHN', sample_rate=6),
Channel(location='30', channel='HHZ', sample_rate=6)]
# expected lengths when querying for gui below. CHANGE THIS
# IF YOU CHANGE THE PREVIOUS channels VARIABLE
expected_lengths = [4, 4, 4, 4, 3, 3, 3, 2, 2, 1]
s.channels.extend(channels)
db.session.commit()
args = dict(request_start=datetime.utcnow(),
request_end=datetime.utcnow(),
event_distance_deg=9,
arrival_time=datetime.utcnow(),
data=b'',
event_id=e.id,
datacenter_id=dc.id,
download_id=run.id)
segments = []
# and now it will work:
for c in channels:
segments.append(Segment(channel_id=c.id, **args))
db.session.add_all(segments)
db.session.commit()
for leng, segment in zip(expected_lengths, segments):
# assert the other segments are the expected lengh. Note that leng INCLUDES current
# segment whereas siblings DOES NOT. So compare to leng-1:
assert segment.siblings().count() == leng-1
def test_prepare_for_download(self, db, tt_ak135_tts):
# prepare:
urlread_sideeffect = None # use defaults from class
events_df = self.get_events_df(urlread_sideeffect, db.session)
net, sta, loc, cha = [], [], [], []
datacenters_df, eidavalidator = \
self.get_datacenters_df(urlread_sideeffect, db.session, None, self.routing_service,
net, sta, loc, cha, db_bufsize=self.db_buf_size)
channels_df = self.get_channels_df(urlread_sideeffect, db.session,
datacenters_df, eidavalidator, net,
sta, loc, cha, None, None, 100,
False, None, None, -1,
self.db_buf_size)
assert len(
channels_df
) == 12 # just to be sure. If failing, we might have changed the class default
# events_df
# id magnitude latitude longitude depth_km time
# 0 1 3.0 1.0 1.0 60.0 2016-05-08 05:17:11.500
# 1 2 4.0 90.0 90.0 2.0 2016-05-08 01:45:30.300
# channels_df:
# id station_id latitude longitude datacenter_id start_time end_time network station location channel
# 0 1 1 1.0 1.0 1 2003-01-01 NaT GE FLT1 HHE
# 1 2 1 1.0 1.0 1 2003-01-01 NaT GE FLT1 HHN
# 2 3 1 1.0 1.0 1 2003-01-01 NaT GE FLT1 HHZ
# 3 4 2 90.0 90.0 1 2009-01-01 NaT n1 s c1
# 4 5 2 90.0 90.0 1 2009-01-01 NaT n1 s c2
# 5 6 2 90.0 90.0 1 2009-01-01 NaT n1 s c3
# 6 7 3 1.0 1.0 2 2003-01-01 NaT IA BAKI BHE
# 7 8 3 1.0 1.0 2 2003-01-01 NaT IA BAKI BHN
# 8 9 3 1.0 1.0 2 2003-01-01 NaT IA BAKI BHZ
# 9 10 4 90.0 90.0 2 2009-01-01 NaT n2 s c1
# 10 11 4 90.0 90.0 2 2009-01-01 NaT n2 s c2
# 11 12 4 90.0 90.0 2 2009-01-01 NaT n2 s c3
# take all segments:
segments_df = merge_events_stations(events_df,
channels_df,
dict(minmag=10,
maxmag=10,
minmag_radius=100,
maxmag_radius=200),
tttable=tt_ak135_tts)
# segments_df:
# channel_id station_id datacenter_id network station location channel event_distance_deg event_id depth_km time arrival_time
# 0 1 1 1 GE FLT1 HHE 500.555 1 60.0 2016-05-08 05:17:11.500 2017-05-10 12:39:13.463745
# 1 2 1 1 GE FLT1 HHN 500.555 1 60.0 2016-05-08 05:17:11.500 2017-05-10 12:39:13.463745
# 2 3 1 1 GE FLT1 HHZ 500.555 1 60.0 2016-05-08 05:17:11.500 2017-05-10 12:39:13.463745
# 3 4 2 1 n1 s c1 89.000 1 60.0 2016-05-08 05:17:11.500 NaT
# 4 5 2 1 n1 s c2 89.000 1 60.0 2016-05-08 05:17:11.500 NaT
# 5 6 2 1 n1 s c3 89.0 1 60.0 2016-05-08 05:17:11.500 NaT
# 6 7 3 2 IA BAKI BHE 0.0 1 60.0 2016-05-08 05:17:11.500 NaT
# 7 8 3 2 IA BAKI BHN 0.0 1 60.0 2016-05-08 05:17:11.500 NaT
# 8 9 3 2 IA BAKI BHZ 0.0 1 60.0 2016-05-08 05:17:11.500 NaT
# 9 10 4 2 n2 s c1 89.0 1 60.0 2016-05-08 05:17:11.500 NaT
# 10 11 4 2 n2 s c2 89.0 1 60.0 2016-05-08 05:17:11.500 NaT
# 11 12 4 2 n2 s c3 89.0 1 60.0 2016-05-08 05:17:11.500 NaT
# 12 1 1 1 GE FLT1 HHE 89.0 2 2.0 2016-05-08 01:45:30.300 NaT
# 13 2 1 1 GE FLT1 HHN 89.0 2 2.0 2016-05-08 01:45:30.300 NaT
# 14 3 1 1 GE FLT1 HHZ 89.0 2 2.0 2016-05-08 01:45:30.300 NaT
# 15 4 2 1 n1 s c1 0.0 2 2.0 2016-05-08 01:45:30.300 NaT
# 16 5 2 1 n1 s c2 0.0 2 2.0 2016-05-08 01:45:30.300 NaT
# 17 6 2 1 n1 s c3 0.0 2 2.0 2016-05-08 01:45:30.300 NaT
# 18 7 3 2 IA BAKI BHE 89.0 2 2.0 2016-05-08 01:45:30.300 NaT
# 19 8 3 2 IA BAKI BHN 89.0 2 2.0 2016-05-08 01:45:30.300 NaT
# 20 9 3 2 IA BAKI BHZ 89.0 2 2.0 2016-05-08 01:45:30.300 NaT
# 21 10 4 2 n2 s c1 0.0 2 2.0 2016-05-08 01:45:30.300 NaT
# 22 11 4 2 n2 s c2 0.0 2 2.0 2016-05-08 01:45:30.300 NaT
# 23 12 4 2 n2 s c3 0.0 2 2.0 2016-05-08 01:45:30.300 NaT
# make a copy of evts_stations_df cause we will modify in place the data frame
# segments_df = self.get_arrivaltimes(urlread_sideeffect, evts_stations_df.copy(),
# [1,2], ['P', 'Q'],
# 'ak135', mp_max_workers=1)
expected = len(
segments_df
) # no segment on db, we should have all segments to download
wtimespan = [1, 2]
assert Segment.id.key not in segments_df.columns
assert Segment.download_id.key not in segments_df.columns
orig_seg_df = segments_df.copy()
# define a dc_dataselect_manager for open data only:
dc_dataselect_manager = DcDataselectManager(datacenters_df,
Authorizer(None), False)
segments_df, request_timebounds_need_update = \
prepare_for_download(db.session, orig_seg_df, dc_dataselect_manager, wtimespan,
retry_seg_not_found=True,
retry_url_err=True,
retry_mseed_err=True,
retry_client_err=True,
retry_server_err=True,
retry_timespan_err=True,
retry_timespan_warn=True)
assert request_timebounds_need_update is False
# segments_df: (not really the real dataframe, some columns are removed but relevant data is ok):
# channel_id datacenter_id network station location channel event_distance_deg event_id arrival_time start_time end_time
# 0 1 1 GE FLT1 HHE 0.0 1 2016-05-08 05:17:12.500 2016-05-08 05:16:12 2016-05-08 05:19:12
# 1 2 1 GE FLT1 HHN 0.0 1 2016-05-08 05:17:12.500 2016-05-08 05:16:12 2016-05-08 05:19:12
# 2 3 1 GE FLT1 HHZ 0.0 1 2016-05-08 05:17:12.500 2016-05-08 05:16:12 2016-05-08 05:19:12
# 3 4 1 n1 s c1 89.0 1 2016-05-08 05:17:12.500 2016-05-08 05:16:12 2016-05-08 05:19:12
# 4 5 1 n1 s c2 89.0 1 2016-05-08 05:17:12.500 2016-05-08 05:16:12 2016-05-08 05:19:12
# 5 6 1 n1 s c3 89.0 1 2016-05-08 05:17:12.500 2016-05-08 05:16:12 2016-05-08 05:19:12
# 6 7 2 IA BAKI BHE 0.0 1 2016-05-08 05:17:12.500 2016-05-08 05:16:12 2016-05-08 05:19:12
# 7 8 2 IA BAKI BHN 0.0 1 2016-05-08 05:17:12.500 2016-05-08 05:16:12 2016-05-08 05:19:12
# 8 9 2 IA BAKI BHZ 0.0 1 2016-05-08 05:17:12.500 2016-05-08 05:16:12 2016-05-08 05:19:12
# 9 10 2 n2 s c1 89.0 1 2016-05-08 05:17:12.500 2016-05-08 05:16:12 2016-05-08 05:19:12
# 10 11 2 n2 s c2 89.0 1 2016-05-08 05:17:12.500 2016-05-08 05:16:12 2016-05-08 05:19:12
# 11 12 2 n2 s c3 89.0 1 2016-05-08 05:17:12.500 2016-05-08 05:16:12 2016-05-08 05:19:12
# 12 1 1 GE FLT1 HHE 89.0 2 2016-05-08 01:45:31.300 2016-05-08 01:44:31 2016-05-08 01:47:31
# 13 2 1 GE FLT1 HHN 89.0 2 2016-05-08 01:45:31.300 2016-05-08 01:44:31 2016-05-08 01:47:31
# 14 3 1 GE FLT1 HHZ 89.0 2 2016-05-08 01:45:31.300 2016-05-08 01:44:31 2016-05-08 01:47:31
# 15 4 1 n1 s c1 0.0 2 2016-05-08 01:45:31.300 2016-05-08 01:44:31 2016-05-08 01:47:31
# 16 5 1 n1 s c2 0.0 2 2016-05-08 01:45:31.300 2016-05-08 01:44:31 2016-05-08 01:47:31
# 17 6 1 n1 s c3 0.0 2 2016-05-08 01:45:31.300 2016-05-08 01:44:31 2016-05-08 01:47:31
# 18 7 2 IA BAKI BHE 89.0 2 2016-05-08 01:45:31.300 2016-05-08 01:44:31 2016-05-08 01:47:31
# 19 8 2 IA BAKI BHN 89.0 2 2016-05-08 01:45:31.300 2016-05-08 01:44:31 2016-05-08 01:47:31
# 20 9 2 IA BAKI BHZ 89.0 2 2016-05-08 01:45:31.300 2016-05-08 01:44:31 2016-05-08 01:47:31
# 21 10 2 n2 s c1 0.0 2 2016-05-08 01:45:31.300 2016-05-08 01:44:31 2016-05-08 01:47:31
# 22 11 2 n2 s c2 0.0 2 2016-05-08 01:45:31.300 2016-05-08 01:44:31 2016-05-08 01:47:31
# 23 12 2 n2 s c3 0.0 2 2016-05-08 01:45:31.300 2016-05-08 01:44:31 2016-05-08 01:47:31
assert Segment.id.key in segments_df.columns
assert Segment.download_id.key not in segments_df.columns
assert len(segments_df) == expected
# assert len(db.session.query(Segment.id).all()) == len(segments_df)
assert all(x[0] is None
for x in db.session.query(Segment.download_code).all())
assert all(x[0] is None for x in db.session.query(Segment.data).all())
# mock an already downloaded segment.
# Set the first 7 to have a particular download status code
urlerr, mseederr, outtime_err, outtime_warn = \
s2scodes.url_err, s2scodes.mseed_err, s2scodes.timespan_err, s2scodes.timespan_warn
downloadstatuscodes = [
None, urlerr, mseederr, 413, 505, outtime_err, outtime_warn
]
for i, download_code in enumerate(downloadstatuscodes):
dic = segments_df.iloc[i].to_dict()
dic['download_code'] = download_code
dic['download_id'] = self.run.id
# hack for deleting unused columns:
for col in [
Station.network.key, Station.station.key,
Channel.location.key, Channel.channel.key
]:
if col in dic:
del dic[col]
# convet numpy values to python scalars:
# pandas 20+ seems to keep numpy types in to_dict
# https://github.com/pandas-dev/pandas/issues/13258
# this was not the case in pandas 0.19.2
# sql alchemy does not like that
# (Curiosly, our pd2sql methods still work fine (we should check why)
# So, quick and dirty:
for k in dic.keys():
if hasattr(dic[k], "item"):
dic[k] = dic[k].item()
# postgres complains about nan primary keys
if math.isnan(dic.get(Segment.id.key, 0)):
del dic[Segment.id.key]
# now we can safely add it:
db.session.add(Segment(**dic))
db.session.commit()
assert len(db.session.query(
Segment.id).all()) == len(downloadstatuscodes)
# Now we have an instance of all possible errors on the db (5 in total) and a new
# instance (not on the db). Assure all work:
# set the num of instances to download anyway. Their number is the not saved ones, i.e.:
to_download_anyway = len(segments_df) - len(downloadstatuscodes)
for c in product([True, False], [True, False], [True, False],
[True, False], [True, False], [True, False],
[True, False]):
s_df, request_timebounds_need_update = \
prepare_for_download(db.session, orig_seg_df, dc_dataselect_manager, wtimespan,
retry_seg_not_found=c[0],
retry_url_err=c[1],
retry_mseed_err=c[2],
retry_client_err=c[3],
retry_server_err=c[4],
retry_timespan_err=c[5],
retry_timespan_warn=c[6])
to_download_in_this_case = sum(
c) # count the True's (bool sum works in python)
assert len(s_df) == to_download_anyway + to_download_in_this_case
assert request_timebounds_need_update is False
# now change the window time span and see that everything is to be downloaded again:
# do it for any retry combinations, as it should ALWAYS return "everything has to be
# re-downloaded"
wtimespan[1] += 5
for c in product([True, False], [True, False], [True, False],
[True, False], [True, False], [True, False],
[True, False]):
s_df, request_timebounds_need_update = \
prepare_for_download(db.session, orig_seg_df, dc_dataselect_manager, wtimespan,
retry_seg_not_found=c[0],
retry_url_err=c[1],
retry_mseed_err=c[2],
retry_client_err=c[3],
retry_server_err=c[4],
retry_timespan_err=c[5],
retry_timespan_warn=c[6])
assert len(s_df) == len(orig_seg_df)
assert request_timebounds_need_update is True # because we changed wtimespan
# this hol
# now test that we raise a NothingToDownload
# first, write all remaining segments to db, with 204 code so they will not be
# re-downloaded
for i in range(len(segments_df)):
download_code = 204
dic = segments_df.iloc[i].to_dict()
dic['download_code'] = download_code
dic['download_id'] = self.run.id
# hack for deleting unused columns:
for col in [
Station.network.key, Station.station.key,
Channel.location.key, Channel.channel.key
]:
if col in dic:
del dic[col]
# convet numpy values to python scalars:
# pandas 20+ seems to keep numpy types in to_dict
# https://github.com/pandas-dev/pandas/issues/13258
# this was not the case in pandas 0.19.2
# sql alchemy does not like that
# (Curiosly, our pd2sql methods still work fine (we should check why)
# So, quick and dirty:
for k in dic.keys():
if hasattr(dic[k], "item"):
dic[k] = dic[k].item()
# postgres complains about nan primary keys
if math.isnan(dic.get(Segment.id.key, 0)):
del dic[Segment.id.key]
# now we can safely add it:
# brutal approach: add and commit, if error, rollback
# if error it means we already wrote the segment on db and an uniqueconstraint
# is raised
try:
db.session.add(Segment(**dic))
db.session.commit()
except SQLAlchemyError as _err:
db.session.rollback()
# test that we have the correct number of segments saved:
assert db.session.query(Segment.id).count() == len(orig_seg_df)
# try to test a NothingToDownload:
# reset the old wtimespan otherwise everything will be flagged to be redownloaded:
wtimespan[1] -= 5
with pytest.raises(NothingToDownload):
segments_df, request_timebounds_need_update = \
prepare_for_download(db.session, orig_seg_df, dc_dataselect_manager, wtimespan,
retry_seg_not_found=False,
retry_url_err=False,
retry_mseed_err=False,
retry_client_err=False,
retry_server_err=False,
retry_timespan_err=False,
retry_timespan_warn=False)
def test_segwrapper(
self,
mock_getstream,
mock_getinv,
# fixtures:
db4process,
data):
session = db4process.session
segids = query4process(session, {}).all()
seg_with_inv = \
db4process.segments(with_inventory=True, with_data=True, with_gap=False).one()
sta_with_inv_id = seg_with_inv.station.id
invcache = {}
def read_stream(segment, reload=False):
'''calls segment.stream(reload) asserting that if segment has no
data it raises. This function never raises'''
if segment.data:
segment.stream(reload)
else:
with pytest.raises(
Exception) as exc: # all inventories are None
segment.stream(reload)
prev_staid = None
for segid in [_[0] for _ in segids]:
segment = session.query(Segment).filter(
Segment.id == segid).first()
sta = segment.station
staid = sta.id
assert prev_staid is None or staid >= prev_staid
staequal = prev_staid is not None and staid == prev_staid
prev_staid = staid
segment.station._inventory = invcache.get(sta.id, None)
mock_getinv.reset_mock()
if sta.id != sta_with_inv_id:
with pytest.raises(Exception): # all inventories are None
segment.inventory()
assert mock_getinv.called
# re-call it and assert we raise the previous Exception:
ccc = mock_getinv.call_count
with pytest.raises(Exception): # all inventories are None
segment.inventory()
assert mock_getinv.call_count == ccc
# re-call it with reload=True and assert we raise the previous
# exception, and that we called get_inv:
with pytest.raises(Exception): # all inventories are None
segment.inventory(True)
assert mock_getinv.call_count == ccc + 1
else:
invcache[sta.id] = segment.inventory()
if staequal:
assert not mock_getinv.called
else:
assert mock_getinv.called
assert len(segment.station.inventory_xml) > 0
# re-call it with reload=True and assert we raise the previous
# exception, and that we called get_inv:
ccc = mock_getinv.call_count
segment.inventory(True)
assert mock_getinv.call_count == ccc + 1
# call segment.stream
assert not mock_getstream.called
read_stream(segment)
assert mock_getstream.call_count == 1
read_stream(segment)
assert mock_getstream.call_count == 1
# with reload flag:
read_stream(segment, True)
assert mock_getstream.call_count == 2
mock_getstream.reset_mock()
segs = segment.siblings().all()
# as channel's channel is either 'ok' or 'err' we should never have
# other components
assert len(segs) == 0
# NOW TEST OTHER ORIENTATION PROPERLY. WE NEED TO ADD WELL FORMED SEGMENTS WITH CHANNELS
# WHOSE ORIENTATION CAN BE DERIVED:
staid = session.query(Station.id).first()[0]
dcid = session.query(DataCenter.id).first()[0]
eid = session.query(Event.id).first()[0]
dwid = session.query(Download.id).first()[0]
# add channels
c_1 = Channel(station_id=staid,
location='ok',
channel="AB1",
sample_rate=56.7)
c_2 = Channel(station_id=staid,
location='ok',
channel="AB2",
sample_rate=56.7)
c_3 = Channel(station_id=staid,
location='ok',
channel="AB3",
sample_rate=56.7)
session.add_all([c_1, c_2, c_3])
session.commit()
# add segments. Create attributes (although not strictly necessary to have bytes data)
atts = data.to_segment_dict('trace_GE.APE.mseed')
# build three segments with data:
# "normal" segment
sg1 = Segment(channel_id=c_1.id,
datacenter_id=dcid,
event_id=eid,
download_id=dwid,
event_distance_deg=35,
**atts)
sg2 = Segment(channel_id=c_2.id,
datacenter_id=dcid,
event_id=eid,
download_id=dwid,
event_distance_deg=35,
**atts)
sg3 = Segment(channel_id=c_3.id,
datacenter_id=dcid,
event_id=eid,
download_id=dwid,
event_distance_deg=35,
**atts)
session.add_all([sg1, sg2, sg3])
session.commit()
# start testing:
segids = query4process(session, {}).all()
for segid in [_[0] for _ in segids]:
segment = session.query(Segment).filter(
Segment.id == segid).first()
# staid = segment.station.id
segs = segment.siblings()
if segs.all():
assert segment.id in (sg1.id, sg2.id, sg3.id)
assert len(segs.all()) == 2
def init(self, request, db, data):
# re-init a sqlite database (no-op if the db is not sqlite):
db.create(to_file=True)
# setup a run_id:
r = Download()
db.session.add(r)
db.session.commit()
self.run = r
ws = WebService(id=1, url='eventws')
db.session.add(ws)
db.session.commit()
self.ws = ws
# setup an event:
e1 = Event(id=1, webservice_id=ws.id, event_id='ev1', latitude=8, longitude=9, magnitude=5,
depth_km=4, time=datetime.utcnow())
e2 = Event(id=2, webservice_id=ws.id, event_id='ev2', latitude=8, longitude=9, magnitude=5,
depth_km=4, time=datetime.utcnow())
e3 = Event(id=3, webservice_id=ws.id, event_id='ev3', latitude=8, longitude=9, magnitude=5,
depth_km=4, time=datetime.utcnow())
e4 = Event(id=4, webservice_id=ws.id, event_id='ev4', latitude=8, longitude=9, magnitude=5,
depth_km=4, time=datetime.utcnow())
e5 = Event(id=5, webservice_id=ws.id, event_id='ev5', latitude=8, longitude=9, magnitude=5,
depth_km=4, time=datetime.utcnow())
db.session.add_all([e1, e2, e3, e4, e5])
db.session.commit()
d1 = DataCenter(station_url='asd', dataselect_url='www.dc1/dataselect/query')
d2 = DataCenter(station_url='asd', dataselect_url='www.dc2/dataselect/query')
db.session.add_all([d1, d2])
db.session.commit()
# d1 has one station
s_d1 = Station(datacenter_id=d1.id, latitude=11, longitude=11, network='N1', station='S1',
start_time=datetime.utcnow())
s_d2 = Station(datacenter_id=d1.id, latitude=22.1, longitude=22.1, network='N1',
station='S2a', start_time=datetime.utcnow())
s2_d2 = Station(datacenter_id=d1.id, latitude=22.2, longitude=22.2, network='N2',
station='S2b', start_time=datetime.utcnow())
db.session.add_all([s_d1, s_d2, s2_d2])
db.session.commit()
# we are about to add 3 stations * 4 channels = 12 channels
# we add also 1 segment pre channel
# the segments data is as follows (data, download_code, maxgap)
seg_data = ([None, s2scodes.url_err, None],
[None, s2scodes.mseed_err, None],
[None, None, None],
[None, s2scodes.timespan_err, None],
# station s_d2:
[b'x', 200, 0.2],
[b'x', s2scodes.timespan_warn, 3.9],
[b'x', 200, 0.6],
[b'x', 200, 0.3],
# station s_d3:
[b'x', 200, 0.1],
[b'x', s2scodes.timespan_warn, 3.9],
[b'x', 400, None],
[b'x', 500, None],
)
i = 0
for s in [s_d1, s_d2, s2_d2]:
for cha in ['HHZ', 'HHE', 'HHN', 'ABC']:
c = Channel(station_id=s.id, location='', channel=cha, sample_rate=56.7)
db.session.add(c)
db.session.commit()
data, code, gap = seg_data[i]
i += 1
seg = Segment(channel_id=c.id, datacenter_id=s.datacenter_id,
event_id=e1.id, download_id=r.id,
event_distance_deg=35, request_start=datetime.utcnow(),
arrival_time=datetime.utcnow(),
request_end=datetime.utcnow() + timedelta(seconds=5), data=data,
download_code=code, maxgap_numsamples=gap)
db.session.add(seg)
db.session.commit()
with patch('stream2segment.utils.inputargs.get_session',
return_value=db.session) as mock_session:
yield
def __init__(self, id_, staid):
if staid is not None:
self.station = Segment(staid, None)
else:
self.station = None
self.id = id_
def init(self, request, db, data):
# re-init a sqlite database (no-op if the db is not sqlite):
db.create(to_file=False, process=True)
# init db:
session = db.session
dct = DataCenter(station_url="345fbgfnyhtgrefs",
dataselect_url='edfawrefdc')
session.add(dct)
utcnow = datetime.utcnow()
dwl = Download(run_time=utcnow)
session.add(dwl)
ws = WebService(url='webserviceurl')
session.add(ws)
session.commit()
# id = 'firstevent'
ev1 = Event(event_id='event1',
webservice_id=ws.id,
time=utcnow,
latitude=89.5,
longitude=6,
depth_km=7.1,
magnitude=56)
# note: e2 not used, store in db here anyway...
ev2 = Event(event_id='event2',
webservice_id=ws.id,
time=utcnow + timedelta(seconds=5),
latitude=89.5,
longitude=6,
depth_km=7.1,
magnitude=56)
session.add_all([ev1, ev2])
session.commit() # refresh datacenter id (alo flush works)
d = datetime.utcnow()
s = Station(network='network',
station='station',
datacenter_id=dct.id,
latitude=90,
longitude=-45,
start_time=d)
session.add(s)
channels = [
Channel(location='01', channel='HHE', sample_rate=6),
Channel(location='01', channel='HHN', sample_rate=6),
Channel(location='01', channel='HHZ', sample_rate=6),
Channel(location='01', channel='HHW', sample_rate=6),
Channel(location='02', channel='HHE', sample_rate=6),
Channel(location='02', channel='HHN', sample_rate=6),
Channel(location='02', channel='HHZ', sample_rate=6),
Channel(location='04', channel='HHZ', sample_rate=6),
Channel(location='05', channel='HHE', sample_rate=6),
Channel(location='05gap_merged', channel='HHN', sample_rate=6),
Channel(location='05err', channel='HHZ', sample_rate=6),
Channel(location='05gap_unmerged', channel='HHZ', sample_rate=6)
]
s.channels.extend(channels)
session.commit()
fixed_args = dict(datacenter_id=dct.id, download_id=dwl.id)
# Note: data_gaps_merged is a stream where gaps can be merged via obspy.Stream.merge
# data_gaps_unmerged is a stream where gaps cannot be merged (is a stream of three
# different channels of the same event)
data_gaps_unmerged = data.read("GE.FLT1..HH?.mseed")
data_gaps_merged = data.read("IA.BAKI..BHZ.D.2016.004.head")
data_ok = data.read("GE.FLT1..HH?.mseed")
# create an 'ok' and 'error' Stream, the first by taking the first trace of
# "GE.FLT1..HH?.mseed", the second by maipulating it
obspy_stream = data.read_stream(
"GE.FLT1..HH?.mseed") # read(BytesIO(data_ok))
obspy_trace = obspy_stream[0]
# write data_ok is actually bytes data of 3 traces, write just the first one, we have
# as it is it would be considered a trace with gaps, wwe have
# another trace with gaps
b = BytesIO()
obspy_trace.write(b, format='MSEED')
data_ok = b.getvalue()
data_err = data_ok[:5] # whatever slice should be ok
seedid_ok = seedid_err = obspy_trace.get_id()
seedid_gaps_unmerged = None
seedid_gaps_merged = read(BytesIO(data_gaps_merged))[0].get_id()
for evt, cha in product([ev1], channels):
val = int(cha.location[:2])
mseed = data_gaps_merged if "gap_merged" in cha.location else \
data_err if "err" in cha.location else \
data_gaps_unmerged if 'gap_unmerged' in cha.location else data_ok
seedid = seedid_gaps_merged if "gap_merged" in cha.location else \
seedid_err if 'err' in cha.location else \
seedid_gaps_unmerged if 'gap_unmerged' in cha.location else seedid_ok
# set times. For everything except data_ok, we set a out-of-bounds time:
start_time = evt.time - timedelta(seconds=5)
arrival_time = evt.time - timedelta(seconds=4)
end_time = evt.time - timedelta(seconds=1)
if "gap_merged" not in cha.location and 'err' not in cha.location and \
'gap_unmerged' not in cha.location:
start_time = obspy_trace.stats.starttime.datetime
arrival_time = (
obspy_trace.stats.starttime +
(obspy_trace.stats.endtime - obspy_trace.stats.starttime) /
2).datetime
end_time = obspy_trace.stats.endtime.datetime
seg = Segment(request_start=start_time,
arrival_time=arrival_time,
request_end=end_time,
data=mseed,
data_seed_id=seedid,
event_distance_deg=val,
event_id=evt.id,
**fixed_args)
cha.segments.append(seg)
session.commit()
self.inventory_bytes = data.read("GE.FLT1.xml")
self.inventory = data.read_inv("GE.FLT1.xml")
pfile, cfile = get_templates_fpaths('paramtable.py', 'paramtable.yaml')
self.pymodule = load_source(pfile)
self.config = yaml_load(cfile)
# remove segment_select, we use all segments here:
self.config.pop('segment_select', None)
def init(self, request, db, data):
# re-init a sqlite database (no-op if the db is not sqlite):
db.create(to_file=True)
with patch(
'stream2segment.gui.webapp.mainapp.plots.core._default_size_limits',
return_value=(4, 1)) as mock1:
self.app = create_s2s_show_app(db.dburl, self.pyfile,
self.configfile)
with self.app.app_context():
# create a configured "Session" class
# Session = sessionmaker(bind=self.engine)
# create a Session
# session = Session()
session = self.session
dc = DataCenter(station_url="345fbgfnyhtgrefs",
dataselect_url='edfawrefdc')
session.add(dc)
utcnow = datetime.utcnow()
run = Download(run_time=utcnow)
session.add(run)
ws = WebService(url='webserviceurl')
session.add(ws)
session.commit()
id = 'firstevent'
e1 = Event(event_id='event1',
webservice_id=ws.id,
time=utcnow,
latitude=89.5,
longitude=6,
depth_km=7.1,
magnitude=56)
e2 = Event(event_id='event2',
webservice_id=ws.id,
time=utcnow + timedelta(seconds=5),
latitude=89.5,
longitude=6,
depth_km=7.1,
magnitude=56)
session.add_all([e1, e2])
session.commit() # refresh datacenter id (alo flush works)
d = datetime.utcnow()
s = Station(network='network',
station='station',
datacenter_id=dc.id,
latitude=90,
longitude=-45,
start_time=d)
session.add(s)
channels = [
Channel(location='01', channel='HHE', sample_rate=6),
Channel(location='01', channel='HHN', sample_rate=6),
Channel(location='01', channel='HHZ', sample_rate=6),
Channel(location='01', channel='HHW', sample_rate=6),
Channel(location='02', channel='HHE', sample_rate=6),
Channel(location='02', channel='HHN', sample_rate=6),
Channel(location='02', channel='HHZ', sample_rate=6),
Channel(location='03', channel='HHE', sample_rate=6),
Channel(location='03', channel='HHN', sample_rate=6),
Channel(location='04', channel='HHZ', sample_rate=6),
Channel(location='05', channel='HHE', sample_rate=6),
Channel(location='05gap_merged',
channel='HHN',
sample_rate=6),
Channel(location='05err', channel='HHZ', sample_rate=6),
Channel(location='05gap_unmerged',
channel='HHZ',
sample_rate=6)
]
s.channels.extend(channels)
session.commit()
fixed_args = dict(
datacenter_id=dc.id,
download_id=run.id,
)
data_gaps_unmerged = data.to_segment_dict("GE.FLT1..HH?.mseed")
data_gaps_merged = data.to_segment_dict(
"IA.BAKI..BHZ.D.2016.004.head")
obspy_trace = read(BytesIO(data_gaps_unmerged['data']))[0]
# write data_ok is actually bytes data of 3 traces, write just the first one, we have
# as it is it would be considered a trace with gaps, wwe have
# another trace with gaps
b = BytesIO()
obspy_trace.write(b, format='MSEED')
start, end = obspy_trace.stats.starttime.datetime, obspy_trace.stats.endtime.datetime
data_ok = dict(data_gaps_unmerged,
data=b.getvalue(),
start_time=start,
end_time=end,
arrival_time=start + (end - start) / 3)
data_err = dict(data_ok, data=data_ok['data'][:5])
for ev, c in product([e1, e2], channels):
val = int(c.location[:2])
data_atts = data_gaps_merged if "gap_merged" in c.location else \
data_err if "err" in c.location else data_gaps_unmerged \
if 'gap_unmerged' in c.location else data_ok
seg = Segment(event_distance_deg=val,
event_id=ev.id,
datacenter_id=dc.id,
download_id=run.id,
**data_atts)
c.segments.append(seg)
session.commit()
session.close()
# set inventory
self.inventory = data.read_inv("GE.FLT1.xml")
yield
def init(self, request, db, data):
# re-init a sqlite database (no-op if the db is not sqlite):
db.create(to_file=False)
sess = db.session
run = Download()
sess.add(run)
sess.commit()
dcen = DataCenter(station_url="x/station/abc") # invalid fdsn name
with pytest.raises(IntegrityError):
sess.add(dcen)
sess.commit()
sess.rollback()
# https://service.iris.edu/fdsnws/station/1/
dcen = DataCenter(
station_url="x/station/fdsnws/station/1/") # this is save (fdsn)
sess.add(dcen)
sess.commit()
# this is safe (both provided):
dcen = DataCenter(station_url="x/station/abc",
dataselect_url="x/station/abc")
sess.add(dcen)
sess.commit()
ws = WebService(url='abc')
sess.add(ws)
sess.commit()
event1 = Event(id=1,
event_id='a',
webservice_id=ws.id,
time=datetime.utcnow(),
magnitude=5,
latitude=66,
longitude=67,
depth_km=6)
event2 = Event(id=2,
event_id='b',
webservice_id=ws.id,
time=datetime.utcnow(),
magnitude=5,
latitude=66,
longitude=67,
depth_km=6)
sess.add_all([event1, event2])
sess.commit()
sta1 = Station(id=1,
network='n1',
station='s1',
datacenter_id=dcen.id,
latitude=66,
longitude=67,
start_time=datetime.utcnow())
sta2 = Station(id=2,
network='n2',
station='s1',
datacenter_id=dcen.id,
latitude=66,
longitude=67,
start_time=datetime.utcnow())
sess.add_all([sta1, sta2])
sess.commit()
cha1 = Channel(id=1,
location='l1',
channel='c1',
station_id=sta1.id,
sample_rate=6)
cha2 = Channel(id=2,
location='l2',
channel='c2',
station_id=sta1.id,
sample_rate=6)
cha3 = Channel(id=3,
location='l3',
channel='c3',
station_id=sta1.id,
sample_rate=6)
cha4 = Channel(id=4,
location='l4',
channel='c4',
station_id=sta2.id,
sample_rate=6)
sess.add_all([cha1, cha2, cha3, cha4])
sess.commit()
# segment 1, with two class labels 'a' and 'b'
seg1 = Segment(event_id=event1.id,
channel_id=cha3.id,
datacenter_id=dcen.id,
event_distance_deg=5,
download_id=run.id,
arrival_time=datetime.utcnow(),
request_start=datetime.utcnow(),
request_end=datetime.utcnow())
sess.add(seg1)
sess.commit()
cls1 = Class(label='a')
cls2 = Class(label='b')
sess.add_all([cls1, cls2])
sess.commit()
clb1 = ClassLabelling(segment_id=seg1.id, class_id=cls1.id)
clb2 = ClassLabelling(segment_id=seg1.id, class_id=cls2.id)
sess.add_all([clb1, clb2])
sess.commit()
# segment 2, with one class label 'a'
seg2 = Segment(event_id=event1.id,
channel_id=cha2.id,
datacenter_id=dcen.id,
event_distance_deg=6.6,
download_id=run.id,
arrival_time=datetime.utcnow(),
request_start=datetime.utcnow(),
request_end=datetime.utcnow())
sess.add(seg2)
sess.commit()
clb1 = ClassLabelling(segment_id=seg2.id, class_id=cls1.id)
sess.add_all([clb1])
sess.commit()
# segment 3, no class label 'a' (and with data attr, useful later)
seg3 = Segment(event_id=event1.id,
channel_id=cha1.id,
datacenter_id=dcen.id,
event_distance_deg=7,
download_id=run.id,
data=b'data',
arrival_time=datetime.utcnow(),
request_start=datetime.utcnow(),
request_end=datetime.utcnow())
sess.add(seg3)
sess.commit()