def setUp(self):
# create mapping
Mapping(
timerange=DateTimeTZRange(
UTCDateTime(2002, 1, 1).datetime,
UTCDateTime(2016, 1, 2).datetime),
network="TA", station="A25A", location="", channel="BHE",
new_network="XX", new_station="YY", new_location="00",
new_channel="ZZZ").save()
ContinuousTrace.update_all_mappings()
# index waveform files
[process_file(f) for f in FILES]
def setUp(self):
# index waveform files
[process_file(f) for f in FILES]
User.objects.get_or_create(username='******',
password=make_password('random'))
credentials = base64.b64encode(b'random:random')
self.valid_auth_headers = {
'HTTP_AUTHORIZATION': 'Basic ' + credentials.decode("ISO-8859-1")
}
credentials = base64.b64encode(b'random:random2')
self.invalid_auth_headers = {
'HTTP_AUTHORIZATION': 'Basic ' + credentials.decode("ISO-8859-1")
}
def setUp(self):
# index waveform files
[process_file(f) for f in FILES]
def test_dataselect_query_for_slightly_messed_up_files(self):
# Create a file with interleaved traces and some noise traces
# in-between
traces = [
Trace(data=np.ones(10), header={"starttime": UTCDateTime(0)}),
Trace(data=np.ones(10), header={"starttime": UTCDateTime(5)}),
Trace(data=np.ones(10), header={"starttime": UTCDateTime(10)}),
Trace(data=np.ones(10), header={"starttime": UTCDateTime(-5)}),
Trace(data=np.ones(8), header={"starttime": UTCDateTime(2)}),
Trace(data=np.ones(12), header={"starttime": UTCDateTime(0)}),
Trace(data=np.ones(10), header={"starttime": UTCDateTime(1)})
]
for tr in traces:
tr.stats.network = "XX"
tr.stats.station = "YY"
tr.stats.channel = "EHZ"
# Add a couple more random traces in between. These should just be
# ignored for the query.
traces.insert(0, Trace(
data=np.ones(5),
header={"starttime": UTCDateTime(2), "network": "DD",
"station": "BLA", "channel": "BHZ", "location": "10"}))
traces.insert(4, Trace(
data=np.ones(17),
header={"starttime": UTCDateTime(-2), "network": "DD",
"station": "BLA", "channel": "BHZ", "location": "10"}))
traces.append(Trace(
data=np.ones(2),
header={"starttime": UTCDateTime(5), "network": "AB",
"station": "CD", "channel": "XYZ", "location": ""}))
# Write to a temporary file.
filename = tempfile.mkstemp()[1]
try:
Stream(traces=traces).write(filename, format="mseed")
process_file(filename)
params = {
"network": "XX",
"station": "YY",
"location": "",
"channel": "EHZ"}
def _test_number_of_returned_traces(start, end, expected):
params["start"] = str(UTCDateTime(start))
params["end"] = str(UTCDateTime(end))
response = self.client.get('/fdsnws/dataselect/1/query',
params)
if expected == 0:
self.assertEqual(response.status_code, 204)
return
self.assertEqual(response.status_code, 200)
self.assertTrue('OK' in response.reason_phrase)
st = read(io.BytesIO(response.getvalue()))
self.assertEqual(len(st), expected)
_test_number_of_returned_traces(1, 10, 7)
_test_number_of_returned_traces(1, 9, 6)
_test_number_of_returned_traces(-10, 0, 3)
_test_number_of_returned_traces(-10, -1, 1)
_test_number_of_returned_traces(-10, -4, 1)
_test_number_of_returned_traces(-10, -5, 1)
_test_number_of_returned_traces(-10, -5.5, 0)
_test_number_of_returned_traces(10, 15, 4)
_test_number_of_returned_traces(4, 5, 6)
_test_number_of_returned_traces(-6, -5, 1)
_test_number_of_returned_traces(-6, -1, 1)
_test_number_of_returned_traces(-6, 0, 3)
_test_number_of_returned_traces(10, 15, 4)
_test_number_of_returned_traces(11, 15, 3)
_test_number_of_returned_traces(12, 15, 2)
_test_number_of_returned_traces(14, 17, 2)
_test_number_of_returned_traces(15, 17, 1)
_test_number_of_returned_traces(18, 20, 1)
_test_number_of_returned_traces(19, 20, 1)
_test_number_of_returned_traces(19.5, 20, 0)
finally:
os.remove(filename)
def test_waveform_mappings(self):
def delete_indexed_waveforms():
models.File.objects.all().delete()
assert models.ContinuousTrace.objects.count() == 0
# Let's use an example file from the fdsnws test suite.
filename = os.path.join(os.path.dirname(os.path.dirname(self.path)), "fdsnws", "tests", "data", "TA.A25A.mseed")
assert os.path.exists(filename)
expected_ids = [
"TA.A25A..BHE",
"TA.A25A..BHN",
"TA.A25A..BHZ",
"TA.A25A..LCE",
"TA.A25A..LCQ",
"TA.A25A..LHE",
"TA.A25A..LHN",
"TA.A25A..LHZ",
"TA.A25A..UHE",
"TA.A25A..UHN",
"TA.A25A..UHZ",
"TA.A25A..VCO",
"TA.A25A..VEA",
"TA.A25A..VEC",
"TA.A25A..VEP",
"TA.A25A..VHE",
"TA.A25A..VHN",
"TA.A25A..VHZ",
"TA.A25A..VKI",
"TA.A25A..VM1",
"TA.A25A..VM2",
"TA.A25A..VM3",
]
# Process that file.
process_file(filename)
# Make sure it all got stored in the database.
ids = sorted([_i.seed_id for _i in models.ContinuousTrace.objects.all()])
self.assertEqual(expected_ids, ids)
delete_indexed_waveforms()
# Now create a mapping that does not actually do anything, because
# it is out of temporal range.
models.Mapping(
timerange=DateTimeTZRange(obspy.UTCDateTime(2000, 1, 1).datetime, obspy.UTCDateTime(2000, 1, 2).datetime),
network="TA",
station="A25A",
location="",
channel="BHE",
new_network="XX",
new_station="YY",
new_location="00",
new_channel="ZZZ",
).save()
# Nothing should have changed.
process_file(filename)
ids = sorted([_i.seed_id for _i in models.ContinuousTrace.objects.all()])
self.assertEqual(expected_ids, ids)
delete_indexed_waveforms()
# Now create a mapping that does something.
models.Mapping(
timerange=DateTimeTZRange(obspy.UTCDateTime(2002, 1, 1).datetime, obspy.UTCDateTime(2016, 1, 2).datetime),
network="TA",
station="A25A",
location="",
channel="BHE",
new_network="XX",
new_station="YY",
new_location="00",
new_channel="ZZZ",
).save()
# Nothing should have changed.
process_file(filename)
ids = sorted([_i.seed_id for _i in models.ContinuousTrace.objects.all()])
self.assertEqual(len(ids), len(expected_ids))
self.assertIn("XX.YY.00.ZZZ", ids)
self.assertNotIn("TA.A25A..BHE", ids)
# Now remove the mappings and test the reindexing!
models.Mapping.objects.all().delete()
# Without reindex, nothing changed.
ids = sorted([_i.seed_id for _i in models.ContinuousTrace.objects.all()])
self.assertEqual(len(ids), len(expected_ids))
self.assertIn("XX.YY.00.ZZZ", ids)
self.assertNotIn("TA.A25A..BHE", ids)
# Now reindex - it should have changed.
models.ContinuousTrace.update_all_mappings()
ids = sorted([_i.seed_id for _i in models.ContinuousTrace.objects.all()])
self.assertEqual(expected_ids, ids)
delete_indexed_waveforms()
# One last thing - we'll add a mapping that would match but it does
# not match the path regex - it will thus fail to be applied.
models.Mapping(
timerange=DateTimeTZRange(obspy.UTCDateTime(2002, 1, 1).datetime, obspy.UTCDateTime(2016, 1, 2).datetime),
network="TA",
station="A25A",
location="",
channel="BHE",
new_network="XX",
new_station="YY",
new_location="00",
new_channel="ZZZ",
# Does not match the path - this the mapping is ignored.
full_path_regex="^/very/random/path/.*gse2$",
).save()
process_file(filename)
ids = sorted([_i.seed_id for _i in models.ContinuousTrace.objects.all()])
self.assertEqual(expected_ids, ids)
delete_indexed_waveforms()
def test_waveform_mappings(self):
def delete_indexed_waveforms():
models.File.objects.all().delete()
assert models.ContinuousTrace.objects.count() == 0
# Let's use an example file from the fdsnws test suite.
filename = os.path.join(os.path.dirname(os.path.dirname(self.path)),
"fdsnws", "tests", "data", "TA.A25A.mseed")
assert os.path.exists(filename)
expected_ids = [
'TA.A25A..BHE', 'TA.A25A..BHN', 'TA.A25A..BHZ', 'TA.A25A..LCE',
'TA.A25A..LCQ', 'TA.A25A..LHE', 'TA.A25A..LHN', 'TA.A25A..LHZ',
'TA.A25A..UHE', 'TA.A25A..UHN', 'TA.A25A..UHZ', 'TA.A25A..VCO',
'TA.A25A..VEA', 'TA.A25A..VEC', 'TA.A25A..VEP', 'TA.A25A..VHE',
'TA.A25A..VHN', 'TA.A25A..VHZ', 'TA.A25A..VKI', 'TA.A25A..VM1',
'TA.A25A..VM2', 'TA.A25A..VM3'
]
# Process that file.
process_file(filename)
# Make sure it all got stored in the database.
ids = sorted(
[_i.seed_id for _i in models.ContinuousTrace.objects.all()])
self.assertEqual(expected_ids, ids)
delete_indexed_waveforms()
# Now create a mapping that does not actually do anything, because
# it is out of temporal range.
models.Mapping(timerange=DateTimeTZRange(
obspy.UTCDateTime(2000, 1, 1).datetime,
obspy.UTCDateTime(2000, 1, 2).datetime),
network="TA",
station="A25A",
location="",
channel="BHE",
new_network="XX",
new_station="YY",
new_location="00",
new_channel="ZZZ").save()
# Nothing should have changed.
process_file(filename)
ids = sorted(
[_i.seed_id for _i in models.ContinuousTrace.objects.all()])
self.assertEqual(expected_ids, ids)
delete_indexed_waveforms()
# Now create a mapping that does something.
models.Mapping(timerange=DateTimeTZRange(
obspy.UTCDateTime(2002, 1, 1).datetime,
obspy.UTCDateTime(2016, 1, 2).datetime),
network="TA",
station="A25A",
location="",
channel="BHE",
new_network="XX",
new_station="YY",
new_location="00",
new_channel="ZZZ").save()
# Nothing should have changed.
process_file(filename)
ids = sorted(
[_i.seed_id for _i in models.ContinuousTrace.objects.all()])
self.assertEqual(len(ids), len(expected_ids))
self.assertIn("XX.YY.00.ZZZ", ids)
self.assertNotIn("TA.A25A..BHE", ids)
# Now remove the mappings and test the reindexing!
models.Mapping.objects.all().delete()
# Without reindex, nothing changed.
ids = sorted(
[_i.seed_id for _i in models.ContinuousTrace.objects.all()])
self.assertEqual(len(ids), len(expected_ids))
self.assertIn("XX.YY.00.ZZZ", ids)
self.assertNotIn("TA.A25A..BHE", ids)
# Now reindex - it should have changed.
models.ContinuousTrace.update_all_mappings()
ids = sorted(
[_i.seed_id for _i in models.ContinuousTrace.objects.all()])
self.assertEqual(expected_ids, ids)
delete_indexed_waveforms()
# One last thing - we'll add a mapping that would match but it does
# not match the path regex - it will thus fail to be applied.
models.Mapping(
timerange=DateTimeTZRange(
obspy.UTCDateTime(2002, 1, 1).datetime,
obspy.UTCDateTime(2016, 1, 2).datetime),
network="TA",
station="A25A",
location="",
channel="BHE",
new_network="XX",
new_station="YY",
new_location="00",
new_channel="ZZZ",
# Does not match the path - this the mapping is ignored.
full_path_regex="^/very/random/path/.*gse2$").save()
process_file(filename)
ids = sorted(
[_i.seed_id for _i in models.ContinuousTrace.objects.all()])
self.assertEqual(expected_ids, ids)
delete_indexed_waveforms()
