def test__get_leap_seconds_file(self):
filepath = get_test_data_filepath()
database = os.path.join(filepath, 'timeseries.sqlite')
indexer = Indexer(filepath,
database=database,
loglevel="ERROR")
# test that a bad leap second file path raises an error
self.assertRaisesRegex(OSError,
"^No leap seconds file exists at.*$",
Indexer,
filepath,
database=database,
leap_seconds_file="/some/bad/path/",
loglevel="ERROR")
self.assertRaisesRegex(OSError,
"^No leap seconds file exists at.*$",
indexer._get_leap_seconds_file,
"/some/bad/path/")
# test search
# create a empty leap-seconds.list file
test_file = os.path.normpath(os.path.join(
os.path.dirname(database), "leap-seconds.list"))
open(test_file, 'a').close()
file_path = os.path.normpath(indexer._get_leap_seconds_file("SEARCH"))
self.assertEqual(file_path, test_file)
os.remove(test_file)
def test_download_leap_seconds_file(self):
filepath = get_test_data_filepath()
database = os.path.join(filepath, 'timeseries.sqlite')
indexer = Indexer(filepath, database=database, loglevel="ERROR")
# mock actually downloading the file since this requires a internet
# connection
indexer._download = mock.MagicMock(return_value=requests.Response())
# create a empty leap-seconds.list file
test_file = os.path.join(os.path.dirname(database),
"leap-seconds.list")
file_path = indexer.download_leap_seconds_file(test_file)
# assert that the file was put in the same location as the
# sqlite db
self.assertTrue(os.path.isfile(file_path))
self.assertEqual(file_path, test_file)
os.remove(test_file)
def test_run_bad_index_cmd(self):
"""
Checks that an OSError is raised when there is an error running a
index_cmd. (such as no command found.)
"""
filepath = get_test_data_filepath()
indexer = Indexer(filepath,
filename_pattern="*.mseed",
index_cmd="some_bad_command")
self.assertRaisesRegex(OSError,
"^Required program.* is not installed.*$",
indexer.run)
def test_run(self):
my_uuid = uuid.uuid4().hex
fname = 'test_timeseries_{}'.format(my_uuid)
filepath = get_test_data_filepath()
database = '{}{}.sqlite'.format(filepath, fname)
try:
indexer = Indexer(filepath,
database=database,
filename_pattern="*.mseed",
parallel=2,
loglevel="ERROR")
if indexer._is_index_cmd_installed():
indexer.run(relative_paths=True)
keys = ['network', 'station', 'location', 'channel',
'quality', 'starttime', 'endtime', 'samplerate',
'filename', 'byteoffset', 'bytes', 'hash',
'timeindex', 'timespans', 'timerates', 'format']
NamedRow = namedtuple('NamedRow',
keys)
expected_tsindex_data = \
[
NamedRow(
"CU", "TGUH", "00", "BHZ", "M",
"2018-01-01T00:00:00.000000",
"2018-01-01T00:01:00.000000", 40.0,
"CU/2018/001/"
"CU.TGUH.00.BHZ.2018.001_first_minute.mseed",
0, 4096, "aaaac5315f84cdd174fd8360002a1e3a",
"1514764800.000000=>0,latest=>1",
"[1514764800.000000:1514764860.000000]", None, None),
NamedRow(
"IU", "ANMO", "10", "BHZ", "M",
"2018-01-01T00:00:00.019500",
"2018-01-01T00:00:59.994536", 40.0,
"IU/2018/001/"
"IU.ANMO.10.BHZ.2018.001_first_minute.mseed",
0, 2560, "36a771ca1dc648c505873c164d8b26f2",
"1514764800.019500=>0,latest=>1",
"[1514764800.019500:1514764859.994536]", None, None),
NamedRow(
"IU", "COLA", "10", "BHZ", "M",
"2018-01-01T00:00:00.019500",
"2018-01-01T00:00:59.994538", 40.0,
"IU/2018/001/"
"IU.COLA.10.BHZ.2018.001_first_minute.mseed",
0, 5120, "4ccbb97573ca00ef8c2c4f9c01d27ddf",
"1514764800.019500=>0,latest=>1",
"[1514764800.019500:1514764859.994538]", None, None)]
db_handler = TSIndexDatabaseHandler(database=database)
tsindex_data = db_handler._fetch_index_rows([("I*,C*", "*",
"0?,1?", "*",
"2018-01-01",
"2018-02-01")])
for i in range(0, len(expected_tsindex_data)):
for j in range(0, len(keys)):
self.assertEqual(getattr(expected_tsindex_data[i],
keys[j]),
getattr(tsindex_data[i], keys[j]))
self.assertEqual(len(tsindex_data), len(expected_tsindex_data))
except Exception as err:
raise(err)
finally:
purge(filepath, '^{}.*$'.format(fname))
def test_build_file_list(self):
filepath = get_test_data_filepath()
database = os.path.join(filepath, 'timeseries.sqlite')
indexer = Indexer(filepath,
database=database,
filename_pattern="*.mseed",
loglevel="ERROR")
# test for relative paths
file_list = indexer.build_file_list(relative_paths=True,
reindex=True)
file_list.sort()
self.assertEqual(len(file_list), 3)
self.assertEqual(os.path.normpath(
'CU/2018/001/CU.TGUH.00.BHZ.2018.001_first_minute.mseed'),
file_list[0])
self.assertEqual(os.path.normpath(
'IU/2018/001/IU.ANMO.10.BHZ.2018.001_first_minute.mseed'),
file_list[1])
self.assertEqual(os.path.normpath(
'IU/2018/001/IU.COLA.10.BHZ.2018.001_first_minute.mseed'),
file_list[2])
# case where the root path is outside of the absolute
# data path, to assert that already indexed files are still skipped
indexer = Indexer(tempfile.mkdtemp(),
database=TSIndexDatabaseHandler(database=database),
filename_pattern="*.mseed",
loglevel="ERROR")
self.assertRaisesRegex(OSError,
"^No files matching filename.*$",
indexer.build_file_list,
reindex=True)
# test for absolute paths
# this time pass a TSIndexDatabaseHandler instance as the database
indexer = Indexer(filepath,
database=TSIndexDatabaseHandler(database=database),
filename_pattern="*.mseed",
leap_seconds_file=None,
loglevel="ERROR")
file_list = indexer.build_file_list(reindex=True)
file_list.sort()
self.assertEqual(len(file_list), 3)
self.assertNotEqual(os.path.normpath(
'CU/2018/001/CU.TGUH.00.BHZ.2018.001_first_minute.mseed'),
file_list[0])
self.assertIn(os.path.normpath(
'CU/2018/001/CU.TGUH.00.BHZ.2018.001_first_minute.mseed'),
file_list[0])
self.assertNotEqual(os.path.normpath(
'IU/2018/001/IU.ANMO.10.BHZ.2018.001_first_minute.mseed'),
file_list[1])
self.assertIn(os.path.normpath(
'IU/2018/001/IU.ANMO.10.BHZ.2018.001_first_minute.mseed'),
file_list[1])
self.assertNotEqual(os.path.normpath(
'IU/2018/001/IU.COLA.10.BHZ.2018.001_first_minute.mseed'),
file_list[2])
self.assertIn(os.path.normpath(
'IU/2018/001/IU.COLA.10.BHZ.2018.001_first_minute.mseed'),
file_list[2])
# test that already indexed files (relative and absolute) get skipped.
self.assertRaisesRegex(OSError,
"^No unindexed files matching filename.*$",
indexer.build_file_list,
reindex=False,
relative_paths=False)
self.assertRaisesRegex(OSError,
"^No unindexed files matching filename.*$",
indexer.build_file_list,
reindex=False,
relative_paths=True)
# for this test mock an unindexed file ('data.mseed') to ensure that
# it gets added when reindex is True
mocked_files = [
'CU/2018/001/'
'CU.TGUH.00.BHZ.2018.001_first_minute.mseed',
'IU/2018/001/'
'IU.ANMO.10.BHZ.2018.001_first_minute.mseed',
'IU/2018/001/'
'IU.COLA.10.BHZ.2018.001_first_minute.mseed',
'data.mseed'
]
for i in range(len(mocked_files)):
mocked_files[i] = os.path.normpath(mocked_files[i])
indexer._get_rootpath_files = mock.MagicMock(return_value=mocked_files)
self.assertEqual(indexer.build_file_list(
reindex=False,
relative_paths=False),
['data.mseed'])