class LocalDataStoreTest(unittest.TestCase):
def setUp(self):
self.tmp_dir = tempfile.mkdtemp()
self.data_store = LocalDataStore('test', self.tmp_dir)
self.assertTrue(os.path.isdir(self.tmp_dir))
self.assertEqual(0, len(os.listdir(self.tmp_dir)))
self.data_store.add_pattern("ozone", "/DATA/ozone/*/*.nc")
self.data_store.add_pattern("aerosol", ["/DATA/aerosol/*/*/AERO_V1*.nc", "/DATA/aerosol/*/*/AERO_V2*.nc"])
self.assertEqual(2, len(os.listdir(self.tmp_dir)))
self._existing_local_data_store = DATA_STORE_REGISTRY.get_data_store('local')
DATA_STORE_REGISTRY.add_data_store(LocalDataStore('local', self.tmp_dir))
def tearDown(self):
DATA_STORE_REGISTRY.add_data_store(self._existing_local_data_store)
shutil.rmtree(self.tmp_dir, ignore_errors=True)
def test_name_title_and_is_local(self):
self.assertEqual(self.data_store.id, 'test')
self.assertEqual(self.data_store.title, 'Local Data Sources')
self.assertEqual(self.data_store.is_local, True)
def test_add_pattern(self):
data_sources = self.data_store.query()
self.assertIsNotNone(data_sources)
self.assertEqual(len(data_sources), 2)
new_ds_name = 'a_name'
new_ds = self.data_store.add_pattern(new_ds_name, "a_pat")
self.assertEqual("test.%s" % new_ds_name, new_ds.id)
data_sources = self.data_store.query()
self.assertEqual(len(data_sources), 3)
with self.assertRaises(DataAccessError) as cm:
self.data_store.add_pattern("a_name", "a_pat2")
self.assertEqual("DataStore '{}' returned error: Data source '{}.{}' already exists.".format(
self.data_store.id, self.data_store.id, new_ds_name), str(cm.exception))
data_sources = self.data_store.query()
self.assertEqual(len(data_sources), 3)
def test__repr_html(self):
html = self.data_store._repr_html_()
self.assertEqual(524, len(html), html)
def test_init(self):
data_store2 = LocalDataStore('test', self.tmp_dir)
data_sources = data_store2.query()
self.assertIsNotNone(data_sources)
# self.assertEqual(len(data_sources), 2)
def test_query(self):
local_data_store = LocalDataStore('test', os.path.join(os.path.dirname(__file__),
'resources/datasources/local/'))
data_sources = local_data_store.query()
self.assertEqual(len(data_sources), 2)
data_sources = local_data_store.query(ds_id='local')
self.assertEqual(len(data_sources), 1)
self.assertIsNone(data_sources[0].temporal_coverage())
data_sources = local_data_store.query(ds_id='local_w_temporal')
self.assertEqual(len(data_sources), 1)
self.assertIsNotNone(data_sources[0].temporal_coverage())
def test_load_old_datasource_from_json_dict(self):
test_data = {
'name': 'local.test_name',
'meta_data': {
'temporal_coverage': "2001-01-01 00:00:00,2001-01-31 23:59:59",
'spatial_coverage': "-180,-90,180,90",
'variables': ['var_test_1', 'var_test_2'],
},
"meta_info": {
},
'files': [['file_1', '2002-02-01 00:00:00', '2002-02-01 23:59:59'],
['file_2', '2002-03-01 00:00:00', '2002-03-01 23:59:59']]
}
data_source = LocalDataSource.from_json_dict(json_dict=test_data, data_store=self.data_store)
self.assertIsNotNone(data_source)
self.assertEqual(data_source.temporal_coverage(),
TimeRangeLike.convert(test_data.get('meta_data').get('temporal_coverage')))
self.assertEqual(data_source.spatial_coverage(),
PolygonLike.convert(test_data.get('meta_data').get('spatial_coverage')))
self.assertListEqual([var.get('name') for var in data_source.variables_info],
test_data.get('meta_data').get('variables'))
test_data = {
'name': 'local.test_name',
'meta_data': {
'temporal_covrage': "2001-01-01 00:00:00,2001-01-31 23:59:59",
'spatial_coverage': "-180,-90,180,90",
'variables': ['var_test_1', 'var_test_2'],
},
"meta_info": {
},
'files': [['file_1', '2002-02-01 00:00:00', '2002-02-01 23:59:59'],
['file_2', '2002-03-01 00:00:00', '2002-03-01 23:59:59']]
}
data_source = LocalDataSource.from_json_dict(json_dict=test_data, data_store=self.data_store)
self.assertIsNotNone(data_source)
self.assertEqual(data_source.temporal_coverage(),
TimeRangeLike.convert(test_data.get('meta_data').get('temporal_covrage')))
self.assertEqual(data_source.spatial_coverage(),
PolygonLike.convert(test_data.get('meta_data').get('spatial_coverage')))
self.assertListEqual([var.get('name') for var in data_source.variables_info],
test_data.get('meta_data').get('variables'))
def test_load_datasource_from_json_dict(self):
test_data = {
'name': 'local.test_name2',
"meta_info": {
"temporal_coverage_start": "2001-01-01T00:00:00",
"temporal_coverage_end": "2001-01-31T23:59:59",
"bbox_maxx": "180.0",
"bbox_maxy": "90.0",
"bbox_minx": "-180.0",
"bbox_miny": "-90.0",
"variables": [
{
"name": "var_1",
"units": "kelvin",
"long_name": "var_1 long name..",
"standard_name": "std_var_1"
},
{
"name": "var_2",
"units": "celsius",
"long_name": "var_2 long name..",
"standard_name": "std_var_2"
}
]
},
'files': [['file_1', '2002-02-01 00:00:00', '2002-02-01 23:59:59'],
['file_2', '2002-03-01 00:00:00', '2002-03-01 23:59:59']]
}
data_source = LocalDataSource.from_json_dict(json_dict=test_data, data_store=self.data_store)
self.assertIsNotNone(data_source)
self.assertEqual(data_source.temporal_coverage(),
TimeRangeLike.convert("{},{}".format(
test_data.get('meta_info').get('temporal_coverage_start'),
test_data.get('meta_info').get('temporal_coverage_end'))
))
self.assertEqual(data_source.spatial_coverage(),
PolygonLike.convert(",".join([
test_data.get('meta_info').get('bbox_minx'),
test_data.get('meta_info').get('bbox_miny'),
test_data.get('meta_info').get('bbox_maxx'),
test_data.get('meta_info').get('bbox_maxy'),
])))
self.assertListEqual(data_source.variables_info, test_data.get('meta_info').get('variables'))
class LocalFilePatternDataStoreTest(unittest.TestCase):
def setUp(self):
self.tmp_dir = tempfile.mkdtemp()
self.data_store = LocalDataStore('test', self.tmp_dir)
self.assertTrue(os.path.isdir(self.tmp_dir))
self.assertEqual(0, len(os.listdir(self.tmp_dir)))
self.data_store.add_pattern("ozone", "/DATA/ozone/*/*.nc")
self.data_store.add_pattern(
"aerosol",
["/DATA/aerosol/*/*/AERO_V1*.nc", "/DATA/aerosol/*/*/AERO_V2*.nc"])
self.assertEqual(2, len(os.listdir(self.tmp_dir)))
self._existing_local_data_store = DATA_STORE_REGISTRY.get_data_store(
'local')
DATA_STORE_REGISTRY.add_data_store(
LocalDataStore('local', self.tmp_dir))
def tearDown(self):
DATA_STORE_REGISTRY.add_data_store(self._existing_local_data_store)
shutil.rmtree(self.tmp_dir, ignore_errors=True)
def test_add_pattern(self):
data_sources = self.data_store.query()
self.assertIsNotNone(data_sources)
self.assertEqual(len(data_sources), 2)
new_ds = self.data_store.add_pattern("a_name", "a_pat")
self.assertEqual('test.a_name', new_ds.name)
data_sources = self.data_store.query()
self.assertEqual(len(data_sources), 3)
with self.assertRaises(ValueError) as cm:
self.data_store.add_pattern("a_name", "a_pat2")
self.assertEqual(
"Local data store 'test' already contains a data source named 'test.a_name'",
str(cm.exception))
data_sources = self.data_store.query()
self.assertEqual(len(data_sources), 3)
def test__repr_html(self):
html = self.data_store._repr_html_()
self.assertEqual(524, len(html))
def test_init(self):
data_store2 = LocalDataStore('test', self.tmp_dir)
data_sources = data_store2.query()
self.assertIsNotNone(data_sources)
# self.assertEqual(len(data_sources), 2)
def test_query(self):
local_data_store = LocalDataStore(
'test',
os.path.join(os.path.dirname(__file__),
'resources/datasources/local/'))
data_sources = local_data_store.query()
self.assertEqual(len(data_sources), 2)
data_sources = local_data_store.query('local')
self.assertEqual(len(data_sources), 1)
self.assertIsNone(data_sources[0].temporal_coverage())
data_sources = local_data_store.query('local_w_temporal')
self.assertEqual(len(data_sources), 1)
self.assertIsNotNone(data_sources[0].temporal_coverage())
def test_load_datasource_from_json_dict(self):
test_data = { # noqa
'name':
'local.test_name',
'meta_data': {
'type': "FILE_PATTERN",
'data_store': 'local',
'temporal_coverage': "2001-01-01 00:00:00,2001-01-31 23:59:59",
'spatial_coverage': "0,10,20,30",
'variables': ['var_test_1', 'var_test_2'],
'source': 'local.previous_test',
'last_update': None
},
'files': [['file_1', '2002-02-01 00:00:00', '2002-02-01 23:59:59'],
['file_2', '2002-03-01 00:00:00', '2002-03-01 23:59:59']]
}
self.assertEqual(True, True)
class LocalDataStoreTest(unittest.TestCase):
def setUp(self):
self.tmp_dir = tempfile.mkdtemp()
self.data_store = LocalDataStore('test', self.tmp_dir)
self.assertTrue(os.path.isdir(self.tmp_dir))
self.assertEqual(0, len(os.listdir(self.tmp_dir)))
self.data_store.add_pattern("ozone", "/DATA/ozone/*/*.nc")
self.data_store.add_pattern(
"aerosol",
["/DATA/aerosol/*/*/AERO_V1*.nc", "/DATA/aerosol/*/*/AERO_V2*.nc"])
self.assertEqual(2, len(os.listdir(self.tmp_dir)))
self._existing_local_data_store = DATA_STORE_REGISTRY.get_data_store(
'local')
DATA_STORE_REGISTRY.add_data_store(
LocalDataStore('local', self.tmp_dir))
def tearDown(self):
DATA_STORE_REGISTRY.add_data_store(self._existing_local_data_store)
shutil.rmtree(self.tmp_dir, ignore_errors=True)
def test_name_title_and_is_local(self):
self.assertEqual(self.data_store.id, 'test')
self.assertEqual(self.data_store.title, 'Local Data Sources')
self.assertEqual(self.data_store.is_local, True)
def test_create_data_source(self):
new_ds_id = 'test_name.2008'
new_ds = self.data_store.create_data_source(new_ds_id)
self.assertEqual("test.%s" % new_ds_id, new_ds.id)
new_ds_id = 'test_name.200*'
with self.assertRaises(DataAccessError) as cm:
self.data_store.create_data_source(new_ds_id)
self.assertEqual('Unaccepted characters in Data Source name',
str(cm.exception))
def test_add_pattern(self):
data_sources = self.data_store.query()
self.assertIsNotNone(data_sources)
self.assertEqual(len(data_sources), 2)
new_ds_name = 'a_name'
new_ds = self.data_store.add_pattern(new_ds_name, "a_pat")
self.assertEqual("test.%s" % new_ds_name, new_ds.id)
data_sources = self.data_store.query()
self.assertEqual(len(data_sources), 3)
with self.assertRaises(DataAccessError) as cm:
self.data_store.add_pattern("a_name", "a_pat2")
self.assertEqual(
'Data store "{}": Data source "{}.{}" already exists.'.format(
self.data_store.id, self.data_store.id, new_ds_name),
str(cm.exception))
data_sources = self.data_store.query()
self.assertEqual(len(data_sources), 3)
def test__repr_html(self):
html = self.data_store._repr_html_()
self.assertEqual(524, len(html), html)
def test_init(self):
data_store2 = LocalDataStore('test', self.tmp_dir)
data_sources = data_store2.query()
self.assertIsNotNone(data_sources)
# self.assertEqual(len(data_sources), 2)
def test_query(self):
local_data_store = LocalDataStore(
'test',
os.path.join(os.path.dirname(__file__),
'resources/datasources/local/'))
data_sources = local_data_store.query()
self.assertEqual(len(data_sources), 2)
data_sources = local_data_store.query(ds_id='local')
self.assertEqual(len(data_sources), 1)
self.assertIsNone(data_sources[0].temporal_coverage())
data_sources = local_data_store.query(ds_id='local_w_temporal')
self.assertEqual(len(data_sources), 1)
self.assertIsNotNone(data_sources[0].temporal_coverage())
def test_load_old_datasource_from_json_dict(self):
test_data = {
'name':
'local.test_name',
'meta_data': {
'temporal_coverage': "2001-01-01 00:00:00,2001-01-31 23:59:59",
'spatial_coverage': "-180,-90,180,90",
'variables': ['var_test_1', 'var_test_2'],
},
"meta_info": {},
'files': [['file_1', '2002-02-01 00:00:00', '2002-02-01 23:59:59'],
['file_2', '2002-03-01 00:00:00', '2002-03-01 23:59:59']]
}
data_source = LocalDataSource.from_json_dict(
json_dict=test_data, data_store=self.data_store)
self.assertIsNotNone(data_source)
self.assertEqual(
data_source.temporal_coverage(),
TimeRangeLike.convert(
test_data.get('meta_data').get('temporal_coverage')))
self.assertEqual(
data_source.spatial_coverage(),
PolygonLike.convert(
test_data.get('meta_data').get('spatial_coverage')))
self.assertListEqual(
[var.get('name') for var in data_source.variables_info],
test_data.get('meta_data').get('variables'))
test_data = {
'name':
'local.test_name',
'meta_data': {
'temporal_covrage': "2001-01-01 00:00:00,2001-01-31 23:59:59",
'spatial_coverage': "-180,-90,180,90",
'variables': ['var_test_1', 'var_test_2'],
},
"meta_info": {},
'files': [['file_1', '2002-02-01 00:00:00', '2002-02-01 23:59:59'],
['file_2', '2002-03-01 00:00:00', '2002-03-01 23:59:59']]
}
data_source = LocalDataSource.from_json_dict(
json_dict=test_data, data_store=self.data_store)
self.assertIsNotNone(data_source)
self.assertEqual(
data_source.temporal_coverage(),
TimeRangeLike.convert(
test_data.get('meta_data').get('temporal_covrage')))
self.assertEqual(
data_source.spatial_coverage(),
PolygonLike.convert(
test_data.get('meta_data').get('spatial_coverage')))
self.assertListEqual(
[var.get('name') for var in data_source.variables_info],
test_data.get('meta_data').get('variables'))
def test_load_datasource_from_json_dict(self):
test_data = {
'name':
'local.test_name2',
"meta_info": {
"temporal_coverage_start":
"2001-01-01T00:00:00",
"temporal_coverage_end":
"2001-01-31T23:59:59",
"bbox_maxx":
"180.0",
"bbox_maxy":
"90.0",
"bbox_minx":
"-180.0",
"bbox_miny":
"-90.0",
"variables": [{
"name": "var_1",
"units": "kelvin",
"long_name": "var_1 long name..",
"standard_name": "std_var_1"
}, {
"name": "var_2",
"units": "celsius",
"long_name": "var_2 long name..",
"standard_name": "std_var_2"
}]
},
'files': [['file_1', '2002-02-01 00:00:00', '2002-02-01 23:59:59'],
['file_2', '2002-03-01 00:00:00', '2002-03-01 23:59:59']]
}
data_source = LocalDataSource.from_json_dict(
json_dict=test_data, data_store=self.data_store)
self.assertIsNotNone(data_source)
self.assertEqual(
data_source.temporal_coverage(),
TimeRangeLike.convert("{},{}".format(
test_data.get('meta_info').get('temporal_coverage_start'),
test_data.get('meta_info').get('temporal_coverage_end'))))
self.assertEqual(
data_source.spatial_coverage(),
PolygonLike.convert(",".join([
test_data.get('meta_info').get('bbox_minx'),
test_data.get('meta_info').get('bbox_miny'),
test_data.get('meta_info').get('bbox_maxx'),
test_data.get('meta_info').get('bbox_maxy'),
])))
self.assertListEqual(data_source.variables_info,
test_data.get('meta_info').get('variables'))
def test_cache_synch(self):
#
# check if test.asynch is not present
#
ds_asynch = self.data_store.query('test.asynch')
self.assertEqual(len(ds_asynch), 0)
#
# manualy create test-asynch dataset to simulate
# an copy from a second process
#
test_asynch = {
"name": "test.asynch",
"meta_info": {
"status": "PROCESSING"
},
"files": [["/DATA/asynch/*/*.nc"]]
}
json_file = os.path.join(self.data_store.data_store_path,
'test.asynch.json')
with open(json_file, 'w') as f:
json.dump(test_asynch, f)
#
# check the new dataset is not visible
#
ds_asynch = self.data_store.query('test.asynch')
self.assertEqual(len(ds_asynch), 0)
#
# invalidate the cache and check if the new dataset
# is now visible
#
self.data_store.invalidate()
ds_asynch = self.data_store.query('test.asynch')
self.assertEqual(len(ds_asynch), 1)