def get_data_sources(self, data_store_id: str, monitor: Monitor) -> list:
"""
Get data sources for a given data store.
:param data_store_id: ID of the data store
:param monitor: a progress monitor
:return: JSON-serializable list of data sources, sorted by name.
"""
data_store = DATA_STORE_REGISTRY.get_data_store(data_store_id)
if data_store is None:
raise ValueError('Unknown data store: "%s"' % data_store_id)
data_sources = data_store.query(monitor=monitor)
if data_store_id == 'esa_cci_odp':
# Filter ESA Open Data Portal data sources
data_source_dict = {ds.id: ds for ds in data_sources}
# noinspection PyTypeChecker
data_source_ids = filter_fileset(data_source_dict.keys(),
includes=conf.get_config_value('included_data_sources', default=None),
excludes=conf.get_config_value('excluded_data_sources', default=None))
data_sources = [data_source_dict[ds_id] for ds_id in data_source_ids]
data_sources = sorted(data_sources, key=lambda ds: ds.title or ds.id)
return [dict(id=data_source.id,
title=data_source.title,
meta_info=data_source.meta_info) for data_source in data_sources]
def test_get_config_value(self):
with self.assertRaises(ValueError) as e:
conf.get_config_value(None)
self.assertEqual(str(e.exception), 'name argument must be given')
with self.assertRaises(ValueError) as e:
conf.get_config_value('')
self.assertEqual(str(e.exception), 'name argument must be given')
value = conf.get_config_value('_im_not_in_', 'Yes!')
self.assertEqual(value, 'Yes!')
def test_get_config_value(self):
with self.assertRaises(ValueError) as e:
get_config_value(None)
self.assertEqual(str(e.exception), 'name argument must be given')
with self.assertRaises(ValueError) as e:
get_config_value('')
self.assertEqual(str(e.exception), 'name argument must be given')
value = get_config_value('_im_not_in_', 'Yes!')
self.assertEqual(value, 'Yes!')
def __init__(self):
super().__init__('esdc', title='Earth System Data Cube', is_local=True)
esdc_data_source_defs = conf.get_config_value('esdc_data_sources', [])
self._data_sources = OrderedDict()
for ds_id, title, local_path in esdc_data_source_defs:
if not ds_id.startswith('esdc.'):
ds_id = 'esdc.' + ds_id
cube = None
try:
cube = cablab.Cube.open(local_path)
# print('Success registering ESDC data source "%s"!' % ds_id)
except Exception as e:
warnings.warn('Failed registering ESDC data source "%s": %s' % (ds_id, e))
# print('Failed to register ESDC data source "%s": %s' % (ds_id, e))
pass
if cube:
self._data_sources[ds_id] = EsdcDataSource(self, ds_id, title, cube)