def get_data_sources(self, data_store_id: str, monitor: Monitor) -> list:
"""
Get data sources for a given data store.
:param data_store_id: ID of the data store
:param monitor: a progress monitor
:return: JSON-serializable list of data sources, sorted by name.
"""
data_store = DATA_STORE_REGISTRY.get_data_store(data_store_id)
if data_store is None:
raise ValueError('Unknown data store: "%s"' % data_store_id)
data_sources = data_store.query(monitor=monitor)
if data_store_id == 'esa_cci_odp':
# Filter ESA Open Data Portal data sources
data_source_dict = {ds.id: ds for ds in data_sources}
# noinspection PyTypeChecker
data_source_ids = filter_fileset(data_source_dict.keys(),
includes=conf.get_config_value('included_data_sources', default=None),
excludes=conf.get_config_value('excluded_data_sources', default=None))
data_sources = [data_source_dict[ds_id] for ds_id in data_source_ids]
data_sources = sorted(data_sources, key=lambda ds: ds.title or ds.id)
return [dict(id=data_source.id,
title=data_source.title,
meta_info=data_source.meta_info) for data_source in data_sources]
def test_include_exclude(self):
filtered_data_sources = filter_fileset(
ALL_DATA_SOURCES,
includes=['esacci.CLOUD.mon.*.multi-platform.*', 'esacci.OZONE.*'],
excludes=['*.day.*', '*.ATSR2-AATSR.*'])
print(repr(sorted(filtered_data_sources)))
self.assertEqual(sorted(filtered_data_sources), [
'esacci.CLOUD.mon.L3C.CLD_PRODUCTS.AVHRR.multi-platform.AVHRR-AM.2-0.r1',
'esacci.CLOUD.mon.L3C.CLD_PRODUCTS.AVHRR.multi-platform.AVHRR-PM.2-0.r1',
'esacci.OZONE.mon.L3.NP.multi-sensor.multi-platform.MERGED.fv0002.r1',
])
def test_exclude(self):
filtered_data_sources = filter_fileset(
ALL_DATA_SOURCES,
excludes=[
'*.day.*', '*.satellite-orbit-frequency.*', '*.multi-sensor.*'
])
print(repr(sorted(filtered_data_sources)))
self.assertEqual(sorted(filtered_data_sources), [
'esacci.AEROSOL.5-days.L3C.AEX.GOMOS.Envisat.aergom.2-19.r1',
'esacci.AEROSOL.mon.L3C.AER_PRODUCTS.AATSR.Envisat.ORAC.03-02.r1',
'esacci.AEROSOL.mon.L3C.AER_PRODUCTS.AATSR.Envisat.SU.4-21.r1',
'esacci.AEROSOL.mon.L3C.AER_PRODUCTS.ATSR-2.ERS-2.ORAC.03-02.r1',
'esacci.AEROSOL.mon.L3C.AER_PRODUCTS.ATSR-2.ERS-2.SU.4-21.r1',
'esacci.AEROSOL.mon.L3C.AOD.MERIS.Envisat.MERIS_ENVISAT.2-2.r1',
'esacci.CLOUD.mon.L3C.CLD_PRODUCTS.AVHRR.multi-platform.AVHRR-AM.2-0.r1',
'esacci.CLOUD.mon.L3C.CLD_PRODUCTS.AVHRR.multi-platform.AVHRR-PM.2-0.r1',
'esacci.CLOUD.mon.L3C.CLD_PRODUCTS.MODIS.Aqua.MODIS_AQUA.2-0.r1',
'esacci.CLOUD.mon.L3C.CLD_PRODUCTS.MODIS.Terra.MODIS_TERRA.2-0.r1',
'esacci.LC.13-yrs.L4.WB.asar.Envisat.Map.4-0.r1'
])
def test_none(self):
filtered_data_sources = filter_fileset(ALL_DATA_SOURCES)
self.assertIs(filtered_data_sources, ALL_DATA_SOURCES)