def init_plugin(ext_registry: extension.ExtensionRegistry):
ext_registry.add_extension(
loader=extension.import_component(
'xcube_gen_vito.iproc:VitoS2PlusInputProcessor'),
point=EXTENSION_POINT_INPUT_PROCESSORS,
name='vito-s2plus-l2',
description='VITO Sentinel-2 Plus Level 2 NetCDF inputs')
def _register_cli_commands(ext_registry: extension.ExtensionRegistry):
"""
Register xcube's standard CLI commands.
"""
cli_command_names = [
'chunk',
'compute',
'dump',
'edit',
'extract',
'gen',
'genpts',
'grid',
'level',
'optimize',
'prune',
'resample',
'serve',
'timeit',
'vars2dim',
'verify',
]
for cli_command_name in cli_command_names:
ext_registry.add_extension(
loader=extension.import_component(f'xcube.cli.{cli_command_name}:{cli_command_name}'),
point=EXTENSION_POINT_CLI_COMMANDS,
name=cli_command_name
)
def _register_input_processors(ext_registry: extension.ExtensionRegistry):
"""
Register xcube's standard input processors used by "xcube gen" and gen_cube().
"""
ext_registry.add_extension(
loader=extension.import_component('xcube.core.gen.iproc:DefaultInputProcessor', call=True),
point=EXTENSION_POINT_INPUT_PROCESSORS, name='default',
description='Single-scene NetCDF/CF inputs in xcube standard format'
)
def init_plugin(ext_registry: extension.ExtensionRegistry):
ext_registry.add_extension(
loader=extension.import_component('xcube_cds.store:CDSDataStore'),
point=EXTENSION_POINT_DATA_STORES,
name=CDS_DATA_STORE_ID,
description='Climate Data Store API')
ext_registry.add_extension(
loader=extension.import_component('xcube_cds.store:CDSDataOpener'),
point=EXTENSION_POINT_DATA_OPENERS,
name=CDS_DATA_OPENER_ID,
description='xarray.Dataset in NetCDF format '
'from Climate Data Store API')
def init_plugin(ext_registry: extension.ExtensionRegistry):
ext_registry.add_extension(
loader=extension.import_component(
'xcube_gen_rbins.iproc:RbinsSeviriHighrocSceneInputProcessor'),
point=EXTENSION_POINT_INPUT_PROCESSORS,
name='rbins-seviri-highroc-scene-l2',
description='RBINS SEVIRI HIGHROC single-scene Level-2 NetCDF inputs')
ext_registry.add_extension(
loader=extension.import_component(
'xcube_gen_rbins.iproc:RbinsSeviriHighrocDailyInputProcessor'),
point=EXTENSION_POINT_INPUT_PROCESSORS,
name='rbins-seviri-highroc-daily-l2',
description='RBINS SEVIRI HIGHROC daily Level-2 NetCDF inputs')
def _register_dataset_ios(ext_registry: extension.ExtensionRegistry):
"""
Register xcube's standard dataset I/O components used by various CLI and API functions.
"""
ext_registry.add_extension(
loader=extension.import_component('xcube.core.dsio:ZarrDatasetIO', call=True),
point=EXTENSION_POINT_DATASET_IOS, name=FORMAT_NAME_ZARR,
description='Zarr file format (http://zarr.readthedocs.io)',
ext='zarr', modes={'r', 'w', 'a'}
)
ext_registry.add_extension(
loader=extension.import_component('xcube.core.dsio:Netcdf4DatasetIO', call=True),
point=EXTENSION_POINT_DATASET_IOS, name=FORMAT_NAME_NETCDF4,
description='NetCDF-4 file format',
ext='nc', modes={'r', 'w', 'a'}
)
ext_registry.add_extension(
loader=extension.import_component('xcube.core.dsio:CsvDatasetIO', call=True),
point=EXTENSION_POINT_DATASET_IOS, name=FORMAT_NAME_CSV,
description='CSV file format',
ext='csv', modes={'r', 'w'}
)
ext_registry.add_extension(
loader=extension.import_component('xcube.core.dsio:MemDatasetIO', call=True),
point=EXTENSION_POINT_DATASET_IOS, name=FORMAT_NAME_MEM,
description='In-memory dataset I/O',
ext='mem', modes={'r', 'w', 'a'}
)
def _register_data_stores(ext_registry: extension.ExtensionRegistry):
"""
Register xcube's standard data stores.
"""
factory = 'xcube.core.store.fs.registry:get_fs_data_store_class'
for storage_id, storage_description in _FS_STORAGE_ITEMS:
loader = extension.import_component(factory, call_args=[storage_id])
ext_registry.add_extension(
point=EXTENSION_POINT_DATA_STORES,
loader=loader,
name=storage_id,
description=f'Data store that uses a {storage_description}'
)
def test_init_plugin(self):
ext_reg = ExtensionRegistry()
init_plugin(ext_reg)
self.assertTrue(
ext_reg.has_extension('xcube.core.gen.iproc', 'default'))
self.assertTrue(ext_reg.has_extension('xcube.core.dsio', 'zarr'))
self.assertTrue(ext_reg.has_extension('xcube.core.dsio', 'netcdf4'))
self.assertTrue(ext_reg.has_extension('xcube.core.dsio', 'csv'))
self.assertTrue(ext_reg.has_extension('xcube.core.dsio', 'mem'))
self.assertTrue(ext_reg.has_extension('xcube.cli', 'compute'))
self.assertTrue(ext_reg.has_extension('xcube.cli', 'extract'))
self.assertTrue(ext_reg.has_extension('xcube.cli', 'gen'))
self.assertTrue(ext_reg.has_extension('xcube.cli', 'level'))
self.assertTrue(ext_reg.has_extension('xcube.cli', 'optimize'))
def test_plugin(self):
"""Assert xcube extensions registered by xcube-sh"""
registry = ExtensionRegistry()
init_plugin(registry)
self.assertEqual({'xcube.cli': {'sh_cli': {
'component': '<not loaded yet>',
'name': 'sh_cli',
'point': 'xcube.cli'}},
'xcube.core.store': {
'sentinelhub': {
'component': '<not loaded yet>',
'description': 'SENTINEL Hub Cloud API',
'name': 'sentinelhub',
'point': 'xcube.core.store'}},
'xcube.core.store.opener': {
'dataset[cube]:zarr:sentinelhub': {
'component': '<not loaded yet>',
'description': 'xarray.Dataset cubes in Zarr format from SENTINEL Hub Cloud API',
'name': 'dataset[cube]:zarr:sentinelhub',
'point': 'xcube.core.store.opener'
}}},
registry.to_dict())
def init_plugin(ext_registry: extension.ExtensionRegistry):
"""xcube CCI ODP extensions"""
ext_registry.add_extension(
loader=extension.import_component('xcube_cci.dataaccess:CciOdpDataStore'),
point=EXTENSION_POINT_DATA_STORES,
name=DATA_STORE_ID,
description='ESA CCI Open Data Portal',
data_store_notices=[dict(id='terminologyClarification',
title='Terminology Clarification',
content='The ESA CCI Open Data Portal (ODP) utilises an '
'"[ontology](http://vocab-test.ceda.ac.uk/ontology/cci/'
'cci-content/index.html) whose terms might slightly '
'differ from the ones used in this software.\n'
'For example, a *Dataset* in the CCI terminology may '
'refer to all data products generated by a certain CCI '
'project using a specific configuration of algorithms '
'and auxiliary data.\n'
'In this software, a *Data Source* refers to a subset '
'(a file set) of a given ODP dataset whose data share a '
'common spatio-temporal grid and/or share other common '
'properties, e.g. the instrument used for the original '
'measurements.\n'
'In addition, the term *Dataset* is used to represent '
'in-memory instances of gridded data sources or subsets '
'of them.',
intent='primary',
icon='info-sign'),
dict(id='dataCompleteness',
title='Data Completeness',
content='This data store currently provides **only a subset of '
'all datasets** provided by the "ESA CCI Open Data '
'Portal (ODP), namely gridded datasets originally stored '
'in NetCDF format.\n'
'In upcoming versions, the store will also allow for '
'browsing and accessing the remaining ODP datasets. '
'This includes gridded data in TIFF format and also '
'vector data using ESRI Shapefile format.\n'
'For the time being users can download the missing '
'vector data from the ODP FTP server]'
'(http://cci.esa.int/data#ftp) '
'`ftp://anon-ftp.ceda.ac.uk/neodc/esacci/`\n'
'* CCI Glaciers in FTP directory `glaciers`\n'
'* CCI Ice Sheets in FTP directories '
'`ice_sheets_antarctica` and `ice_sheets_greenland',
intent='warning',
icon='warning-sign')])
ext_registry.add_extension(
loader=extension.import_component('xcube_cci.dataaccess:CciOdpCubeOpener'),
point=EXTENSION_POINT_DATA_OPENERS,
name=CUBE_OPENER_ID,
description='xarray.Dataset cubes in Zarr format from ESA CCI Open Data Portal')
ext_registry.add_extension(
loader=extension.import_component('xcube_cci.dataaccess:CciOdpDatasetOpener'),
point=EXTENSION_POINT_DATA_OPENERS,
name=DATASET_OPENER_ID,
description='xarray.Dataset in Zarr format from ESA CCI Open Data Portal')
class PluginTest(unittest.TestCase):
def setUp(self):
self.ext_registry = ExtensionRegistry()
def test_get_xcube_default_plugins(self):
plugins = get_plugins()
self.assertIsNotNone(plugins)
self.assertIn('xcube', plugins)
def test_load_plugins_by_entry_points(self):
plugins = load_plugins([EntryPoint('test', init_plugin)], ext_registry=self.ext_registry)
self.assertEqual(dict(test=dict(name='test', doc='A test plugin that registers test extensions')), plugins)
self.assertTrue(self.ext_registry.has_extension('test.util.test_plugin', 'ext1'))
self.assertTrue(self.ext_registry.has_extension('test.util.test_plugin', 'ext2'))
self.assertTrue(self.ext_registry.has_extension('test.util.test_plugin', 'ext3'))
def test_load_plugins_by_module_discovery(self):
entry_points = discover_plugin_modules(module_prefixes=['test'])
plugins = load_plugins(entry_points, ext_registry=self.ext_registry)
self.assertEqual(dict(test=dict(name='test', doc='A test plugin that registers test extensions')), plugins)
self.assertTrue(self.ext_registry.has_extension('test.util.test_plugin', 'ext1'))
self.assertTrue(self.ext_registry.has_extension('test.util.test_plugin', 'ext2'))
self.assertTrue(self.ext_registry.has_extension('test.util.test_plugin', 'ext3'))
def test_load_plugins_by_bad_entry_point(self):
plugins = load_plugins([EntryPointBad('test', init_plugin)], ext_registry=self.ext_registry)
self.assertEqual({}, plugins)
self.assertEqual([], self.ext_registry.find_components('test.util.test_plugin'))
def test_load_plugins_by_bad_init_plugin(self):
plugins = load_plugins([EntryPoint('test', init_plugin_bad)], ext_registry=self.ext_registry)
self.assertEqual({}, plugins)
self.assertEqual([], self.ext_registry.find_components('test.util.test_plugin'))
def test_load_plugins_init_plugin_not_callable(self):
plugins = load_plugins([EntryPoint('test', "init_plugin_not_callable")], ext_registry=self.ext_registry)
self.assertEqual({}, plugins)
self.assertEqual([], self.ext_registry.find_components('test'))
def test_load_plugins_by_failing_module_discovery(self):
entry_points = discover_plugin_modules(module_prefixes=['random'])
plugins = load_plugins(entry_points, ext_registry=self.ext_registry)
self.assertEqual({}, plugins)
self.assertEqual([], self.ext_registry.find_components('test.util.test_plugin'))
def init_plugin(ext_registry: ExtensionRegistry):
"""A test plugin that registers test extensions"""
ext_registry.add_extension(component=object(),
point='test.util.test_plugin',
name='ext1')
ext_registry.add_extension(component=object(),
point='test.util.test_plugin',
name='ext2')
ext_registry.add_extension(component=object(),
point='test.util.test_plugin',
name='ext3')
def _register_data_stores(ext_registry: extension.ExtensionRegistry):
"""
Register xcube's standard data stores.
"""
ext_registry.add_extension(
loader=extension.import_component('xcube.core.store.stores.memory:MemoryDataStore'),
point=EXTENSION_POINT_DATA_STORES, name='memory',
description='Memory data store'
)
ext_registry.add_extension(
loader=extension.import_component('xcube.core.store.stores.directory:DirectoryDataStore'),
point=EXTENSION_POINT_DATA_STORES, name='directory',
description='Directory data store'
)
ext_registry.add_extension(
loader=extension.import_component('xcube.core.store.stores.s3:S3DataStore'),
point=EXTENSION_POINT_DATA_STORES, name='s3',
description='AWS S3 data store'
)
def init_plugin(ext_registry: extension.ExtensionRegistry):
ext_registry.add_extension(
loader=extension.import_component(
'xcube_gen_bc.iproc:SnapOlciHighrocL2InputProcessor'),
point=EXTENSION_POINT_INPUT_PROCESSORS,
name='snap-olci-highroc-l2',
description='SNAP Sentinel-3 OLCI HIGHROC Level-2 NetCDF inputs')
ext_registry.add_extension(
loader=extension.import_component(
'xcube_gen_bc.iproc:SnapOlciCyanoAlertL2InputProcessor'),
point=EXTENSION_POINT_INPUT_PROCESSORS,
name='snap-olci-cyanoalert-l2',
description='SNAP Sentinel-3 OLCI CyanoAlert Level-2 NetCDF inputs')
ext_registry.add_extension(
loader=extension.import_component(
'xcube_gen_bc.iproc:CMEMSInputProcessor'),
point=EXTENSION_POINT_INPUT_PROCESSORS,
name='cmems',
description='Single-scene daily or hourly CMEMS NetCDF/CF inputs')
def init_plugin(ext_registry: extension.ExtensionRegistry):
# xcube SentinelHub extensions
ext_registry.add_extension(
loader=extension.import_component('xcube_sh.main:cli'),
point=EXTENSION_POINT_CLI_COMMANDS,
name='sh_cli')
# xcube DataAccessor extensions
ext_registry.add_extension(loader=extension.import_component(
'xcube_sh.store:SentinelHubDataStore'),
point=EXTENSION_POINT_DATA_STORES,
name=SH_DATA_STORE_ID,
description='SENTINEL Hub Cloud API')
# xcube DataAccessor extensions
ext_registry.add_extension(
loader=extension.import_component(
'xcube_sh.store:SentinelHubDataOpener'),
point=EXTENSION_POINT_DATA_OPENERS,
name=SH_DATA_OPENER_ID,
description=
'xarray.Dataset cubes in Zarr format from SENTINEL Hub Cloud API')
def test_protocol(self):
ext_reg = ExtensionRegistry()
a_obj = A()
def b_loader(ext):
return B(name=ext.name)
a_ext = ext_reg.add_extension(component=a_obj, point='A', name='test')
b_ext = ext_reg.add_extension(loader=b_loader, point='B', name='test')
self.assertEqual(True, ext_reg.has_extension('A', 'test'))
self.assertIsInstance(a_ext, Extension)
self.assertEqual('test', a_ext.name)
self.assertEqual('A', a_ext.point)
self.assertEqual(False, a_ext.is_lazy)
self.assertIs(a_obj, a_ext.component)
self.assertIs(a_obj, a_ext.component)
self.assertIs(a_obj, ext_reg.get_component('A', 'test'))
self.assertIs(a_ext, ext_reg.get_extension('A', 'test'))
self.assertEqual(True, ext_reg.has_extension('B', 'test'))
self.assertIsInstance(b_ext, Extension)
self.assertEqual('test', b_ext.name)
self.assertEqual('B', b_ext.point)
self.assertEqual(True, b_ext.is_lazy)
b_obj = b_ext.component
self.assertIs(b_obj, b_ext.component)
self.assertIs(b_obj, ext_reg.get_component('B', 'test'))
self.assertIs(b_ext, ext_reg.get_extension('B', 'test'))
self.assertEqual({'name': 'test'}, b_obj.kwargs)
self.assertEqual([a_ext], ext_reg.find_extensions('A'))
self.assertEqual([b_ext], ext_reg.find_extensions('B'))
self.assertEqual([], ext_reg.find_extensions('C'))
self.assertEqual([a_obj], ext_reg.find_components('A'))
self.assertEqual([b_obj], ext_reg.find_components('B'))
self.assertEqual([], ext_reg.find_components('C'))
ext_reg.remove_extension('A', 'test')
self.assertEqual(False, ext_reg.has_extension('A', 'test'))
ext_reg.remove_extension('B', 'test')
self.assertEqual(False, ext_reg.has_extension('B', 'test'))
def setUp(self):
self.ext_registry = ExtensionRegistry()
def test_get_extension_illegal(self):
ext_reg = ExtensionRegistry()
self.assertEqual(False, ext_reg.has_extension('A', 'test'))
self.assertEqual(None, ext_reg.get_extension('A', 'test'))
def test_get_component_illegal(self):
ext_reg = ExtensionRegistry()
with self.assertRaises(ValueError) as cm:
ext_reg.get_component('A', 'test')
self.assertEqual("extension 'test' not found for extension point 'A'", f'{cm.exception}')
def init_plugin(ext_registry: extension.ExtensionRegistry):
"""xcube SentinelHub extensions"""
ext_registry.add_extension(
loader=extension.import_component('xcube_sh.cli:cli'),
point=EXTENSION_POINT_CLI_COMMANDS,
name='sh_cli')
def test_find(self):
ext_reg = ExtensionRegistry()
obj1 = A()
obj2 = A()
obj3 = A()
obj4 = B()
obj5 = B()
obj6 = B()
# noinspection PyUnusedLocal
def load_obj3(ext: Extension):
return obj3
# noinspection PyUnusedLocal
def load_obj4(ext: Extension):
return obj4
ext_reg.add_extension(component=obj1, point='A', name='a1', description='knorg')
ext_reg.add_extension(component=obj2, point='A', name='a2', description='gnatz')
ext_reg.add_extension(loader=load_obj3, point='A', name='a3', description='gnatz')
ext_reg.add_extension(loader=load_obj4, point='B', name='b1', description='knorg')
ext_reg.add_extension(component=obj5, point='B', name='b2', description='gnatz')
ext_reg.add_extension(component=obj6, point='B', name='b3', description='knorg')
def is_knorg(ext: Extension):
return ext.metadata.get('description') == 'knorg'
def is_gnatz(ext: Extension):
return ext.metadata.get('description') == 'gnatz'
result = ext_reg.find_extensions('A', predicate=is_knorg)
self.assertEqual(1, len(result))
result = ext_reg.find_components('A', predicate=is_knorg)
self.assertEqual(1, len(result))
result = ext_reg.find_extensions('B', predicate=is_knorg)
self.assertEqual(2, len(result))
result = ext_reg.find_extensions('C', predicate=is_knorg)
self.assertEqual(0, len(result))
result = ext_reg.find_components('C', predicate=is_knorg)
self.assertEqual(0, len(result))
result = ext_reg.find_extensions('A', predicate=is_gnatz)
self.assertEqual(2, len(result))
result = ext_reg.find_components('A', predicate=is_gnatz)
self.assertEqual(2, len(result))
result = ext_reg.find_extensions('B', predicate=is_gnatz)
self.assertEqual(1, len(result))
result = ext_reg.find_components('B', predicate=is_gnatz)
self.assertEqual(1, len(result))
result = ext_reg.find_extensions('C', predicate=is_gnatz)
self.assertEqual(0, len(result))
result = ext_reg.find_components('C', predicate=is_gnatz)
self.assertEqual(0, len(result))
def init_plugin(ext_registry: extension.ExtensionRegistry):
"""xcube gen Service extensions"""
ext_registry.add_extension(loader=extension.import_component('xcube_geodb.cli.main:cli'),
point=EXTENSION_POINT_CLI_COMMANDS,
name='gen_cli')