def _register_dataset_ios(ext_registry: extension.ExtensionRegistry):
"""
Register xcube's standard dataset I/O components used by various CLI and API functions.
"""
ext_registry.add_extension(
loader=extension.import_component('xcube.core.dsio:ZarrDatasetIO', call=True),
point=EXTENSION_POINT_DATASET_IOS, name=FORMAT_NAME_ZARR,
description='Zarr file format (http://zarr.readthedocs.io)',
ext='zarr', modes={'r', 'w', 'a'}
)
ext_registry.add_extension(
loader=extension.import_component('xcube.core.dsio:Netcdf4DatasetIO', call=True),
point=EXTENSION_POINT_DATASET_IOS, name=FORMAT_NAME_NETCDF4,
description='NetCDF-4 file format',
ext='nc', modes={'r', 'w', 'a'}
)
ext_registry.add_extension(
loader=extension.import_component('xcube.core.dsio:CsvDatasetIO', call=True),
point=EXTENSION_POINT_DATASET_IOS, name=FORMAT_NAME_CSV,
description='CSV file format',
ext='csv', modes={'r', 'w'}
)
ext_registry.add_extension(
loader=extension.import_component('xcube.core.dsio:MemDatasetIO', call=True),
point=EXTENSION_POINT_DATASET_IOS, name=FORMAT_NAME_MEM,
description='In-memory dataset I/O',
ext='mem', modes={'r', 'w', 'a'}
)
def init_plugin(ext_registry: extension.ExtensionRegistry):
"""xcube CCI ODP extensions"""
ext_registry.add_extension(
loader=extension.import_component('xcube_cci.dataaccess:CciOdpDataStore'),
point=EXTENSION_POINT_DATA_STORES,
name=DATA_STORE_ID,
description='ESA CCI Open Data Portal',
data_store_notices=[dict(id='terminologyClarification',
title='Terminology Clarification',
content='The ESA CCI Open Data Portal (ODP) utilises an '
'"[ontology](http://vocab-test.ceda.ac.uk/ontology/cci/'
'cci-content/index.html) whose terms might slightly '
'differ from the ones used in this software.\n'
'For example, a *Dataset* in the CCI terminology may '
'refer to all data products generated by a certain CCI '
'project using a specific configuration of algorithms '
'and auxiliary data.\n'
'In this software, a *Data Source* refers to a subset '
'(a file set) of a given ODP dataset whose data share a '
'common spatio-temporal grid and/or share other common '
'properties, e.g. the instrument used for the original '
'measurements.\n'
'In addition, the term *Dataset* is used to represent '
'in-memory instances of gridded data sources or subsets '
'of them.',
intent='primary',
icon='info-sign'),
dict(id='dataCompleteness',
title='Data Completeness',
content='This data store currently provides **only a subset of '
'all datasets** provided by the "ESA CCI Open Data '
'Portal (ODP), namely gridded datasets originally stored '
'in NetCDF format.\n'
'In upcoming versions, the store will also allow for '
'browsing and accessing the remaining ODP datasets. '
'This includes gridded data in TIFF format and also '
'vector data using ESRI Shapefile format.\n'
'For the time being users can download the missing '
'vector data from the ODP FTP server]'
'(http://cci.esa.int/data#ftp) '
'`ftp://anon-ftp.ceda.ac.uk/neodc/esacci/`\n'
'* CCI Glaciers in FTP directory `glaciers`\n'
'* CCI Ice Sheets in FTP directories '
'`ice_sheets_antarctica` and `ice_sheets_greenland',
intent='warning',
icon='warning-sign')])
ext_registry.add_extension(
loader=extension.import_component('xcube_cci.dataaccess:CciOdpCubeOpener'),
point=EXTENSION_POINT_DATA_OPENERS,
name=CUBE_OPENER_ID,
description='xarray.Dataset cubes in Zarr format from ESA CCI Open Data Portal')
ext_registry.add_extension(
loader=extension.import_component('xcube_cci.dataaccess:CciOdpDatasetOpener'),
point=EXTENSION_POINT_DATA_OPENERS,
name=DATASET_OPENER_ID,
description='xarray.Dataset in Zarr format from ESA CCI Open Data Portal')
def init_plugin(ext_registry: extension.ExtensionRegistry):
ext_registry.add_extension(
loader=extension.import_component('xcube_cds.store:CDSDataStore'),
point=EXTENSION_POINT_DATA_STORES,
name=CDS_DATA_STORE_ID,
description='Climate Data Store API')
ext_registry.add_extension(
loader=extension.import_component('xcube_cds.store:CDSDataOpener'),
point=EXTENSION_POINT_DATA_OPENERS,
name=CDS_DATA_OPENER_ID,
description='xarray.Dataset in NetCDF format '
'from Climate Data Store API')
def init_plugin(ext_registry: extension.ExtensionRegistry):
ext_registry.add_extension(
loader=extension.import_component(
'xcube_gen_rbins.iproc:RbinsSeviriHighrocSceneInputProcessor'),
point=EXTENSION_POINT_INPUT_PROCESSORS,
name='rbins-seviri-highroc-scene-l2',
description='RBINS SEVIRI HIGHROC single-scene Level-2 NetCDF inputs')
ext_registry.add_extension(
loader=extension.import_component(
'xcube_gen_rbins.iproc:RbinsSeviriHighrocDailyInputProcessor'),
point=EXTENSION_POINT_INPUT_PROCESSORS,
name='rbins-seviri-highroc-daily-l2',
description='RBINS SEVIRI HIGHROC daily Level-2 NetCDF inputs')
def init_plugin(ext_registry: extension.ExtensionRegistry):
ext_registry.add_extension(
loader=extension.import_component(
'xcube_gen_vito.iproc:VitoS2PlusInputProcessor'),
point=EXTENSION_POINT_INPUT_PROCESSORS,
name='vito-s2plus-l2',
description='VITO Sentinel-2 Plus Level 2 NetCDF inputs')
def _register_cli_commands(ext_registry: extension.ExtensionRegistry):
"""
Register xcube's standard CLI commands.
"""
cli_command_names = [
'chunk',
'compute',
'dump',
'edit',
'extract',
'gen',
'genpts',
'grid',
'level',
'optimize',
'prune',
'resample',
'serve',
'timeit',
'vars2dim',
'verify',
]
for cli_command_name in cli_command_names:
ext_registry.add_extension(
loader=extension.import_component(f'xcube.cli.{cli_command_name}:{cli_command_name}'),
point=EXTENSION_POINT_CLI_COMMANDS,
name=cli_command_name
)
def test_import_component(self):
loader = import_component('test.util.test_extension:TestComponent')
self.assertTrue(callable(loader))
extension = Extension('test_point',
'test_component',
component='dummy')
component = loader(extension)
self.assertIs(TestComponent, component)
def test_import_component_and_call(self):
loader = import_component('test.util.test_extension:TestComponent', call=True, call_args=[42],
call_kwargs={'help': '!'})
self.assertTrue(callable(loader))
extension = Extension('test', 'test', component='x')
component = loader(extension)
self.assertIsInstance(component, TestComponent)
self.assertEqual((42,), component.args)
self.assertEqual({'help': '!'}, component.kwargs)
def _register_input_processors(ext_registry: extension.ExtensionRegistry):
"""
Register xcube's standard input processors used by "xcube gen" and gen_cube().
"""
ext_registry.add_extension(
loader=extension.import_component('xcube.core.gen.iproc:DefaultInputProcessor', call=True),
point=EXTENSION_POINT_INPUT_PROCESSORS, name='default',
description='Single-scene NetCDF/CF inputs in xcube standard format'
)
def _register_data_stores(ext_registry: extension.ExtensionRegistry):
"""
Register xcube's standard data stores.
"""
ext_registry.add_extension(
loader=extension.import_component('xcube.core.store.stores.memory:MemoryDataStore'),
point=EXTENSION_POINT_DATA_STORES, name='memory',
description='Memory data store'
)
ext_registry.add_extension(
loader=extension.import_component('xcube.core.store.stores.directory:DirectoryDataStore'),
point=EXTENSION_POINT_DATA_STORES, name='directory',
description='Directory data store'
)
ext_registry.add_extension(
loader=extension.import_component('xcube.core.store.stores.s3:S3DataStore'),
point=EXTENSION_POINT_DATA_STORES, name='s3',
description='AWS S3 data store'
)
def init_plugin(ext_registry: extension.ExtensionRegistry):
ext_registry.add_extension(
loader=extension.import_component(
'xcube_gen_bc.iproc:SnapOlciHighrocL2InputProcessor'),
point=EXTENSION_POINT_INPUT_PROCESSORS,
name='snap-olci-highroc-l2',
description='SNAP Sentinel-3 OLCI HIGHROC Level-2 NetCDF inputs')
ext_registry.add_extension(
loader=extension.import_component(
'xcube_gen_bc.iproc:SnapOlciCyanoAlertL2InputProcessor'),
point=EXTENSION_POINT_INPUT_PROCESSORS,
name='snap-olci-cyanoalert-l2',
description='SNAP Sentinel-3 OLCI CyanoAlert Level-2 NetCDF inputs')
ext_registry.add_extension(
loader=extension.import_component(
'xcube_gen_bc.iproc:CMEMSInputProcessor'),
point=EXTENSION_POINT_INPUT_PROCESSORS,
name='cmems',
description='Single-scene daily or hourly CMEMS NetCDF/CF inputs')
def test_import_and_transform_component(self):
def transform(imported_component: Any, loaded_extension_:Extension):
return imported_component(-1, name=loaded_extension_.name)
loader = import_component('test.util.test_extension:TestComponent', transform=transform)
self.assertTrue(callable(loader))
extension = Extension('test_point', 'test_component', component='dummy')
component = loader(extension)
self.assertIsInstance(component, TestComponent)
self.assertEqual((-1,), component.args)
self.assertEqual({'name': 'test_component'}, component.kwargs)
def init_plugin(ext_registry: extension.ExtensionRegistry):
# xcube SentinelHub extensions
ext_registry.add_extension(
loader=extension.import_component('xcube_sh.main:cli'),
point=EXTENSION_POINT_CLI_COMMANDS,
name='sh_cli')
# xcube DataAccessor extensions
ext_registry.add_extension(loader=extension.import_component(
'xcube_sh.store:SentinelHubDataStore'),
point=EXTENSION_POINT_DATA_STORES,
name=SH_DATA_STORE_ID,
description='SENTINEL Hub Cloud API')
# xcube DataAccessor extensions
ext_registry.add_extension(
loader=extension.import_component(
'xcube_sh.store:SentinelHubDataOpener'),
point=EXTENSION_POINT_DATA_OPENERS,
name=SH_DATA_OPENER_ID,
description=
'xarray.Dataset cubes in Zarr format from SENTINEL Hub Cloud API')
def _register_data_stores(ext_registry: extension.ExtensionRegistry):
"""
Register xcube's standard data stores.
"""
factory = 'xcube.core.store.fs.registry:get_fs_data_store_class'
for storage_id, storage_description in _FS_STORAGE_ITEMS:
loader = extension.import_component(factory, call_args=[storage_id])
ext_registry.add_extension(
point=EXTENSION_POINT_DATA_STORES,
loader=loader,
name=storage_id,
description=f'Data store that uses a {storage_description}'
)
def _add_extension(class_name: str, ext_name, description):
loader = extension.import_component(class_name)
ext_registry.add_extension(
point=EXTENSION_POINT_DATA_OPENERS,
loader=loader,
name=ext_name,
description=description
)
ext_registry.add_extension(
point=EXTENSION_POINT_DATA_WRITERS,
loader=loader,
name=ext_name,
description=description
)
def _add_fs_data_accessor_ext(point: str,
ext_type: str,
protocol: str,
data_type: str,
format_id: str):
factory_args = (protocol, data_type, format_id)
loader = extension.import_component(factory,
call_args=factory_args)
ext_registry.add_extension(
point=point,
loader=loader,
name=f'{data_type}:{format_id}:{protocol}',
description=f'Data {ext_type} for'
f' a {data_accessor_description}'
f' in {storage_description}'
)
def init_plugin(ext_registry: extension.ExtensionRegistry):
"""xcube gen Service extensions"""
ext_registry.add_extension(loader=extension.import_component('xcube_geodb.cli.main:cli'),
point=EXTENSION_POINT_CLI_COMMANDS,
name='gen_cli')
def init_plugin(ext_registry: extension.ExtensionRegistry):
"""xcube SentinelHub extensions"""
ext_registry.add_extension(
loader=extension.import_component('xcube_sh.cli:cli'),
point=EXTENSION_POINT_CLI_COMMANDS,
name='sh_cli')
