def __init__(self, label: str, total_work: float):
assert_given(label, 'label')
assert_condition(total_work > 0,
'total_work must be greater than zero')
self.label = label
self.total_work = total_work
self.state: Optional[ProgressState] = None
def from_code(cls,
*code: Union[str, Callable],
callable_name: str = None,
module_name: str = None,
callable_params: Dict[str, Any] = None) -> 'CodeConfig':
"""
Create a code configuration from the given *code* which may be
given as one or more plain text strings or callables.
This will create a configuration that uses an inline
``code_string`` which contains the source code.
:param code: The code.
:param callable_name: The callable name.
If not given, will be inferred from first callable.
Otherwise it defaults to "process_dataset".
:param module_name: The module name. If not given,
defaults to "user_code".
:param callable_params: The parameters passed
as keyword-arguments to the callable.
:return: A new code configuration.
"""
assert_given(code, 'code')
inline_code, callable_ref = _normalize_inline_code(
code, module_name=module_name, callable_name=callable_name)
return CodeConfig(callable_ref=callable_ref,
inline_code=inline_code,
callable_params=callable_params)
def on_end(self, state_stack: Sequence[ProgressState]):
assert_given(state_stack, name="state_stack")
self._state_stack = state_stack
self._current_sender = "on_end"
if self._running and len(state_stack) == 1:
self._stop_timer(False)
def from_file_set(cls,
file_set: Union[FileSet, str, Any],
callable_ref: str,
install_required: Optional[bool] = None,
callable_params: Optional[Dict[str, Any]] = None) \
-> 'CodeConfig':
"""
Create a code configuration from a file set.
:param file_set: The file set.
May be a path or a :class:FileSet instance.
:param callable_ref: Reference to the callable in the *file_set*,
must have form "<module-name>:<callable-name>"
:param install_required: Whether the *file_set* is
a package that must be installed.
:param callable_params: Parameters to be passed
as keyword-arguments to the the callable.
:return: A new code configuration.
"""
assert_given(file_set, 'file_set')
assert_given(callable_ref, 'callable_ref')
return CodeConfig(callable_ref=callable_ref,
file_set=_normalize_file_set(file_set),
install_required=install_required,
callable_params=callable_params)
def __init__(self, **store_params):
self._s3, store_params = S3Mixin.consume_s3fs_params(store_params)
self._bucket_name, store_params = S3Mixin.consume_bucket_name_param(
store_params)
assert_given(self._bucket_name, 'bucket_name')
assert_condition(
not store_params,
f'Unknown keyword arguments: {", ".join(store_params.keys())}')
def _assert_valid_type_specifier(cls, type_specifier: Optional[str]):
assert_given(type_specifier, 'type_specifier')
base_type_specifier = cls._get_base_type_specifier()
if not base_type_specifier.is_satisfied_by(type_specifier):
raise ValueError(
'type_specifier must satisfy'
f' type specifier "{base_type_specifier}", but was "{type_specifier}"'
)
def has_data(self, data_id: str, type_specifier: str = None) -> bool:
assert_given(data_id, 'data_id')
actual_type_specifier = self._get_type_specifier_for_data_id(data_id)
if actual_type_specifier is not None:
if type_specifier is None or actual_type_specifier.satisfies(
type_specifier):
path = self._resolve_data_id_to_path(data_id)
return os.path.exists(path)
return False
def has_data(self, data_id: str, type_specifier: str = None) -> bool:
assert_given(data_id, 'data_id')
if data_id not in self._data_dict:
return False
if type_specifier is not None:
data_type_specifier = get_type_specifier(self._data_dict[data_id])
if data_type_specifier is None or not data_type_specifier.satisfies(
type_specifier):
return False
return True
def _get_accessor_id_parts(cls,
data_id: str,
require=True) -> Optional[Tuple[str, str, str]]:
assert_given(data_id, 'data_id')
_, ext = os.path.splitext(data_id)
accessor_id_parts = _FILENAME_EXT_TO_ACCESSOR_ID_PARTS.get(ext)
if not accessor_id_parts and require:
raise DataStoreError(
f'A dataset named "{data_id}" is not supported')
return accessor_id_parts
def __init__(self,
data_id: str,
num_levels: int,
type_specifier: Union[
str, TypeSpecifier] = TYPE_SPECIFIER_MULTILEVEL_DATASET,
**kwargs):
assert_given(data_id, 'data_id')
assert_given(num_levels, 'num_levels')
super().__init__(data_id=data_id,
type_specifier=type_specifier,
**kwargs)
self.num_levels = num_levels
def __init__(self,
store_id: str,
store_params: Dict[str, Any] = None,
title: str = None,
description: str = None):
assert_given(store_id, name='store_id')
if store_params is not None:
assert_instance(store_params, dict, name='store_params')
self._store_id = store_id
self._store_params = store_params
self._title = title
self._description = description
def _guess_accessor_id_parts(self, data_id: str, require=True) \
-> Optional[Tuple[str, str, str]]:
assert_given(data_id, 'data_id')
ext = self._get_filename_ext(data_id)
data_type_alias = _FILENAME_EXT_TO_DATA_TYPE_ALIAS.get(ext)
format_name = _FILENAME_EXT_TO_FORMAT.get(ext)
if data_type_alias is None or format is None:
if require:
raise DataStoreError(f'Cannot determine data type for '
f' data resource {data_id!r}')
return None
return data_type_alias, format_name, self.protocol
def __init__(self,
name: str,
dtype: str,
dims: Sequence[str],
attrs: Mapping[str, any] = None):
assert_given(name, 'name')
assert_given(dtype, 'dtype')
self.name = name
self.dtype = dtype
self.dims = tuple(dims)
self.ndim = len(self.dims)
self.attrs = _convert_nans_to_none(
dict(attrs)) if attrs is not None else None
def __init__(self,
store_id: str = None,
opener_id: str = None,
data_id: str = None,
store_params: Mapping[str, Any] = None,
open_params: Mapping[str, Any] = None):
assert_condition(store_id or opener_id,
'One of store_id and opener_id must be given')
assert_given(data_id, 'data_id')
self.store_id = store_id
self.opener_id = opener_id
self.data_id = data_id
self.store_params = store_params or {}
self.open_params = open_params or {}
def __init__(self,
data_id: str,
num_levels: int,
*,
data_type: DataTypeLike = MULTI_LEVEL_DATASET_TYPE,
**kwargs):
assert_given(data_id, 'data_id')
assert_given(num_levels, 'num_levels')
super().__init__(data_id=data_id, data_type=data_type, **kwargs)
assert_true(
MULTI_LEVEL_DATASET_TYPE.is_super_type_of(data_type),
f'illegal data_type,'
f' must be compatible with {MULTI_LEVEL_DATASET_TYPE!r}')
self.num_levels = num_levels
def __init__(self,
label: str,
total_work: float,
interval: float = 0.1,
initial_interval: float = 0):
super().__init__()
assert_given(label, 'label')
assert_condition(total_work > 0, 'total_work must be greater than zero')
self._label = label
self._total_work = total_work
self._state: Optional[ProgressState] = None
self._initial_interval = initial_interval
self._interval = interval
self._last_worked = 0
self._running = False
def _normalize_int_pair(
value: Any,
name: str = None,
default: Optional[Tuple[int,
int]] = UNDEFINED) -> Optional[Tuple[int, int]]:
if isinstance(value, int):
return value, value
elif value is not None:
x, y = value
return int(x), int(y)
elif default != UNDEFINED:
return default
else:
assert_given(name, 'name')
raise ValueError(f'{name} must be an int'
f' or a sequence of two ints')
def _normalize_number_pair(
value: Any,
name: str = None,
default: Optional[Tuple[Number, Number]] = UNDEFINED
) -> Optional[Tuple[Number, Number]]:
if isinstance(value, (float, int)):
x, y = value, value
return _to_int_or_float(x), _to_int_or_float(y)
elif value is not None:
x, y = value
return _to_int_or_float(x), _to_int_or_float(y)
elif default != UNDEFINED:
return default
else:
assert_given(name, 'name')
raise ValueError(f'{name} must be a number'
f' or a sequence of two numbers')
def new_data_writer(writer_id: str,
extension_registry: Optional[ExtensionRegistry] = None,
**writer_params) -> 'DataWriter':
"""
Get an instance of the data writer identified by *writer_id*.
The optional, extra writer parameters *writer_params* may be used by data store
(``xcube.core.store.DataStore``) implementations so they can share their internal state with the writer.
:param writer_id: The data writer identifier.
:param extension_registry: Optional extension registry. If not given, the global extension registry will be used.
:param writer_params: Extra writer parameters.
:return: A data writer instance.
"""
assert_given(writer_id, 'writer_id')
extension_registry = extension_registry or get_extension_registry()
return extension_registry.get_component(EXTENSION_POINT_DATA_WRITERS,
writer_id)(**writer_params)
def __init__(self,
data_id: str,
type_specifier: Union[str, TypeSpecifier],
crs: str = None,
bbox: Tuple[float, float, float, float] = None,
spatial_res: float = None,
time_range: Tuple[Optional[str], Optional[str]] = None,
time_period: str = None,
open_params_schema: JsonObjectSchema = None):
assert_given(data_id, 'data_id')
self._assert_valid_type_specifier(type_specifier)
self.data_id = data_id
self.type_specifier = TypeSpecifier.normalize(type_specifier)
self.crs = crs
self.bbox = tuple(bbox) if bbox else None
self.spatial_res = spatial_res
self.time_range = tuple(time_range) if time_range else None
self.time_period = time_period
self.open_params_schema = open_params_schema
def from_callable(cls,
_callable: Callable,
callable_params: Dict[str, Any] = None) -> 'CodeConfig':
"""
Create a code configuration from the callable *_callable*.
Note, the resulting code configuration is only
valid in a local context. It cannot be JSON-serialized.
To pass such configurations to a service, convert it first
using the :meth:to_service first.
:param _callable: A function or class
:param callable_params: The parameters passed
as keyword-arguments to the callable.
:return: A new code configuration.
"""
assert_given(_callable, '_callable')
return CodeConfig(_callable=_callable, callable_params=callable_params)
def callback(self, sender: str, elapsed: float,
state_stack: Sequence[ProgressState]):
assert_given(state_stack, "ProgressStates")
state = state_stack[0]
callback = {
"sender": sender,
"state": {
"label": state.label,
"total_work": state.total_work,
"error": state.exc_info_text or False,
"progress": state.progress,
"elapsed": elapsed,
}
}
callback_api_uri = self.callback_config.api_uri
callback_api_access_token = self.callback_config.access_token
header = {"Authorization": f"Bearer {callback_api_access_token}"}
return requests.put(callback_api_uri, json=callback, headers=header)
def __init__(self,
name: str,
dtype: str,
dims: Sequence[str],
*,
chunks: Sequence[int] = None,
attrs: Mapping[Hashable, any] = None,
**additional_properties):
assert_given(name, 'name')
assert_given(dtype, 'dtype')
assert_not_none(dims, 'dims')
if additional_properties:
warnings.warn(f'Additional properties received;'
f' will be ignored: {additional_properties}')
self.name = name
self.dtype = dtype
self.dims = tuple(dims)
self.chunks = tuple(chunks) if chunks else None
self.attrs = _attrs_to_json(attrs) if attrs else None
def from_github_archive(cls,
gh_org: str,
gh_repo: str,
gh_tag: str,
gh_release: str,
callable_ref: str,
callable_params: Optional[Dict[str, Any]] = None,
gh_username: Optional[str] = None,
gh_token: Optional[str] = None):
"""
Create a code configuration from a GitHub archive.
:param gh_org: GitHub organisation name or user name
:param gh_repo: GitHub repository name
:param gh_tag: GitHub release tag
:param gh_release: The name of a GitHub release. It is used to
form the sub-path into the archive. The sub-path has the form
"<gh_repo>-<gh_release>".
:param callable_ref: Reference to the callable in the *file_set*,
must have form "<module-name>:<callable-name>"
:param callable_params: Parameters to be passed
as keyword-arguments to the the callable.
:param gh_username: Optional GitHub user name.
:param gh_token: Optional GitHub user name.
:return:
"""
assert_given(gh_org, 'gh_org')
assert_given(gh_org, 'gh_repo')
assert_given(gh_org, 'gh_tag')
assert_given(gh_org, 'gh_release')
gh_url = f'https://github.com/{gh_org}/{gh_repo}/archive/{gh_tag}.zip'
gh_sub_path = f'{gh_repo}-{gh_release}'
gh_params = None
if gh_token is not None:
gh_params = dict(token=gh_token, username=gh_username or gh_org)
elif gh_username is not None:
assert_given(gh_token, 'gh_token') # fails always
return CodeConfig(file_set=FileSet(gh_url,
sub_path=gh_sub_path,
storage_params=gh_params),
callable_ref=callable_ref,
callable_params=callable_params)
def __init__(self,
path: str,
sub_path: str = None,
includes: Collection[str] = None,
excludes: Collection[str] = None,
storage_params: Dict[str, Any] = None):
assert_instance(path, str, 'path')
assert_given(path, 'path')
if sub_path is not None:
assert_instance(sub_path, str, 'sub_path')
self._path = path
self._sub_path = sub_path
self._storage_params = dict(
storage_params) if storage_params is not None else None
self._includes = list(includes) if includes is not None else None
self._excludes = list(excludes) if excludes is not None else None
# computed members
self._include_patterns = _translate_patterns(includes or [])
self._exclude_patterns = _translate_patterns(excludes or [])
# cached, computed members
self._details: Optional[_FileSetDetails] = None
def __init__(self,
data_id: str,
data_type: DataTypeLike,
*,
crs: str = None,
bbox: Tuple[float, float, float, float] = None,
time_range: Tuple[Optional[str], Optional[str]] = None,
time_period: str = None,
open_params_schema: JsonObjectSchema = None,
**additional_properties):
assert_given(data_id, 'data_id')
if additional_properties:
warnings.warn(f'Additional properties received;'
f' will be ignored: {additional_properties}')
self.data_id = data_id
self.data_type = DataType.normalize(data_type)
self.crs = crs
self.bbox = tuple(bbox) if bbox else None
self.time_range = tuple(time_range) if time_range else None
self.time_period = time_period
self.open_params_schema = open_params_schema
def select_temporal_subset(dataset: xr.Dataset,
time_range: TimeRange,
time_name: str = 'time') -> xr.Dataset:
"""
Select a temporal subset from *dataset* given *time_range*.
:param dataset: The dataset. Must include time
:param time_range: Time range given as two time stamps
(start, end) that may be (ISO) strings or datetime objects.
:param time_name: optional name of the time coordinate variable.
Defaults to "time".
:return:
"""
assert_given(time_range, 'time_range')
time_name = time_name or 'time'
if time_name not in dataset:
raise ValueError(f'cannot compute temporal subset: variable'
f' "{time_name}" not found in dataset')
time_1, time_2 = time_range
time_1 = pd.to_datetime(time_1) if time_1 is not None else None
time_2 = pd.to_datetime(time_2) if time_2 is not None else None
if time_1 is None and time_2 is None:
return dataset
if time_2 is not None:
delta = time_2 - time_2.floor('1D')
if delta == pd.Timedelta('0 days 00:00:00'):
time_2 += pd.Timedelta('1D')
try:
return dataset.sel({time_name or 'time': slice(time_1, time_2)})
except TypeError:
calendar = dataset.time.encoding.get('calendar')
time_1 = cftime.datetime(time_1.year,
time_1.month,
time_1.day,
calendar=calendar)
time_2 = cftime.datetime(time_2.year,
time_2.month,
time_2.day,
calendar=calendar)
return dataset.sel({time_name or 'time': slice(time_1, time_2)})
def new_data_opener(opener_id: str,
extension_registry: Optional[ExtensionRegistry] = None,
**opener_params) -> 'DataOpener':
"""
Get an instance of the data opener identified by *opener_id*.
The optional, extra opener parameters *opener_params* may
be used by data store (``xcube.core.store.DataStore``)
implementations so they can share their internal state with the opener.
:param opener_id: The data opener identifier.
:param extension_registry: Optional extension registry.
If not given, the global extension registry will be used.
:param opener_params: Extra opener parameters.
:return: A data opener instance.
"""
assert_given(opener_id, 'opener_id')
extension_registry = extension_registry or get_extension_registry()
if not extension_registry.has_extension(EXTENSION_POINT_DATA_OPENERS,
opener_id):
raise DataStoreError(f'A data opener named'
f' {opener_id!r} is not registered')
return extension_registry.get_component(EXTENSION_POINT_DATA_OPENERS,
opener_id)(**opener_params)
def __init__(self,
variable_names: Sequence[str] = None,
crs: str = None,
bbox: Tuple[float, float, float, float] = None,
spatial_res: float = None,
time_range: Tuple[str, Optional[str]] = None,
time_period: str = None):
assert_given(variable_names, 'variable_names')
assert_given(bbox, 'bbox')
assert_given(spatial_res, 'spatial_res')
assert_given(time_range, 'time_range')
self.variable_names = tuple(variable_names)
self.crs = str(crs)
self.bbox = tuple(bbox)
self.spatial_res = float(spatial_res)
self.time_range = tuple(time_range)
self.time_period = str(time_period)
def __init__(self,
input_config: InputConfig = None,
input_configs: Sequence[InputConfig] = None,
cube_config: CubeConfig = None,
output_config: OutputConfig = None,
callback_config: Optional[CallbackConfig] = None):
assert_condition(
input_config or input_configs,
'one of input_config and input_configs must be given')
assert_condition(
not (input_config and input_configs),
'input_config and input_configs cannot be given both')
if input_config:
input_configs = [input_config]
assert_given(input_configs, 'input_configs')
assert_given(cube_config, 'cube_config')
assert_given(output_config, 'output_config')
self.input_configs = input_configs
self.cube_config = cube_config
self.output_config = output_config
self.callback_config = callback_config
