def __init__(self, label: str, total_work: float): assert_given(label, 'label') assert_condition(total_work > 0, 'total_work must be greater than zero') self.label = label self.total_work = total_work self.state: Optional[ProgressState] = None
def from_code(cls, *code: Union[str, Callable], callable_name: str = None, module_name: str = None, callable_params: Dict[str, Any] = None) -> 'CodeConfig': """ Create a code configuration from the given *code* which may be given as one or more plain text strings or callables. This will create a configuration that uses an inline ``code_string`` which contains the source code. :param code: The code. :param callable_name: The callable name. If not given, will be inferred from first callable. Otherwise it defaults to "process_dataset". :param module_name: The module name. If not given, defaults to "user_code". :param callable_params: The parameters passed as keyword-arguments to the callable. :return: A new code configuration. """ assert_given(code, 'code') inline_code, callable_ref = _normalize_inline_code( code, module_name=module_name, callable_name=callable_name) return CodeConfig(callable_ref=callable_ref, inline_code=inline_code, callable_params=callable_params)
def on_end(self, state_stack: Sequence[ProgressState]): assert_given(state_stack, name="state_stack") self._state_stack = state_stack self._current_sender = "on_end" if self._running and len(state_stack) == 1: self._stop_timer(False)
def from_file_set(cls, file_set: Union[FileSet, str, Any], callable_ref: str, install_required: Optional[bool] = None, callable_params: Optional[Dict[str, Any]] = None) \ -> 'CodeConfig': """ Create a code configuration from a file set. :param file_set: The file set. May be a path or a :class:FileSet instance. :param callable_ref: Reference to the callable in the *file_set*, must have form "<module-name>:<callable-name>" :param install_required: Whether the *file_set* is a package that must be installed. :param callable_params: Parameters to be passed as keyword-arguments to the the callable. :return: A new code configuration. """ assert_given(file_set, 'file_set') assert_given(callable_ref, 'callable_ref') return CodeConfig(callable_ref=callable_ref, file_set=_normalize_file_set(file_set), install_required=install_required, callable_params=callable_params)
def __init__(self, **store_params): self._s3, store_params = S3Mixin.consume_s3fs_params(store_params) self._bucket_name, store_params = S3Mixin.consume_bucket_name_param( store_params) assert_given(self._bucket_name, 'bucket_name') assert_condition( not store_params, f'Unknown keyword arguments: {", ".join(store_params.keys())}')
def _assert_valid_type_specifier(cls, type_specifier: Optional[str]): assert_given(type_specifier, 'type_specifier') base_type_specifier = cls._get_base_type_specifier() if not base_type_specifier.is_satisfied_by(type_specifier): raise ValueError( 'type_specifier must satisfy' f' type specifier "{base_type_specifier}", but was "{type_specifier}"' )
def has_data(self, data_id: str, type_specifier: str = None) -> bool: assert_given(data_id, 'data_id') actual_type_specifier = self._get_type_specifier_for_data_id(data_id) if actual_type_specifier is not None: if type_specifier is None or actual_type_specifier.satisfies( type_specifier): path = self._resolve_data_id_to_path(data_id) return os.path.exists(path) return False
def has_data(self, data_id: str, type_specifier: str = None) -> bool: assert_given(data_id, 'data_id') if data_id not in self._data_dict: return False if type_specifier is not None: data_type_specifier = get_type_specifier(self._data_dict[data_id]) if data_type_specifier is None or not data_type_specifier.satisfies( type_specifier): return False return True
def _get_accessor_id_parts(cls, data_id: str, require=True) -> Optional[Tuple[str, str, str]]: assert_given(data_id, 'data_id') _, ext = os.path.splitext(data_id) accessor_id_parts = _FILENAME_EXT_TO_ACCESSOR_ID_PARTS.get(ext) if not accessor_id_parts and require: raise DataStoreError( f'A dataset named "{data_id}" is not supported') return accessor_id_parts
def __init__(self, data_id: str, num_levels: int, type_specifier: Union[ str, TypeSpecifier] = TYPE_SPECIFIER_MULTILEVEL_DATASET, **kwargs): assert_given(data_id, 'data_id') assert_given(num_levels, 'num_levels') super().__init__(data_id=data_id, type_specifier=type_specifier, **kwargs) self.num_levels = num_levels
def __init__(self, store_id: str, store_params: Dict[str, Any] = None, title: str = None, description: str = None): assert_given(store_id, name='store_id') if store_params is not None: assert_instance(store_params, dict, name='store_params') self._store_id = store_id self._store_params = store_params self._title = title self._description = description
def _guess_accessor_id_parts(self, data_id: str, require=True) \ -> Optional[Tuple[str, str, str]]: assert_given(data_id, 'data_id') ext = self._get_filename_ext(data_id) data_type_alias = _FILENAME_EXT_TO_DATA_TYPE_ALIAS.get(ext) format_name = _FILENAME_EXT_TO_FORMAT.get(ext) if data_type_alias is None or format is None: if require: raise DataStoreError(f'Cannot determine data type for ' f' data resource {data_id!r}') return None return data_type_alias, format_name, self.protocol
def __init__(self, name: str, dtype: str, dims: Sequence[str], attrs: Mapping[str, any] = None): assert_given(name, 'name') assert_given(dtype, 'dtype') self.name = name self.dtype = dtype self.dims = tuple(dims) self.ndim = len(self.dims) self.attrs = _convert_nans_to_none( dict(attrs)) if attrs is not None else None
def __init__(self, store_id: str = None, opener_id: str = None, data_id: str = None, store_params: Mapping[str, Any] = None, open_params: Mapping[str, Any] = None): assert_condition(store_id or opener_id, 'One of store_id and opener_id must be given') assert_given(data_id, 'data_id') self.store_id = store_id self.opener_id = opener_id self.data_id = data_id self.store_params = store_params or {} self.open_params = open_params or {}
def __init__(self, data_id: str, num_levels: int, *, data_type: DataTypeLike = MULTI_LEVEL_DATASET_TYPE, **kwargs): assert_given(data_id, 'data_id') assert_given(num_levels, 'num_levels') super().__init__(data_id=data_id, data_type=data_type, **kwargs) assert_true( MULTI_LEVEL_DATASET_TYPE.is_super_type_of(data_type), f'illegal data_type,' f' must be compatible with {MULTI_LEVEL_DATASET_TYPE!r}') self.num_levels = num_levels
def __init__(self, label: str, total_work: float, interval: float = 0.1, initial_interval: float = 0): super().__init__() assert_given(label, 'label') assert_condition(total_work > 0, 'total_work must be greater than zero') self._label = label self._total_work = total_work self._state: Optional[ProgressState] = None self._initial_interval = initial_interval self._interval = interval self._last_worked = 0 self._running = False
def _normalize_int_pair( value: Any, name: str = None, default: Optional[Tuple[int, int]] = UNDEFINED) -> Optional[Tuple[int, int]]: if isinstance(value, int): return value, value elif value is not None: x, y = value return int(x), int(y) elif default != UNDEFINED: return default else: assert_given(name, 'name') raise ValueError(f'{name} must be an int' f' or a sequence of two ints')
def _normalize_number_pair( value: Any, name: str = None, default: Optional[Tuple[Number, Number]] = UNDEFINED ) -> Optional[Tuple[Number, Number]]: if isinstance(value, (float, int)): x, y = value, value return _to_int_or_float(x), _to_int_or_float(y) elif value is not None: x, y = value return _to_int_or_float(x), _to_int_or_float(y) elif default != UNDEFINED: return default else: assert_given(name, 'name') raise ValueError(f'{name} must be a number' f' or a sequence of two numbers')
def new_data_writer(writer_id: str, extension_registry: Optional[ExtensionRegistry] = None, **writer_params) -> 'DataWriter': """ Get an instance of the data writer identified by *writer_id*. The optional, extra writer parameters *writer_params* may be used by data store (``xcube.core.store.DataStore``) implementations so they can share their internal state with the writer. :param writer_id: The data writer identifier. :param extension_registry: Optional extension registry. If not given, the global extension registry will be used. :param writer_params: Extra writer parameters. :return: A data writer instance. """ assert_given(writer_id, 'writer_id') extension_registry = extension_registry or get_extension_registry() return extension_registry.get_component(EXTENSION_POINT_DATA_WRITERS, writer_id)(**writer_params)
def __init__(self, data_id: str, type_specifier: Union[str, TypeSpecifier], crs: str = None, bbox: Tuple[float, float, float, float] = None, spatial_res: float = None, time_range: Tuple[Optional[str], Optional[str]] = None, time_period: str = None, open_params_schema: JsonObjectSchema = None): assert_given(data_id, 'data_id') self._assert_valid_type_specifier(type_specifier) self.data_id = data_id self.type_specifier = TypeSpecifier.normalize(type_specifier) self.crs = crs self.bbox = tuple(bbox) if bbox else None self.spatial_res = spatial_res self.time_range = tuple(time_range) if time_range else None self.time_period = time_period self.open_params_schema = open_params_schema
def from_callable(cls, _callable: Callable, callable_params: Dict[str, Any] = None) -> 'CodeConfig': """ Create a code configuration from the callable *_callable*. Note, the resulting code configuration is only valid in a local context. It cannot be JSON-serialized. To pass such configurations to a service, convert it first using the :meth:to_service first. :param _callable: A function or class :param callable_params: The parameters passed as keyword-arguments to the callable. :return: A new code configuration. """ assert_given(_callable, '_callable') return CodeConfig(_callable=_callable, callable_params=callable_params)
def callback(self, sender: str, elapsed: float, state_stack: Sequence[ProgressState]): assert_given(state_stack, "ProgressStates") state = state_stack[0] callback = { "sender": sender, "state": { "label": state.label, "total_work": state.total_work, "error": state.exc_info_text or False, "progress": state.progress, "elapsed": elapsed, } } callback_api_uri = self.callback_config.api_uri callback_api_access_token = self.callback_config.access_token header = {"Authorization": f"Bearer {callback_api_access_token}"} return requests.put(callback_api_uri, json=callback, headers=header)
def __init__(self, name: str, dtype: str, dims: Sequence[str], *, chunks: Sequence[int] = None, attrs: Mapping[Hashable, any] = None, **additional_properties): assert_given(name, 'name') assert_given(dtype, 'dtype') assert_not_none(dims, 'dims') if additional_properties: warnings.warn(f'Additional properties received;' f' will be ignored: {additional_properties}') self.name = name self.dtype = dtype self.dims = tuple(dims) self.chunks = tuple(chunks) if chunks else None self.attrs = _attrs_to_json(attrs) if attrs else None
def from_github_archive(cls, gh_org: str, gh_repo: str, gh_tag: str, gh_release: str, callable_ref: str, callable_params: Optional[Dict[str, Any]] = None, gh_username: Optional[str] = None, gh_token: Optional[str] = None): """ Create a code configuration from a GitHub archive. :param gh_org: GitHub organisation name or user name :param gh_repo: GitHub repository name :param gh_tag: GitHub release tag :param gh_release: The name of a GitHub release. It is used to form the sub-path into the archive. The sub-path has the form "<gh_repo>-<gh_release>". :param callable_ref: Reference to the callable in the *file_set*, must have form "<module-name>:<callable-name>" :param callable_params: Parameters to be passed as keyword-arguments to the the callable. :param gh_username: Optional GitHub user name. :param gh_token: Optional GitHub user name. :return: """ assert_given(gh_org, 'gh_org') assert_given(gh_org, 'gh_repo') assert_given(gh_org, 'gh_tag') assert_given(gh_org, 'gh_release') gh_url = f'https://github.com/{gh_org}/{gh_repo}/archive/{gh_tag}.zip' gh_sub_path = f'{gh_repo}-{gh_release}' gh_params = None if gh_token is not None: gh_params = dict(token=gh_token, username=gh_username or gh_org) elif gh_username is not None: assert_given(gh_token, 'gh_token') # fails always return CodeConfig(file_set=FileSet(gh_url, sub_path=gh_sub_path, storage_params=gh_params), callable_ref=callable_ref, callable_params=callable_params)
def __init__(self, path: str, sub_path: str = None, includes: Collection[str] = None, excludes: Collection[str] = None, storage_params: Dict[str, Any] = None): assert_instance(path, str, 'path') assert_given(path, 'path') if sub_path is not None: assert_instance(sub_path, str, 'sub_path') self._path = path self._sub_path = sub_path self._storage_params = dict( storage_params) if storage_params is not None else None self._includes = list(includes) if includes is not None else None self._excludes = list(excludes) if excludes is not None else None # computed members self._include_patterns = _translate_patterns(includes or []) self._exclude_patterns = _translate_patterns(excludes or []) # cached, computed members self._details: Optional[_FileSetDetails] = None
def __init__(self, data_id: str, data_type: DataTypeLike, *, crs: str = None, bbox: Tuple[float, float, float, float] = None, time_range: Tuple[Optional[str], Optional[str]] = None, time_period: str = None, open_params_schema: JsonObjectSchema = None, **additional_properties): assert_given(data_id, 'data_id') if additional_properties: warnings.warn(f'Additional properties received;' f' will be ignored: {additional_properties}') self.data_id = data_id self.data_type = DataType.normalize(data_type) self.crs = crs self.bbox = tuple(bbox) if bbox else None self.time_range = tuple(time_range) if time_range else None self.time_period = time_period self.open_params_schema = open_params_schema
def select_temporal_subset(dataset: xr.Dataset, time_range: TimeRange, time_name: str = 'time') -> xr.Dataset: """ Select a temporal subset from *dataset* given *time_range*. :param dataset: The dataset. Must include time :param time_range: Time range given as two time stamps (start, end) that may be (ISO) strings or datetime objects. :param time_name: optional name of the time coordinate variable. Defaults to "time". :return: """ assert_given(time_range, 'time_range') time_name = time_name or 'time' if time_name not in dataset: raise ValueError(f'cannot compute temporal subset: variable' f' "{time_name}" not found in dataset') time_1, time_2 = time_range time_1 = pd.to_datetime(time_1) if time_1 is not None else None time_2 = pd.to_datetime(time_2) if time_2 is not None else None if time_1 is None and time_2 is None: return dataset if time_2 is not None: delta = time_2 - time_2.floor('1D') if delta == pd.Timedelta('0 days 00:00:00'): time_2 += pd.Timedelta('1D') try: return dataset.sel({time_name or 'time': slice(time_1, time_2)}) except TypeError: calendar = dataset.time.encoding.get('calendar') time_1 = cftime.datetime(time_1.year, time_1.month, time_1.day, calendar=calendar) time_2 = cftime.datetime(time_2.year, time_2.month, time_2.day, calendar=calendar) return dataset.sel({time_name or 'time': slice(time_1, time_2)})
def new_data_opener(opener_id: str, extension_registry: Optional[ExtensionRegistry] = None, **opener_params) -> 'DataOpener': """ Get an instance of the data opener identified by *opener_id*. The optional, extra opener parameters *opener_params* may be used by data store (``xcube.core.store.DataStore``) implementations so they can share their internal state with the opener. :param opener_id: The data opener identifier. :param extension_registry: Optional extension registry. If not given, the global extension registry will be used. :param opener_params: Extra opener parameters. :return: A data opener instance. """ assert_given(opener_id, 'opener_id') extension_registry = extension_registry or get_extension_registry() if not extension_registry.has_extension(EXTENSION_POINT_DATA_OPENERS, opener_id): raise DataStoreError(f'A data opener named' f' {opener_id!r} is not registered') return extension_registry.get_component(EXTENSION_POINT_DATA_OPENERS, opener_id)(**opener_params)
def __init__(self, variable_names: Sequence[str] = None, crs: str = None, bbox: Tuple[float, float, float, float] = None, spatial_res: float = None, time_range: Tuple[str, Optional[str]] = None, time_period: str = None): assert_given(variable_names, 'variable_names') assert_given(bbox, 'bbox') assert_given(spatial_res, 'spatial_res') assert_given(time_range, 'time_range') self.variable_names = tuple(variable_names) self.crs = str(crs) self.bbox = tuple(bbox) self.spatial_res = float(spatial_res) self.time_range = tuple(time_range) self.time_period = str(time_period)
def __init__(self, input_config: InputConfig = None, input_configs: Sequence[InputConfig] = None, cube_config: CubeConfig = None, output_config: OutputConfig = None, callback_config: Optional[CallbackConfig] = None): assert_condition( input_config or input_configs, 'one of input_config and input_configs must be given') assert_condition( not (input_config and input_configs), 'input_config and input_configs cannot be given both') if input_config: input_configs = [input_config] assert_given(input_configs, 'input_configs') assert_given(cube_config, 'cube_config') assert_given(output_config, 'output_config') self.input_configs = input_configs self.cube_config = cube_config self.output_config = output_config self.callback_config = callback_config