def _prune(input_path: str = None, dry_run: bool = False, monitor=None): from xcube.core.chunk import get_empty_dataset_chunks from xcube.core.dsio import guess_dataset_format from xcube.core.dsio import open_cube input_format = guess_dataset_format(input_path) if input_format != FORMAT_NAME_ZARR: raise click.ClickException("input must be a cube in ZARR format") monitor(f'Opening cube from {input_path!r}...') with open_cube(input_path) as cube: monitor('Identifying empty blocks...') empty_chunks = get_empty_dataset_chunks(cube) num_deleted = 0 for var_name, chunk_indices in empty_chunks.items(): monitor( f'Deleting {len(chunk_indices)} empty block file(s) for variable {var_name!r}...' ) for chunk_index in chunk_indices: ok = _delete_block_file(input_path, var_name, chunk_index, dry_run, monitor) if ok: num_deleted += 1 monitor(f'Done, {num_deleted} block file(s) deleted.')
def vars2dim(cube, variable, dim_name, output=None, format=None): """ Convert cube variables into new dimension. Moves all variables of CUBE into into a single new variable <var-name> with a new dimension DIM-NAME and writes the results to OUTPUT. """ from xcube.core.dsio import guess_dataset_format from xcube.core.dsio import open_dataset, write_dataset from xcube.core.vars2dim import vars_to_dim import os if not output: dirname = os.path.dirname(cube) basename = os.path.basename(cube) basename, ext = os.path.splitext(basename) output = os.path.join(dirname, basename + '-vars2dim' + ext) format_name = format if format else guess_dataset_format(output) with open_dataset(input_path=cube) as ds: converted_dataset = vars_to_dim(ds, dim_name=dim_name, var_name=variable) write_dataset(converted_dataset, output_path=output, format_name=format_name)
def _rectify(input_path: str, xy_names: Optional[Tuple[str, str]], var_names: Optional[Sequence[str]], output_path: str, output_format: Optional[str], output_size: Optional[Tuple[int, int]], output_tile_size: Optional[Tuple[int, int]], output_point: Optional[Tuple[float, float]], output_res: Optional[float], delta: float, dry_run: bool, monitor): from xcube.core.dsio import guess_dataset_format from xcube.core.dsio import open_dataset from xcube.core.dsio import write_dataset from xcube.core.rectify import rectify_dataset from xcube.core.rectify import ImageGeom from xcube.core.sentinel3 import is_sentinel3_product from xcube.core.sentinel3 import open_sentinel3_product if not output_format: output_format = guess_dataset_format(output_path) output_geom = None if output_size is not None and output_point is not None and output_res is not None: output_geom = ImageGeom(size=output_size, x_min=output_point[0], y_min=output_point[1], xy_res=output_res) elif output_size is not None or output_point is not None or output_res is not None: raise click.ClickException( 'SIZE, POINT, and RES must all be given or none of them.') monitor(f'Opening dataset from {input_path!r}...') if is_sentinel3_product(input_path): src_ds = open_sentinel3_product(input_path) else: src_ds = open_dataset(input_path) monitor('Rectifying...') reproj_ds = rectify_dataset(src_ds, xy_names=xy_names, var_names=var_names, output_geom=output_geom, tile_size=output_tile_size, uv_delta=delta) if reproj_ds is None: monitor( f'Dataset {input_path} does not seem to have an intersection with bounding box' ) return monitor(f'Writing rectified dataset to {output_path!r}...') if not dry_run: write_dataset(reproj_ds, output_path, output_format) monitor(f'Done.')
def guess_ml_dataset_format(path: str) -> str: """ Guess a multilevel-dataset format for a file system path or URL given by *path*. :param path: A file system path or URL. :return: The name of a dataset format guessed from *path*. """ if path.endswith('.levels'): return FORMAT_NAME_LEVELS if path.endswith('.py'): return FORMAT_NAME_SCRIPT return guess_dataset_format(path)
def _resample_in_time(input_path: str = None, variables: Sequence[str] = None, metadata: Dict[str, Any] = None, output_path: str = DEFAULT_OUTPUT_PATH, output_format: str = None, methods: Sequence[str] = (DEFAULT_RESAMPLING_METHOD,), frequency: str = DEFAULT_RESAMPLING_FREQUENCY, offset: str = None, base: int = DEFAULT_RESAMPLING_BASE, interp_kind: str = DEFAULT_INTERPOLATION_KIND, tolerance: str = None, dry_run: bool = False, monitor=None): from xcube.core.dsio import guess_dataset_format from xcube.core.dsio import open_cube from xcube.core.dsio import write_cube from xcube.core.resample import resample_in_time from xcube.core.update import update_dataset_chunk_encoding if not output_format: output_format = guess_dataset_format(output_path) monitor(f'Opening cube from {input_path!r}...') with open_cube(input_path) as ds: monitor('Resampling...') agg_ds = resample_in_time(ds, frequency=frequency, method=methods, offset=offset, base=base, interp_kind=interp_kind, tolerance=tolerance, time_chunk_size=1, var_names=variables, metadata=metadata) agg_ds = update_dataset_chunk_encoding(agg_ds, chunk_sizes={}, format_name=output_format, in_place=True) monitor(f'Writing resampled cube to {output_path!r}...') if not dry_run: write_cube(agg_ds, output_path, output_format, cube_asserted=True) monitor(f'Done.')
def chunk(cube, output, format=None, params=None, chunks=None): """ (Re-)chunk xcube dataset. Changes the external chunking of all variables of CUBE according to CHUNKS and writes the result to OUTPUT. Note: There is a possibly more efficient way to (re-)chunk datasets through the dedicated tool "rechunker", see https://rechunker.readthedocs.io. """ chunk_sizes = None if chunks: chunk_sizes = parse_cli_kwargs(chunks, metavar="CHUNKS") for k, v in chunk_sizes.items(): if not isinstance(v, int) or v <= 0: raise click.ClickException( "Invalid value for CHUNKS, " f"chunk sizes must be positive integers: {chunks}") write_kwargs = dict() if params: write_kwargs = parse_cli_kwargs(params, metavar="PARAMS") from xcube.core.chunk import chunk_dataset from xcube.core.dsio import guess_dataset_format from xcube.core.dsio import open_dataset, write_dataset format_name = format if format else guess_dataset_format(output) with open_dataset(input_path=cube) as ds: if chunk_sizes: for k in chunk_sizes: if k not in ds.dims: raise click.ClickException( "Invalid value for CHUNKS, " f"{k!r} is not the name of any dimension: {chunks}") chunked_dataset = chunk_dataset(ds, chunk_sizes=chunk_sizes, format_name=format_name) write_dataset(chunked_dataset, output_path=output, format_name=format_name, **write_kwargs)
def _prune(input_path: str, dry_run: bool, monitor: Monitor): from xcube.core.chunk import get_empty_dataset_chunks from xcube.core.dsio import guess_dataset_format from xcube.core.dsio import open_dataset input_format = guess_dataset_format(input_path) if input_format != FORMAT_NAME_ZARR: raise click.ClickException("input must be a dataset in Zarr format") num_deleted_total = 0 monitor(f'Opening dataset from {input_path!r}...', 1) with open_dataset(input_path) as dataset: monitor('Identifying empty chunks...', 1) for var_name, chunk_indices in get_empty_dataset_chunks(dataset): num_empty_chunks = 0 num_deleted = 0 for chunk_index in chunk_indices: num_empty_chunks += 1 if num_empty_chunks == 1: monitor( f'Found empty chunks in variable {var_name!r}, ' f'deleting block files...', 2) ok = _delete_block_file(input_path, var_name, chunk_index, dry_run, monitor) if ok: num_deleted += 1 if num_deleted > 0: monitor( f'Deleted {num_deleted} block file(s) ' f'for variable {var_name!r}.', 2) elif num_empty_chunks > 0: monitor( f'No block files for variable {var_name!r} ' f'could be deleted.', 2) num_deleted_total += num_deleted monitor(f'Done, {num_deleted_total} block file(s) deleted total.', 1)
def gen_cube(input_paths: Sequence[str] = None, input_processor_name: str = None, input_processor_params: Dict = None, input_reader_name: str = None, input_reader_params: Dict[str, Any] = None, output_region: Tuple[float, float, float, float] = None, output_size: Tuple[int, int] = DEFAULT_OUTPUT_SIZE, output_resampling: str = DEFAULT_OUTPUT_RESAMPLING, output_path: str = DEFAULT_OUTPUT_PATH, output_writer_name: str = None, output_writer_params: Dict[str, Any] = None, output_metadata: NameAnyDict = None, output_variables: NameDictPairList = None, processed_variables: NameDictPairList = None, profile_mode: bool = False, no_sort_mode: bool = False, append_mode: bool = None, dry_run: bool = False, monitor: Callable[..., None] = None) -> bool: """ Generate a xcube dataset from one or more input files. :param no_sort_mode: :param input_paths: The input paths. :param input_processor_name: Name of a registered input processor (xcube.core.gen.inputprocessor.InputProcessor) to be used to transform the inputs. :param input_processor_params: Parameters to be passed to the input processor. :param input_reader_name: Name of a registered input reader (xcube.core.util.dsio.DatasetIO). :param input_reader_params: Parameters passed to the input reader. :param output_region: Output region as tuple of floats: (lon_min, lat_min, lon_max, lat_max). :param output_size: The spatial dimensions of the output as tuple of ints: (width, height). :param output_resampling: The resampling method for the output. :param output_path: The output directory. :param output_writer_name: Name of an output writer (xcube.core.util.dsio.DatasetIO) used to write the cube. :param output_writer_params: Parameters passed to the output writer. :param output_metadata: Extra metadata passed to output cube. :param output_variables: Output variables. :param processed_variables: Processed variables computed on-the-fly. :param profile_mode: Whether profiling should be enabled. :param append_mode: Deprecated. The function will always either insert, replace, or append new time slices. :param dry_run: Doesn't write any data. For testing. :param monitor: A progress monitor. :return: True for success. """ if append_mode is not None: warnings.warn( 'append_mode in gen_cube() is deprecated, ' 'time slices will now always be inserted, replaced, or appended.') if input_processor_name is None: input_processor_name = 'default' elif input_processor_name == '': raise ValueError('input_processor_name must not be empty') input_processor_class = find_input_processor_class(input_processor_name) if not input_processor_class: raise ValueError( f'Unknown input_processor_name {input_processor_name!r}') if not issubclass(input_processor_class, InputProcessor): raise ValueError( f'Invalid input_processor_name {input_processor_name!r}: ' f'must name a sub-class of {InputProcessor.__qualname__}') try: input_processor = input_processor_class( **(input_processor_params or {})) except (ValueError, TypeError) as e: raise ValueError( f'Invalid input_processor_name or input_processor_params: {e}' ) from e input_reader = find_dataset_io(input_reader_name or input_processor.input_reader) if not input_reader: raise ValueError(f'Unknown input_reader_name {input_reader_name!r}') if not output_path: raise ValueError('Missing output_path') output_writer_name = output_writer_name or guess_dataset_format( output_path) if not output_writer_name: raise ValueError( f'Failed to guess output_writer_name from path {output_path}') output_writer = find_dataset_io(output_writer_name, modes={'w', 'a'}) if not output_writer: raise ValueError(f'Unknown output_writer_name {output_writer_name!r}') if monitor is None: # noinspection PyUnusedLocal def monitor(*args): pass input_paths = [ input_file for f in input_paths for input_file in glob.glob(f, recursive=True) ] if not no_sort_mode and len(input_paths) > 1: input_paths = _get_sorted_input_paths(input_processor, input_paths) if not dry_run: output_dir = os.path.abspath(os.path.dirname(output_path)) os.makedirs(output_dir, exist_ok=True) effective_input_reader_params = dict(input_processor.input_reader_params or {}) effective_input_reader_params.update(input_reader_params or {}) effective_output_writer_params = output_writer_params or {} status = False ds_count = len(input_paths) ds_count_ok = 0 ds_index = 0 for input_file in input_paths: monitor( f'processing dataset {ds_index + 1} of {ds_count}: {input_file!r}...' ) # noinspection PyTypeChecker status = _process_input(input_processor, input_reader, effective_input_reader_params, output_writer, effective_output_writer_params, input_file, output_size, output_region, output_resampling, output_path, output_metadata, output_variables, processed_variables, profile_mode, dry_run, monitor) ds_index += 1 if status: ds_count_ok += 1 monitor(f'{ds_count_ok} of {ds_count} datasets processed successfully, ' f'{ds_count - ds_count_ok} were dropped due to errors') return status
def guess_cube_format(path: str) -> str: if path.endswith('.levels'): return FORMAT_NAME_LEVELS return guess_dataset_format(path)
def _rectify(input_path: str, xy_names: Optional[Tuple[str, str]], var_names: Optional[Sequence[str]], output_path: str, output_format: Optional[str], output_size: Optional[Tuple[int, int]], output_tile_size: Optional[Tuple[int, int]], output_point: Optional[Tuple[float, float]], output_res: Optional[float], output_crs: Optional[str], delta: float, dry_run: bool, monitor): import pyproj.crs from xcube.core.dsio import guess_dataset_format from xcube.core.dsio import open_dataset from xcube.core.dsio import write_dataset from xcube.core.gridmapping import GridMapping from xcube.core.resampling import rectify_dataset from xcube.core.sentinel3 import is_sentinel3_product from xcube.core.sentinel3 import open_sentinel3_product if not output_format: output_format = guess_dataset_format(output_path) output_gm = None output_gm_given = (output_size is not None, output_point is not None, output_res is not None, output_crs is not None) if all(output_gm_given): output_gm = GridMapping.regular(size=output_size, xy_min=output_point, xy_res=output_res, crs=pyproj.crs.CRS.from_user_input(output_crs)) elif any(output_gm_given): raise click.ClickException('SIZE, POINT, RES, and CRS must all be given or none of them.') monitor(f'Opening dataset from {input_path!r}...') if is_sentinel3_product(input_path): src_ds = open_sentinel3_product(input_path) else: src_ds = open_dataset(input_path) monitor('Rectifying...') rectified_ds = rectify_dataset(src_ds, xy_var_names=xy_names, var_names=var_names, target_gm=output_gm, tile_size=output_tile_size, uv_delta=delta) if rectified_ds is None: monitor(f'Dataset {input_path} does not seem to have an intersection with bounding box') return monitor(f'Writing rectified dataset to {output_path!r}...') if not dry_run: write_dataset(rectified_ds, output_path, output_format) monitor(f'Done.')
def compute(script: str, cube: List[str], input_var_names: str, input_params: str, output_path: str, output_format: str, output_var_name: str, output_var_dtype: str): """ Compute a cube from one or more other cubes. The command computes a cube variable from other cube variables in CUBEs using a user-provided Python function in SCRIPT. The SCRIPT must define a function named "compute": \b def compute(*input_vars: numpy.ndarray, input_params: Mapping[str, Any] = None, dim_coords: Mapping[str, np.ndarray] = None, dim_ranges: Mapping[str, Tuple[int, int]] = None) \\ -> numpy.ndarray: # Compute new numpy array from inputs # output_array = ... return output_array where input_vars are numpy arrays (chunks) in the order given by VARIABLES or given by the variable names returned by an optional "initialize" function that my be defined in SCRIPT too, see below. input_params is a mapping of parameter names to values according to PARAMS or the ones returned by the aforesaid "initialize" function. dim_coords is a mapping from dimension name to coordinate labels for the current chunk to be computed. dim_ranges is a mapping from dimension name to index ranges into coordinate arrays of the cube. The SCRIPT may define a function named "initialize": \b def initialize(input_cubes: Sequence[xr.Dataset], input_var_names: Sequence[str], input_params: Mapping[str, Any]) \\ -> Tuple[Sequence[str], Mapping[str, Any]]: # Compute new variable names and/or new parameters # new_input_var_names = ... # new_input_params = ... return new_input_var_names, new_input_params where input_cubes are the respective CUBEs, input_var_names the respective VARIABLES, and input_params are the respective PARAMS. The "initialize" function can be used to validate the data cubes, extract the desired variables in desired order and to provide some extra processing parameters passed to the "compute" function. Note that if no input variable names are specified, no variables are passed to the "compute" function. The SCRIPT may also define a function named "finalize": \b def finalize(output_cube: xr.Dataset, input_params: Mapping[str, Any]) \\ -> Optional[xr.Dataset]: # Optionally modify output_cube and return it or return None return output_cube If defined, the "finalize" function will be called before the command writes the new cube and then exists. The functions may perform a cleaning up or perform side effects such as write the cube to some sink. If the functions returns None, the CLI will *not* write any cube data. """ from xcube.cli.common import parse_cli_kwargs from xcube.core.compute import compute_cube from xcube.core.dsio import open_cube from xcube.core.dsio import guess_dataset_format, find_dataset_io input_paths = cube compute_function_name = "compute" initialize_function_name = "initialize" finalize_function_name = "finalize" with open(script, "r") as fp: code = fp.read() locals_dict = dict() exec(code, globals(), locals_dict) input_var_names = list(map(lambda s: s.strip(), input_var_names.split(","))) if input_var_names else None compute_function = _get_function(locals_dict, compute_function_name, script, force=True) initialize_function = _get_function(locals_dict, initialize_function_name, script, force=False) finalize_function = _get_function(locals_dict, finalize_function_name, script, force=False) input_params = parse_cli_kwargs(input_params, "PARAMS") input_cubes = [] for input_path in input_paths: input_cubes.append(open_cube(input_path=input_path)) if initialize_function: input_var_names, input_params = initialize_function(input_cubes, input_var_names, input_params) output_cube = compute_cube(compute_function, *input_cubes, input_var_names=input_var_names, input_params=input_params, output_var_name=output_var_name, output_var_dtype=output_var_dtype) if finalize_function: output_cube = finalize_function(output_cube) if output_cube is not None: output_format = output_format or guess_dataset_format(output_path) dataset_io = find_dataset_io(output_format, {"w"}) dataset_io.write(output_cube, output_path)