def main(gen_config_path: str, store_configs_path: str = None, verbose: bool = False): """ Generator tool for data cubes. Creates cube views from one or more cube stores, resamples them to a common grid, optionally performs some cube transformation, and writes the resulting cube to some target cube store. *gen_config_path* is the cube generator configuration. It may be provided as a JSON or YAML file (file extensions ".json" or ".yaml"). If the *gen_config_path* argument is omitted, it is expected that the cube generator configuration is piped as a JSON string. *store_configs_path* is a path to a JSON file with data store configurations. It is a mapping of names to configured stores. Entries are dictionaries that have a mandatory "store_id" property which is a name of a registered xcube data store. The optional "store_params" property may define data store specific parameters. :param gen_config_path: Cube generation configuration. It may be provided as a JSON or YAML file (file extensions ".json" or ".yaml"). If the REQUEST file argument is omitted, it is expected that the cube generator configuration is piped as a JSON string. :param store_configs_path: A JSON file that maps store names to parameterized stores. :param verbose: Whether to output progress information to stdout. """ store_pool = DataStorePool.from_file( store_configs_path) if store_configs_path else DataStorePool() gen_config = GenConfig.from_file(gen_config_path, verbose=verbose) if gen_config.callback_config: ApiProgressCallbackObserver(gen_config.callback_config).activate() if verbose: ConsoleProgressObserver().activate() with observe_progress('Generating cube', 100) as cm: cm.will_work(10) cubes = open_cubes(gen_config.input_configs, cube_config=gen_config.cube_config, store_pool=store_pool) cm.will_work(10) cube = resample_and_merge_cubes(cubes, cube_config=gen_config.cube_config) cm.will_work(80) data_id = write_cube(cube, output_config=gen_config.output_config, store_pool=store_pool) if verbose: print('Cube "{}" generated within {:.2f} seconds'.format( str(data_id), cm.state.total_time))
def test_from_yaml_file(self): store_configs = { "ram-1": { "store_id": "memory" }, "ram-2": { "store_id": "memory" } } path = 'test-store-configs.yaml' with open(path, 'w') as fp: yaml.dump(store_configs, fp, indent=2) try: pool = DataStorePool.from_file(path) self.assertIsInstance(pool, DataStorePool) self.assertEqual(['ram-1', 'ram-2'], pool.store_instance_ids) finally: import os os.remove(path)
def _assert_file_ok(self, format_name: str, root_1="/root1", root_2="/root2", use_env_vars=False): if use_env_vars: store_configs = self._get_test_config( root_1='${_TEST_ROOT_1}', root_2='${_TEST_ROOT_2}' ) import os os.environ['_TEST_ROOT_1'] = root_1 os.environ['_TEST_ROOT_2'] = root_2 else: store_configs = self._get_test_config( root_1=root_1, root_2=root_2 ) path = 'test-store-configs.' + format_name with open(path, 'w') as fp: mod = yaml if format_name == 'yaml' else json mod.dump(store_configs, fp, indent=2) try: pool = DataStorePool.from_file(path) self.assertIsInstance(pool, DataStorePool) self.assertEqual(['ram-1', 'ram-2'], pool.store_instance_ids) config_1 = pool.get_store_config('ram-1') self.assertIsInstance(config_1, DataStoreConfig) self.assertEqual( {'store_id': 'memory', 'store_params': {'root': root_1}}, config_1.to_dict()) config_2 = pool.get_store_config('ram-2') self.assertIsInstance(config_2, DataStoreConfig) self.assertEqual( {'store_id': 'memory', 'store_params': {'root': root_2}}, config_2.to_dict()) finally: import os os.remove(path)
def dump(output_file_path: Optional[str], config_file_path: Optional[str], data_type: Optional[str], short_form: bool, include_props: str, exclude_props: str, csv_format: bool, yaml_format: bool, json_format: bool): """ Dump metadata of given data stores. Dumps data store metadata and metadata for a store's data resources for given data stores into a JSON file. Data stores may be selected and configured by a configuration file CONFIG, which may have JSON or YAML format. For example, this YAML configuration configures a single directory data store: \b this_dir: title: Current Dir description: A store that represents my current directory store_id: "directory" store_params: base_dir: "." """ from xcube.core.store import DataStoreConfig from xcube.core.store import DataStorePool import yaml import json import os.path if csv_format: output_format = 'csv' ext = '.csv' elif yaml_format: output_format = 'yaml' ext = '.yml' elif json_format: output_format = 'json' ext = '.json' elif output_file_path is not None: path_no_ext, ext = os.path.splitext(output_file_path) if ext in ('.csv', '.txt'): output_format = 'csv' elif ext in ('.yaml', '.yml'): output_format = 'yaml' else: output_format = 'json' else: output_format = 'json' ext = '.json' if output_file_path is None: path_no_ext, _ = os.path.splitext(_DEFAULT_DUMP_OUTPUT) output_file_path = path_no_ext + ext include_props = _parse_props(include_props) if include_props else None exclude_props = _parse_props(exclude_props) if exclude_props else None if short_form: short_include_props = _parse_props(_SHORT_INCLUDE) include_props = include_props or {} for data_key in ('store', 'data', 'var'): include_props[data_key] = include_props.get(data_key, set()).union( short_include_props[data_key]) if config_file_path: store_pool = DataStorePool.from_file(config_file_path) else: extensions = get_extension_registry().find_extensions( EXTENSION_POINT_DATA_STORES) store_configs = { extension.name: DataStoreConfig(extension.name, title=extension.metadata.get('title'), description=extension.metadata.get('description')) for extension in extensions if extension.name not in ('memory', 'directory', 's3') } store_pool = DataStorePool(store_configs) dump_data = _get_store_data_var_tuples(store_pool, data_type, include_props, exclude_props) if output_format == 'csv': column_names = None column_names_set = None rows = [] for store_dict, data_dict, var_dict in dump_data: if store_dict is None: break row = {} row.update({'store.' + k: v for k, v in store_dict.items()}) row.update({'data.' + k: v for k, v in data_dict.items()}) row.update({'var.' + k: v for k, v in var_dict.items()}) rows.append(row) if column_names_set is None: column_names = list(row.keys()) column_names_set = set(column_names) else: for k in row.keys(): if k not in column_names_set: column_names.append(k) column_names_set.add(k) def format_cell_value(value: Any) -> str: return str(value) if value is not None else '' sep = '\t' with open(output_file_path, 'w') as fp: if column_names: fp.write(sep.join(column_names) + '\n') for row in rows: fp.write( sep.join( map(format_cell_value, tuple(row.get(k) for k in column_names))) + '\n') print(f'Dumped {len(rows)} store entry/ies to {output_file_path}.') else: last_store_dict = None last_data_dict = None vars_list = [] data_list = [] store_list = [] for store_dict, data_dict, var_dict in dump_data: if data_dict is not last_data_dict or data_dict is None: if last_data_dict is not None: last_data_dict['data_vars'] = vars_list vars_list = [] data_list.append(last_data_dict) last_data_dict = data_dict if store_dict is not last_store_dict or store_dict is None: if last_store_dict is not None: last_store_dict['data'] = data_list data_list = [] store_list.append(last_store_dict) last_store_dict = store_dict if var_dict: vars_list.append(var_dict) with open(output_file_path, 'w') as fp: if output_format == 'json': json.dump(dict(stores=store_list), fp, indent=2) else: yaml.dump(dict(stores=store_list), fp, indent=2) print( f'Dumped entries of {len(store_list)} store(s) to {output_file_path}.' )
def dump(output_file_path: str, config_file_path: Optional[str], type_specifier: Optional[str]): """ Dump metadata of given data stores. Dumps data store metadata and metadata for a store's data resources for given data stores into a JSON file. Data stores may be selected and configured by a configuration file CONFIG, which may have JSON or YAML format. For example, this YAML configuration configures a single directory data store: \b this_dir: title: Current Dir description: A store that represents my current directory store_id: "directory" store_params: base_dir: "." """ from xcube.core.store import DataStoreConfig from xcube.core.store import DataStorePool import time if config_file_path: store_pool = DataStorePool.from_file(config_file_path) else: extensions = get_extension_registry().find_extensions( EXTENSION_POINT_DATA_STORES) store_configs = { extension.name: DataStoreConfig(extension.name, title=extension.metadata.get('title'), description=extension.metadata.get('description')) for extension in extensions if extension.name not in ('memory', 'directory', 's3') } store_pool = DataStorePool(store_configs) stores = [] for store_instance_id in store_pool.store_instance_ids: t0 = time.perf_counter() print(f'Generating entries for store "{store_instance_id}"...') try: store_instance = store_pool.get_store(store_instance_id) except BaseException as error: print(f'error: cannot open store "{store_instance_id}": {error}', file=sys.stderr) continue try: search_result = [ dsd.to_dict() for dsd in store_instance.search_data( type_specifier=type_specifier) ] except BaseException as error: print(f'error: cannot search store "{store_instance_id}": {error}', file=sys.stderr) continue store_config = store_pool.get_store_config(store_instance_id) stores.append( dict(store_instance_id=store_instance_id, store_id=store_instance_id, title=store_config.title, description=store_config.description, type_specifier=type_specifier, datasets=search_result)) print('Done after {:.2f} seconds'.format(time.perf_counter() - t0)) with open(output_file_path, 'w') as fp: json.dump(dict(stores=stores), fp, indent=2) print(f'Dumped {len(stores)} store(s) to {output_file_path}.')