def cleanup_cube_bag(cube, store, blocksize=100): """ Remove unused keys from cube datasets. .. important:: All untracked keys which start with the cube's `uuid_prefix` followed by the `KTK_CUBE_UUID_SEPERATOR` (e.g. `my_cube_uuid++seed...`) will be deleted by this routine. These keys may be leftovers from past overwrites or index updates. Parameters ---------- cube: Cube Cube specification. store: Union[simplekv.KeyValueStore, Callable[[], simplekv.KeyValueStore]] KV store. blocksize: int Number of keys to delete at once. Returns ------- bag: dask.bag.Bag A dask bag that performs the given operation. May contain multiple partitions. """ check_store_factory(store) check_blocksize(blocksize) store_obj = store() datasets = discover_datasets_unchecked(uuid_prefix=cube.uuid_prefix, store=store) keys = get_keys_to_clean(cube.uuid_prefix, datasets, store_obj) return db.from_sequence( seq=sorted(keys), partition_size=blocksize).map_partitions(_delete, store=store)
def copy_cube_bag( cube, src_store: StoreFactory, tgt_store: StoreFactory, blocksize: int = 100, overwrite: bool = False, datasets: Optional[Union[Iterable[str], Dict[str, DatasetMetadata]]] = None, ): """ Copy cube from one store to another. Parameters ---------- cube Cube specification. src_store Source KV store. tgt_store Target KV store. overwrite If possibly existing datasets in the target store should be overwritten. blocksize Number of keys to copy at once. datasets Datasets to copy, must all be part of the cube. May be either the result of :func:`~kartothek.api.discover.discover_datasets`, a list of Ktk_cube dataset ID or ``None`` (in which case entire cube will be copied). Returns ------- bag: dask.bag.Bag A dask bag that performs the given operation. May contain multiple partitions. """ check_store_factory(src_store) check_store_factory(tgt_store) check_blocksize(blocksize) assert_stores_different( src_store, tgt_store, cube.ktk_dataset_uuid(cube.seed_dataset) ) keys = get_copy_keys( cube=cube, src_store=src_store, tgt_store=tgt_store, overwrite=overwrite, datasets=datasets, ) return db.from_sequence(seq=sorted(keys), partition_size=blocksize).map_partitions( copy_keys, src_store=src_store, tgt_store=tgt_store )
def collect_stats_bag( cube: Cube, store: StoreFactory, datasets: Optional[Union[Iterable[str], Dict[str, DatasetMetadata]]] = None, blocksize: int = 100, ): """ Collect statistics for given cube. Parameters ---------- cube Cube specification. store KV store that preserves the cube. datasets Datasets to query, must all be part of the cube. May be either the result of :func:`~kartothek.api.discover.discover_datasets`, a list of Ktk_cube dataset ID or ``None`` (in which case auto-discovery will be used). blocksize Number of partitions to scan at once. Returns ------- bag: dask.bag.Bag A dask bag that returns a single result of the form ``Dict[str, Dict[str, int]]`` and contains statistics per ktk_cube dataset ID. """ check_store_factory(store) check_blocksize(blocksize) if not isinstance(datasets, dict): datasets = discover_datasets_unchecked( uuid_prefix=cube.uuid_prefix, store=store, filter_ktk_cube_dataset_ids=datasets, ) all_metapartitions = get_metapartitions_for_stats(datasets) return ( db.from_sequence(seq=all_metapartitions, partition_size=blocksize) .map_partitions(collect_stats_block, store=store) .reduction( perpartition=_obj_to_list, aggregate=_reduce_stats, split_every=False, out_type=db.Bag, ) )
def delete_cube_bag( cube: Cube, store: StoreFactory, blocksize: int = 100, datasets: Optional[Union[Iterable[str], Dict[str, DatasetMetadata]]] = None, ): """ Delete cube from store. .. important:: This routine only deletes tracked files. Garbage and leftovers from old cubes and failed operations are NOT removed. Parameters ---------- cube Cube specification. store KV store. blocksize Number of keys to delete at once. datasets Datasets to delete, must all be part of the cube. May be either the result of :func:`~kartothek.api.discover.discover_datasets`, a list of Ktk_cube dataset ID or ``None`` (in which case entire cube will be deleted). Returns ------- bag: dask.bag.Bag A dask bag that performs the given operation. May contain multiple partitions. """ check_store_factory(store) check_blocksize(blocksize) if not isinstance(datasets, dict): datasets = discover_datasets_unchecked( uuid_prefix=cube.uuid_prefix, store=store, filter_ktk_cube_dataset_ids=datasets, ) keys = set() for ktk_cube_dataset_id in sorted(datasets.keys()): ds = datasets[ktk_cube_dataset_id] keys |= get_dataset_keys(ds) return db.from_sequence(seq=sorted(keys), partition_size=blocksize).map_partitions( _delete, store=store )
def query_cube_bag_internal( cube, store, conditions, datasets, dimension_columns, partition_by, payload_columns, blocksize, ): """ Query cube. For detailed documentation, see :func:`~kartothek.io.eager_cube.query_cube`. Parameters ---------- cube: Cube Cube specification. store: simplekv.KeyValueStore KV store that preserves the cube. conditions: Union[None, Condition, Iterable[Condition], Conjunction] Conditions that should be applied, optional. datasets: Union[None, Iterable[str], Dict[str, kartothek.core.dataset.DatasetMetadata]] Datasets to query, must all be part of the cube. May be either the result of :func:`~kartothek.api.discover.discover_datasets`, a list of Ktk_cube dataset ID or ``None`` (in which case auto-discovery will be used). dimension_columns: Union[None, str, Iterable[str]] Dimension columns of the query, may result in projection. If not provided, dimension columns from cube specification will be used. partition_by: Union[None, str, Iterable[str]] By which column logical partitions should be formed. If not provided, a single partition will be generated. payload_columns: Union[None, str, Iterable[str]] Which columns apart from ``dimension_columns`` and ``partition_by`` should be returned. blocksize: int Partition size of the bag. Returns ------- empty: pandas.DataFrame Empty DataFrame with correct dtypes and column order. bag: dask.bag.Bag Bag of 1-sized partitions of non-empty DataFrames, order by ``partition_by``. Column of DataFrames is alphabetically ordered. Data types are provided on best effort (they are restored based on the preserved data, but may be different due to Pandas NULL-handling, e.g. integer columns may be floats). """ check_store_factory(store) check_blocksize(blocksize) intention, empty, groups = plan_query( cube=cube, store=store, conditions=conditions, datasets=datasets, dimension_columns=dimension_columns, partition_by=partition_by, payload_columns=payload_columns, ) b = (db.from_sequence(seq=groups, partition_size=blocksize).map( load_group, store=store, cube=cube).filter(_not_empty)) if not intention.partition_by: b = (b.reduction( perpartition=list, aggregate=_collect_dfs, split_every=False, out_type=db.Bag, ).map( _quick_concat_or_none, dimension_columns=intention.dimension_columns, partition_columns=cube.partition_columns, ).filter(_not_none)) return empty, b