def append_to_cube_from_dataframe(
data: db.Bag,
cube: Cube,
store: KeyValueStore,
metadata: Optional[Dict[str, Dict[str, Any]]] = None,
df_serializer: Optional[ParquetSerializer] = None,
) -> db.Bag:
"""
Append data to existing cube.
For details on ``data`` and ``metadata``, see :func:`~kartothek.io.eager_cube.build_cube`.
.. important::
Physical partitions must be updated as a whole. If only single rows within a physical partition are updated, the
old data is treated as "removed".
.. hint::
To have better control over the overwrite "mask" (i.e. which partitions are overwritten), you should use
:func:`~kartothek.io.eager_cube.remove_partitions` beforehand.
Parameters
----------
data: dask.bag.Bag
Bag containing dataframes
cube:
Cube specification.
store:
Store to which the data should be written to.
metadata:
Metadata for every dataset, optional. For every dataset, only given keys are updated/replaced. Deletion of
metadata keys is not possible.
df_serializer:
Optional Dataframe to Parquet serializer
Returns
-------
metadata_dict: dask.bag.Bag
A dask bag object containing the compute graph to append to the cube returning the dict of dataset metadata
objects. The bag has a single partition with a single element.
"""
data, ktk_cube_dataset_ids = _ddfs_to_bag(data, cube)
return (
append_to_cube_from_bag_internal(
data=data,
cube=cube,
store=store,
ktk_cube_dataset_ids=ktk_cube_dataset_ids,
metadata=metadata,
df_serializer=df_serializer,
)
.map_partitions(_unpack_list, default=None)
.to_delayed()[0]
)
def update_cube_from_bag(
data: db.Bag,
cube: Cube,
store: StoreFactory,
remove_conditions,
ktk_cube_dataset_ids: Optional[Iterable[str]],
metadata: Optional[Dict[str, Dict[str, Any]]] = None,
df_serializer: Optional[ParquetSerializer] = None,
) -> db.Bag:
"""
Remove partitions and append data to existing cube.
For details on ``data`` and ``metadata``, see :func:`~kartothek.io.eager_cube.build_cube`.
Only datasets in `ktk_cube_dataset_ids` will be affected.
Parameters
----------
data: dask.bag.Bag
Bag containing dataframes
cube:
Cube specification.
store:
Store to which the data should be written to.
remove_conditions
Conditions that select the partitions to remove. Must be a condition that only uses
partition columns.
ktk_cube_dataset_ids:
Datasets that will be written, must be specified in advance.
metadata:
Metadata for every dataset, optional. For every dataset, only given keys are updated/replaced. Deletion of
metadata keys is not possible.
df_serializer:
Optional Dataframe to Parquet serializer
Returns
-------
metadata_dict: dask.bag.Bag
A dask bag object containing the compute graph to append to the cube returning the dict of dataset metadata
objects. The bag has a single partition with a single element.
See Also
--------
:ref:`mutating_datasets`
"""
return append_to_cube_from_bag_internal(
data=data,
cube=cube,
store=store,
remove_conditions=remove_conditions,
ktk_cube_dataset_ids=ktk_cube_dataset_ids,
metadata=metadata,
df_serializer=df_serializer,
)
def append_to_cube_from_bag(data,
cube,
store,
ktk_cube_dataset_ids,
metadata=None):
"""
Append data to existing cube.
For details on ``data`` and ``metadata``, see :meth:`build_cube`.
.. important::
Physical partitions must be updated as a whole. If only single rows within a physical partition are updated, the
old data is treated as "removed".
.. hint::
To have better control over the overwrite "mask" (i.e. which partitions are overwritten), you should use
:meth:`remove_partitions` beforehand or use :meth:`update_cube_from_bag` instead.
Parameters
----------
data: dask.Bag
Bag containing dataframes
cube: kartothek.core.cube.cube.Cube
Cube specification.
store: Callable[[], simplekv.KeyValueStore]
Store to which the data should be written to.
ktk_cube_dataset_ids: Optional[Iterable[str]]
Datasets that will be written, must be specified in advance.
metadata: Optional[Dict[str, Dict[str, Any]]]
Metadata for every dataset, optional. For every dataset, only given keys are updated/replaced. Deletion of
metadata keys is not possible.
Returns
-------
metadata_dict: dask.bag.Bag
A dask bag object containing the compute graph to append to the cube returning the dict of dataset metadata
objects. The bag has a single partition with a single element.
"""
return append_to_cube_from_bag_internal(
data=data,
cube=cube,
store=store,
ktk_cube_dataset_ids=ktk_cube_dataset_ids,
metadata=metadata,
)
def update_cube_from_bag(data,
cube,
store,
remove_conditions,
ktk_cube_dataset_ids,
metadata=None):
"""
Remove partitions and append data to existing cube.
For details on ``data`` and ``metadata``, see :meth:`build_cube`.
Only datasets in `ktk_cube_dataset_ids` will be affected.
Parameters
----------
data: dask.Bag
Bag containing dataframes
cube: kartothek.core.cube.cube.Cube
Cube specification.
store: Callable[[], simplekv.KeyValueStore]
Store to which the data should be written to.
remove_conditions
Conditions that select the partitions to remove. Must be a condition that only uses
partition columns.
ktk_cube_dataset_ids: Optional[Iterable[str]]
Datasets that will be written, must be specified in advance.
metadata: Optional[Dict[str, Dict[str, Any]]]
Metadata for every dataset, optional. For every dataset, only given keys are updated/replaced. Deletion of
metadata keys is not possible.
Returns
-------
metadata_dict: dask.bag.Bag
A dask bag object containing the compute graph to append to the cube returning the dict of dataset metadata
objects. The bag has a single partition with a single element.
"""
return append_to_cube_from_bag_internal(
data=data,
cube=cube,
store=store,
remove_conditions=remove_conditions,
ktk_cube_dataset_ids=ktk_cube_dataset_ids,
metadata=metadata,
)