def _create_catalog(self, conf_catalog, conf_creds) -> DataCatalog:
save_version = generate_current_version()
run_id = pai.current_run_uuid()
if run_id:
save_version += "-" + run_id
return DataCatalog.from_config(conf_catalog,
conf_creds,
save_version=save_version)
def test_from_sane_config_versioned(self, sane_config, dummy_dataframe):
"""Test load and save of versioned data sets from config"""
sane_config["catalog"]["boats"]["versioned"] = True
version = generate_current_version()
catalog = DataCatalog.from_config(**sane_config,
load_versions={"boats": version},
save_version=version)
catalog.save("boats", dummy_dataframe)
path = Path(sane_config["catalog"]["boats"]["filepath"])
path = path / version / path.name
assert path.is_file()
reloaded_df = catalog.load("boats")
assert_frame_equal(reloaded_df, dummy_dataframe)
def test_from_sane_config_versioned(self, sane_config, dummy_dataframe):
"""Test load and save of versioned data sets from config"""
sane_config["catalog"]["boats"]["versioned"] = True
version = generate_current_version()
catalog = DataCatalog.from_config(**sane_config,
load_versions={"boats": version},
save_version=version)
journal = VersionJournal({"project_path": "fake-path"})
catalog.set_version_journal(journal)
assert catalog._journal == journal # pylint: disable=protected-access
catalog.save("boats", dummy_dataframe)
path = Path(sane_config["catalog"]["boats"]["filepath"])
path = path / version / path.name
assert path.is_file()
reloaded_df = catalog.load("boats")
assert_frame_equal(reloaded_df, dummy_dataframe)
def save_version(request):
return request.param or generate_current_version()
def from_config(
cls: Type,
catalog: Optional[Dict[str, Dict[str, Any]]],
credentials: Dict[str, Dict[str, Any]] = None,
load_versions: Dict[str, str] = None,
save_version: str = None,
) -> "DataCatalog":
"""Create a ``DataCatalog`` instance from configuration. This is a
factory method used to provide developers with a way to instantiate
``DataCatalog`` with configuration parsed from configuration files.
Args:
catalog: A dictionary whose keys are the data set names and
the values are dictionaries with the constructor arguments
for classes implementing ``AbstractDataSet``. The data set
class to be loaded is specified with the key ``type`` and their
fully qualified class name. All ``kedro.io`` data set can be
specified by their class name only, i.e. their module name
can be omitted.
credentials: A dictionary containing credentials for different
data sets. Use the ``credentials`` key in a ``AbstractDataSet``
to refer to the appropriate credentials as shown in the example
below.
load_versions: A mapping between dataset names and versions
to load. Has no effect on data sets without enabled versioning.
save_version: Version string to be used for ``save`` operations
by all data sets with enabled versioning. It must: a) be a
case-insensitive string that conforms with operating system
filename limitations, b) always return the latest version when
sorted in lexicographical order.
Returns:
An instantiated ``DataCatalog`` containing all specified
data sets, created and ready to use.
Raises:
DataSetError: When the method fails to create any of the data
sets from their config.
Example:
::
>>> config = {
>>> "cars": {
>>> "type": "CSVLocalDataSet",
>>> "filepath": "cars.csv",
>>> "save_args": {
>>> "index": False
>>> }
>>> },
>>> "boats": {
>>> "type": "CSVS3DataSet",
>>> "filepath": "boats.csv",
>>> "bucket_name": "mck-147789798-bucket",
>>> "credentials": "boats_credentials"
>>> "save_args": {
>>> "index": False
>>> }
>>> }
>>> }
>>>
>>> credentials = {
>>> "boats_credentials": {
>>> "aws_access_key_id": "<your key id>",
>>> "aws_secret_access_key": "<your secret>"
>>> }
>>> }
>>>
>>> catalog = DataCatalog.from_config(config, credentials)
>>>
>>> df = catalog.load("cars")
>>> catalog.save("boats", df)
"""
data_sets = {}
catalog = copy.deepcopy(catalog) or {}
credentials = copy.deepcopy(credentials) or {}
save_version = save_version or generate_current_version()
load_versions = copy.deepcopy(load_versions) or {}
for ds_name, ds_config in catalog.items():
if "type" not in ds_config:
raise DataSetError("`type` is missing from DataSet '{}' "
"catalog configuration".format(ds_name))
if CREDENTIALS_KEY in ds_config:
ds_config[CREDENTIALS_KEY] = _get_credentials(
ds_config.pop(CREDENTIALS_KEY),
credentials # credentials name
)
data_sets[ds_name] = AbstractDataSet.from_config(
ds_name, ds_config, load_versions.get(ds_name), save_version)
return cls(data_sets=data_sets)