def _load(self) -> Dict[str, Dataset]:
if "metadata" in self.channels:
path = self.channels.pop("metadata")
self.hyperparameters["freq"] = MetaData.parse_file(
path / "metadata.json").freq
file_dataset = partial(FileDataset, freq=self.hyperparameters["freq"])
list_dataset = partial(ListDataset, freq=self.hyperparameters["freq"])
datasets = map_dct_values(file_dataset, self.channels)
if self._listify_dataset():
datasets = map_dct_values(list_dataset, datasets)
return datasets
def _load_inputdataconfig(
inputdataconfig: Path) -> Optional[Dict[str, DataConfig]]:
if inputdataconfig.exists():
with inputdataconfig.open() as json_file:
return map_dct_values(DataConfig.parse_obj, json.load(json_file))
return None
def encode_sagemaker_parameters(decoded_params: dict) -> dict:
"""Encode a SageMaker parameters dictionary where all values are strings.
Example:
>>> encode_sagemaker_parameters({
... "foo": [1, 2, 3],
... "bar": "hello"
... })
{'foo': '[1, 2, 3]', 'bar': 'hello'}
"""
return map_dct_values(encode_sagemaker_parameter, decoded_params)
def detrim_and_decode_sagemaker_parameters(trimmed_params: dict) -> dict:
"""Decode a SageMaker parameters dictionary where all values are strings.
Example:
>>> detrim_and_decode_sagemaker_parameters({
... '_0_foo': '[1, ',
... '_1_foo': '2, 3',
... '_2_foo': ']',
... '_0_bar': 'hell',
... '_1_bar': 'o'
... })
{'foo': [1, 2, 3], 'bar': 'hello'}
"""
encoded_params = detrim_sagemaker_parameters(trimmed_params)
return map_dct_values(decode_sagemaker_parameter, encoded_params)
def encode_and_trim_sagemaker_parameters(decoded_params: dict,
max_len: int = 256) -> dict:
"""Encode a SageMaker parameters dictionary where all values are strings then
trim them to account for Sagemaker character size limit.
>>> encode_and_trim_sagemaker_parameters({
... "foo": [1, 2, 3],
... "bar": "hello"
... }, max_len = 4)
{'_0_foo': '[1, ',
'_1_foo': '2, 3',
'_2_foo': ']',
'_0_bar': 'hell',
'_1_bar': 'o'}
"""
endoded_params = map_dct_values(encode_sagemaker_parameter, decoded_params)
return trim_encoded_sagemaker_parameters(endoded_params, max_len)
