def _transform_config(self, s3_config): """Splits the general s3 config into 2 different config objects, one for transfer.TransferConfig and other is the general session config""" config, transfer_config = {}, {} for key, value in s3_config.items(): if key in self._TRANSFER_CONFIG_ALIASES: if key in {"multipart_chunksize", "multipart_threshold"}: # cast human readable sizes (like 24MiB) to integers value = conversions.human_readable_to_bytes(value) else: value = int(value) transfer_config[self._TRANSFER_CONFIG_ALIASES[key]] = value else: config[key] = value return config, transfer_config
def _split_s3_config(self, s3_config): """Splits the general s3 config into 2 different config objects, one for transfer.TransferConfig and other is the general session config""" from boto3.s3.transfer import TransferConfig from dvc.utils import conversions config, transfer_config = {}, {} for key, value in s3_config.items(): if key in self._TRANSFER_CONFIG_ALIASES: if key in {"multipart_chunksize", "multipart_threshold"}: # cast human readable sizes (like 24MiB) to integers value = conversions.human_readable_to_bytes(value) else: value = int(value) transfer_config[self._TRANSFER_CONFIG_ALIASES[key]] = value else: config[key] = value # pylint: disable=attribute-defined-outside-init self._transfer_config = TransferConfig(**transfer_config) return config
def test_conversions_human_readable_to_bytes_invalid(invalid_input): with pytest.raises(ValueError): human_readable_to_bytes(invalid_input)
def test_conversions_human_readable_to_bytes(test_input, expected): assert human_readable_to_bytes(test_input) == expected