def test_create_store_azure(mocker):
# Mock HAzureBlockBlobStore also here, becase otherwise it will try to inherit from
# the mock object `mock_azure` created below, which will fail.
mock_hazure = mocker.patch("minimalkv._hstores.HAzureBlockBlobStore")
mock_azure = mocker.patch("minimalkv.net.azurestore.AzureBlockBlobStore")
create_store(
"azure",
{
"account_name": "ACCOUNT",
"account_key": "KEY",
"container": "cont_name",
"create_if_missing": True,
},
)
mock_azure.assert_called_once_with(
checksum=True,
conn_string=
"DefaultEndpointsProtocol=https;AccountName=ACCOUNT;AccountKey=KEY",
container="cont_name",
create_if_missing=True,
max_connections=2,
public=False,
socket_timeout=(20, 100),
max_block_size=4194304,
max_single_put_size=67108864,
)
mock_hazure.assert_not_called()
def test_create_store_hmem(mocker):
mock_hmem = mocker.patch("minimalkv._hstores.HDictStore")
create_store(
"hmemory",
{
"type": u"memory",
"wrap": u"readonly"
},
)
mock_hmem.assert_called_once_with()
def test_create_store_valueerror():
with pytest.raises(Exception, match="Unknown store type: ABC"):
create_store(
"ABC",
{
"account_name": "ACCOUNT",
"account_key": "KEY",
"container": "cont_name",
"create_if_missing": True,
"use_sas": True,
},
)
def test_create_store_redis(mocker):
mock_Strictredis = mocker.patch("redis.StrictRedis")
create_store(
"redis",
{
"type": u"redis",
"host": u"localhost",
"db": 2
},
)
mock_Strictredis.assert_called_once_with(db=2,
host="localhost",
type="redis")
def test_create_store_fs(mocker):
mock_fs = mocker.patch("minimalkv._store_creation.FilesystemStore")
mock_makedirs = mocker.patch("os.makedirs")
create_store(
"fs",
{
"type": "fs",
"path": "this/is/a/relative/fspath",
"create_if_missing": True
},
)
mock_fs.assert_called_once_with("this/is/a/relative/fspath")
mock_makedirs.assert_called_once_with("this/is/a/relative/fspath")
def test_create_store_azure_inconsistent_params():
with pytest.raises(
Exception,
match="create_if_missing is incompatible with the use of SAS tokens"
):
create_store(
"hazure",
{
"account_name": "ACCOUNT",
"account_key": "KEY",
"container": "cont_name",
"create_if_missing": True,
"use_sas": True,
},
)
def test_create_store_s3(mocker):
mock_s3 = mocker.patch("minimalkv._boto._get_s3bucket")
create_store(
"s3",
{
"host": u"endpoint:1234",
"access_key": u"access_key",
"secret_key": u"secret_key",
"bucket": u"bucketname",
},
)
mock_s3.assert_called_once_with(
host=u"endpoint:1234",
access_key=u"access_key",
secret_key=u"secret_key",
bucket=u"bucketname",
)
def test_create_store_hazure(mocker):
mock_hazure = mocker.patch("minimalkv._hstores.HAzureBlockBlobStore")
create_store(
"hazure",
{
"account_name": "ACCOUNT",
"account_key": "KEY",
"container": "cont_name",
"create_if_missing": True,
},
)
mock_hazure.assert_called_once_with(
checksum=True,
conn_string=
"DefaultEndpointsProtocol=https;AccountName=ACCOUNT;AccountKey=KEY",
container="cont_name",
create_if_missing=True,
max_connections=2,
public=False,
socket_timeout=(20, 100),
max_block_size=4194304,
max_single_put_size=67108864,
)
def get_store(
type: str, create_if_missing: bool = True, **params: Any
) -> KeyValueStore:
"""Return a storage object according to the ``type`` and additional parameters.
The ``type`` must be one of the types below, where each allows requires different
parameters:
* ``"azure"``: Returns a ``minimalkv.azure.AzureBlockBlobStorage``. Parameters are
``"account_name"``, ``"account_key"``, ``"container"``, ``"use_sas"`` and ``"create_if_missing"`` (default: ``True``).
``"create_if_missing"`` has to be ``False`` if ``"use_sas"`` is set. When ``"use_sas"`` is set,
``"account_key"`` is interpreted as Shared Access Signature (SAS) token.FIRE
``"max_connections"``: Maximum number of network connections used by one store (default: ``2``).
``"socket_timeout"``: maximum timeout value in seconds (socket_timeout: ``200``).
``"max_single_put_size"``: max_single_put_size is the largest size upload supported in a single put call.
``"max_block_size"``: maximum block size is maximum size of the blocks(maximum size is <= 100MB)
* ``"s3"``: Returns a plain ``minimalkv.net.botostore.BotoStore``.
Parameters must include ``"host"``, ``"bucket"``, ``"access_key"``, ``"secret_key"``.
Optional parameters are
- ``"force_bucket_suffix"`` (default: ``True``). If set, it is ensured that
the bucket name ends with ``-<access_key>``
by appending this string if necessary;
If ``False``, the bucket name is used as-is.
- ``"create_if_missing"`` (default: ``True`` ). If set, creates the bucket if it does not exist;
otherwise, try to retrieve the bucket and fail with an ``IOError``.
* ``"hs3"`` returns a variant of ``minimalkv.net.botostore.BotoStore`` that allows "/" in the key name.
The parameters are the same as for ``"s3"``
* ``"gcs"``: Returns a ``minimalkv.net.gcstore.GoogleCloudStore``. Parameters are
``"credentials"``, ``"bucket_name"``, ``"bucket_creation_location"``, ``"project"`` and ``"create_if_missing"`` (default: ``True``).
- ``"credentials"``: either the path to a credentials.json file or a *google.auth.credentials.Credentials* object
- ``"bucket_name"``: Name of the bucket the blobs are stored in.
- ``"project"``: The name of the GCStorage project. If a credentials JSON is passed then it contains the project name
and this parameter will be ignored.
- ``"create_if_missing"``: [optional] Create new bucket to store blobs in if ``"bucket_name"`` doesn't exist yet. (default: ``True``).
- ``"bucket_creation_location"``: [optional] If a new bucket is created (create_if_missing=True), the location it will be created in.
If ``None`` then GCloud uses a default location.
* ``"hgcs"``: Like ``"gcs"`` but "/" are allowed in the keynames.
* ``"fs"``: Returns a ``minimalkv.fs.FilesystemStore``. Specify the base path as "path" parameter.
* ``"hfs"`` returns a variant of ``minimalkv.fs.FilesystemStore`` that allows "/" in the key name.
The parameters are the same as for ``"file"``.
* ``"memory"``: Returns a DictStore. Doesn't take any parameters
* ``"redis"``: Returns a RedisStore. Constructs a StrictRedis using params as kwargs.
See StrictRedis documentation for details.
Parameters
----------
type : str
Type of storage to open, with optional storage decorators.
create_if_missing : bool, optional, default = True
Create the "root" of the storage (Azure container, parent directory, S3 bucket,
etc.). Has no effect for stores where this makes no sense, like ``redis`` or
``memory``.
kwargs
Parameters specific to the store type.
Returns
-------
store: KeyValueStore
Key value store of type ``type`` as described in ``kwargs`` parameters.
"""
from minimalkv._store_creation import create_store
from minimalkv._store_decoration import decorate_store
# split off old-style wrappers, if any:
parts = type.split("+")
type = parts.pop(-1)
decorators = list(reversed(parts))
# find new-style wrappers, if any:
wrapspec = params.pop("wrap", "")
wrappers = list(wrapspec.split("+")) if wrapspec else []
# can't have both:
if wrappers:
if decorators:
raise ValueError(
"Adding store wrappers via store type as well as via wrap parameter are not allowed. Preferably use wrap."
)
decorators = wrappers
# create_if_missing is a universal parameter, so it's part of the function signature
# it can be safely ignored by stores where 'creating' makes no sense.
params["create_if_missing"] = create_if_missing
store = create_store(type, params)
# apply wrappers/decorators:
wrapped_store = reduce(decorate_store, decorators, store)
return wrapped_store