def test_create_store_azure(mocker): # Mock HAzureBlockBlobStore also here, becase otherwise it will try to inherit from # the mock object `mock_azure` created below, which will fail. mock_hazure = mocker.patch("minimalkv._hstores.HAzureBlockBlobStore") mock_azure = mocker.patch("minimalkv.net.azurestore.AzureBlockBlobStore") create_store( "azure", { "account_name": "ACCOUNT", "account_key": "KEY", "container": "cont_name", "create_if_missing": True, }, ) mock_azure.assert_called_once_with( checksum=True, conn_string= "DefaultEndpointsProtocol=https;AccountName=ACCOUNT;AccountKey=KEY", container="cont_name", create_if_missing=True, max_connections=2, public=False, socket_timeout=(20, 100), max_block_size=4194304, max_single_put_size=67108864, ) mock_hazure.assert_not_called()
def test_create_store_hmem(mocker): mock_hmem = mocker.patch("minimalkv._hstores.HDictStore") create_store( "hmemory", { "type": u"memory", "wrap": u"readonly" }, ) mock_hmem.assert_called_once_with()
def test_create_store_valueerror(): with pytest.raises(Exception, match="Unknown store type: ABC"): create_store( "ABC", { "account_name": "ACCOUNT", "account_key": "KEY", "container": "cont_name", "create_if_missing": True, "use_sas": True, }, )
def test_create_store_redis(mocker): mock_Strictredis = mocker.patch("redis.StrictRedis") create_store( "redis", { "type": u"redis", "host": u"localhost", "db": 2 }, ) mock_Strictredis.assert_called_once_with(db=2, host="localhost", type="redis")
def test_create_store_fs(mocker): mock_fs = mocker.patch("minimalkv._store_creation.FilesystemStore") mock_makedirs = mocker.patch("os.makedirs") create_store( "fs", { "type": "fs", "path": "this/is/a/relative/fspath", "create_if_missing": True }, ) mock_fs.assert_called_once_with("this/is/a/relative/fspath") mock_makedirs.assert_called_once_with("this/is/a/relative/fspath")
def test_create_store_azure_inconsistent_params(): with pytest.raises( Exception, match="create_if_missing is incompatible with the use of SAS tokens" ): create_store( "hazure", { "account_name": "ACCOUNT", "account_key": "KEY", "container": "cont_name", "create_if_missing": True, "use_sas": True, }, )
def test_create_store_s3(mocker): mock_s3 = mocker.patch("minimalkv._boto._get_s3bucket") create_store( "s3", { "host": u"endpoint:1234", "access_key": u"access_key", "secret_key": u"secret_key", "bucket": u"bucketname", }, ) mock_s3.assert_called_once_with( host=u"endpoint:1234", access_key=u"access_key", secret_key=u"secret_key", bucket=u"bucketname", )
def test_create_store_hazure(mocker): mock_hazure = mocker.patch("minimalkv._hstores.HAzureBlockBlobStore") create_store( "hazure", { "account_name": "ACCOUNT", "account_key": "KEY", "container": "cont_name", "create_if_missing": True, }, ) mock_hazure.assert_called_once_with( checksum=True, conn_string= "DefaultEndpointsProtocol=https;AccountName=ACCOUNT;AccountKey=KEY", container="cont_name", create_if_missing=True, max_connections=2, public=False, socket_timeout=(20, 100), max_block_size=4194304, max_single_put_size=67108864, )
def get_store( type: str, create_if_missing: bool = True, **params: Any ) -> KeyValueStore: """Return a storage object according to the ``type`` and additional parameters. The ``type`` must be one of the types below, where each allows requires different parameters: * ``"azure"``: Returns a ``minimalkv.azure.AzureBlockBlobStorage``. Parameters are ``"account_name"``, ``"account_key"``, ``"container"``, ``"use_sas"`` and ``"create_if_missing"`` (default: ``True``). ``"create_if_missing"`` has to be ``False`` if ``"use_sas"`` is set. When ``"use_sas"`` is set, ``"account_key"`` is interpreted as Shared Access Signature (SAS) token.FIRE ``"max_connections"``: Maximum number of network connections used by one store (default: ``2``). ``"socket_timeout"``: maximum timeout value in seconds (socket_timeout: ``200``). ``"max_single_put_size"``: max_single_put_size is the largest size upload supported in a single put call. ``"max_block_size"``: maximum block size is maximum size of the blocks(maximum size is <= 100MB) * ``"s3"``: Returns a plain ``minimalkv.net.botostore.BotoStore``. Parameters must include ``"host"``, ``"bucket"``, ``"access_key"``, ``"secret_key"``. Optional parameters are - ``"force_bucket_suffix"`` (default: ``True``). If set, it is ensured that the bucket name ends with ``-<access_key>`` by appending this string if necessary; If ``False``, the bucket name is used as-is. - ``"create_if_missing"`` (default: ``True`` ). If set, creates the bucket if it does not exist; otherwise, try to retrieve the bucket and fail with an ``IOError``. * ``"hs3"`` returns a variant of ``minimalkv.net.botostore.BotoStore`` that allows "/" in the key name. The parameters are the same as for ``"s3"`` * ``"gcs"``: Returns a ``minimalkv.net.gcstore.GoogleCloudStore``. Parameters are ``"credentials"``, ``"bucket_name"``, ``"bucket_creation_location"``, ``"project"`` and ``"create_if_missing"`` (default: ``True``). - ``"credentials"``: either the path to a credentials.json file or a *google.auth.credentials.Credentials* object - ``"bucket_name"``: Name of the bucket the blobs are stored in. - ``"project"``: The name of the GCStorage project. If a credentials JSON is passed then it contains the project name and this parameter will be ignored. - ``"create_if_missing"``: [optional] Create new bucket to store blobs in if ``"bucket_name"`` doesn't exist yet. (default: ``True``). - ``"bucket_creation_location"``: [optional] If a new bucket is created (create_if_missing=True), the location it will be created in. If ``None`` then GCloud uses a default location. * ``"hgcs"``: Like ``"gcs"`` but "/" are allowed in the keynames. * ``"fs"``: Returns a ``minimalkv.fs.FilesystemStore``. Specify the base path as "path" parameter. * ``"hfs"`` returns a variant of ``minimalkv.fs.FilesystemStore`` that allows "/" in the key name. The parameters are the same as for ``"file"``. * ``"memory"``: Returns a DictStore. Doesn't take any parameters * ``"redis"``: Returns a RedisStore. Constructs a StrictRedis using params as kwargs. See StrictRedis documentation for details. Parameters ---------- type : str Type of storage to open, with optional storage decorators. create_if_missing : bool, optional, default = True Create the "root" of the storage (Azure container, parent directory, S3 bucket, etc.). Has no effect for stores where this makes no sense, like ``redis`` or ``memory``. kwargs Parameters specific to the store type. Returns ------- store: KeyValueStore Key value store of type ``type`` as described in ``kwargs`` parameters. """ from minimalkv._store_creation import create_store from minimalkv._store_decoration import decorate_store # split off old-style wrappers, if any: parts = type.split("+") type = parts.pop(-1) decorators = list(reversed(parts)) # find new-style wrappers, if any: wrapspec = params.pop("wrap", "") wrappers = list(wrapspec.split("+")) if wrapspec else [] # can't have both: if wrappers: if decorators: raise ValueError( "Adding store wrappers via store type as well as via wrap parameter are not allowed. Preferably use wrap." ) decorators = wrappers # create_if_missing is a universal parameter, so it's part of the function signature # it can be safely ignored by stores where 'creating' makes no sense. params["create_if_missing"] = create_if_missing store = create_store(type, params) # apply wrappers/decorators: wrapped_store = reduce(decorate_store, decorators, store) return wrapped_store