def test_serialize_kvp():
"""Test serializing and deserializing key-value pairs."""
doc = util.to_dict(
args=[util.to_kvp(key='a', value=1),
util.to_kvp(key='b', value='2')])
assert len(doc) == 2
assert doc['a'] == 1
assert doc['b'] == '2'
def WorkerSpec(worker_type: str,
identifier: Optional[str] = None,
variables: Optional[Dict] = None,
env: Optional[Dict] = None,
volume: Optional[str] = None) -> Dict:
"""Get a serialization for a worker specification.
Parameters
----------
worker_type: string
Unique worker type identifier.
identifier: string, default=None
Unique worker identifier. If no identifier is given, a new unique
identifier will be generated.
variables: dict, default=None
Mapping with default values for placeholders in command template
strings.
env: dict, default=None
Default settings for environment variables when executing workflow
steps. These settings can get overridden by step-specific settings.
volume: string, default=None
Identifier for the storage volume that the worker has access to.
Returns
-------
dict
"""
# Set optional environment and variables dictionaries if not given.
env = env if env is not None else dict()
variables = variables if variables is not None else dict()
doc = {
WORKER_ID:
identifier if identifier is not None else util.get_unique_identifier(),
'type':
worker_type,
'env': [util.to_kvp(key=k, value=v) for k, v in env.items()],
'variables':
[util.to_kvp(key=k, value=v) for k, v in variables.items()]
}
if volume:
doc['volume'] = volume
return doc
def S3Bucket(bucket: str, prefix: Optional[str] = None, identifier: Optional[str] = None) -> Dict:
"""Get configuration object for AWS S3 Storage Volume.
Parameters
----------
bucket: string
AWS S3 bucket identifier.
prefix: string, default=None
Key-prefix for all files.
identifier: string, default=None
Optional storage volume identifier.
Returns
-------
dict
"""
return {
'type': S3_STORE,
'id': identifier,
'args': [
util.to_kvp(key='bucket', value=bucket),
util.to_kvp(key='prefix', value=prefix)
]
}
def FStore(basedir: str, identifier: Optional[str] = None) -> Dict:
"""Get configuration object for a file system storage volume.
Parameters
----------
basedir: string
Google Cloud Storage bucket identifier.
identifier: string, default=None
Optional storage volume identifier.
Returns
-------
dict
"""
return {
'type': FS_STORE,
'id': identifier,
'args': [util.to_kvp(key='basedir', value=basedir)]
}
def Sftp(remotedir: str,
hostname: str,
port: Optional[int] = None,
timeout: Optional[float] = None,
look_for_keys: Optional[bool] = False,
sep: Optional[str] = '/',
identifier: Optional[str] = None) -> Dict:
"""Get configuration object for a remote server storage volume that is
accessed via sftp.
Parameters
----------
remotedir: string
Base directory for stored files on the remote server.
hostname: string
Server to connect to.
port: int, default=None
Server port to connect to.
timeout: float, default=None
Optional timeout (in seconds) for the TCP connect.
look_for_keys: bool, default=False
Set to True to enable searching for discoverable private key files
in ``~/.ssh/``.
sep: string, default='/'
Path separator used by the remote file system.
identifier: string, default=None
Unique storage volume identifier.
Returns
-------
dict
"""
return {
'type':
SFTP_STORE,
'id':
identifier,
'args': [
util.to_kvp(key='basedir', value=remotedir),
util.to_kvp(key='hostname', value=hostname),
util.to_kvp(key='port', value=port),
util.to_kvp(key='timeout', value=timeout),
util.to_kvp(key='look_for_keys', value=look_for_keys),
util.to_kvp(key='sep', value=sep)
]
}