def get_workflow(self, params: Dict) -> str:
"""Get the workflow identifier from the parameters or the environment.
Attempts to get the workflow identifier from the list of parsed parameters.
Assumes the the identifier is given by the parameter ``workflow``. If no
workflow identifier is present in the given parameter dictionary an
attempt is made to get the identifier from the environment variable that
is defined for workflows in this context.
Raises a missing configuration error if the value is not set.
Parameters
----------
params: dict
Dictionary of parsed command-line parameters
Returns
-------
string
"""
# Get the parameter value for the workflow identifier. The value will
# be set to None if the parameter is defined for a command but was not
# provided by the user.
workflow_id = params.get('workflow')
# If the workflow value is None, attempt to get it from the environment.
if not workflow_id:
workflow_id = os.environ.get(self.vars['workflow'])
# Raise an error if no workflow identifier was found.
if not workflow_id:
raise err.MissingConfigurationError('workflow identifier')
return workflow_id
def FS(env: Dict) -> FileStore:
"""Factory pattern to create file store instances for the service API. Uses
the environment variables FLOWSERV_FILESTORE_MODULE and
FLOWSERV_FILESTORE_CLASS to create an instance of the file store. If the
environment variables are not set the FileSystemStore is returned as the
default file store.
Parameters
----------
env: dict
Configuration dictionary that provides access to configuration
parameters from the environment.
Returns
-------
flowserv.model.files.base.FileStore
"""
module_name = env.get(FLOWSERV_FILESTORE_MODULE)
class_name = env.get(FLOWSERV_FILESTORE_CLASS)
# If both environment variables are None return the default file store.
# Otherwise, import the specified module and return an instance of the
# controller class. An error is raised if only one of the two environment
# variables is set.
if module_name is None and class_name is None:
from flowserv.model.files.fs import FileSystemStore
return FileSystemStore(env=env)
elif module_name is not None and class_name is not None:
from importlib import import_module
module = import_module(module_name)
return getattr(module, class_name)(env=env)
raise err.MissingConfigurationError('file store')
def APP() -> str:
"""Get the value for the FLOWSERV_APP variable from the environment. Raises
a missing configuration error if the value is not set.
Returns
-------
string
"""
app_key = os.environ.get(FLOWSERV_APP)
if not app_key:
raise err.MissingConfigurationError('workflow identifier')
return app_key
def init(force=False):
"""Initialize database and base directories for the API."""
if not force:
click.echo('This will erase an existing database.')
click.confirm('Continue?', default=True, abort=True)
# Create a new instance of the database. Raise errors if the database URL
# is not set.
config = env()
connect_url = config.get(FLOWSERV_DB)
if connect_url is None:
raise err.MissingConfigurationError('database Url')
DB(connect_url=connect_url).init()
def __init__(self, env: Dict):
"""Initialize the base directory. The directory is created if it does
not exist.
Parameters
----------
env: dict
Configuration object that provides access to configuration
parameters in the environment.
"""
self.basedir = env.get(FLOWSERV_BASEDIR)
if self.basedir is None:
raise err.MissingConfigurationError('API base directory')
def access_token(self) -> str:
"""Get the value for the user access token from the environment.
The environment variable that holds the token (*FLOWSERV_ACCESS_TOKEN*)
is the same accross both applications.
Raises a missing configuration error if the value is not set.
Returns
-------
string
"""
access_token = os.environ.get(self.vars['token'])
if not access_token:
raise err.MissingConfigurationError('access token')
return access_token
def __init__(self,
service: APIFactory,
worker_config: Optional[Dict] = None):
"""Initialize the function that is used to execute individual workflow
steps. The run workflow function in this module executes all steps
within sub-processes in the same environment as the workflow
controller.
NOTE: Using the provided execution function is intended for development
and private use only. It is not recommended (and very dangerous) to
use this function in a public setting.
Parameters
----------
service: flowserv.service.api.APIFactory, default=None
API factory for service callbach during asynchronous workflow
execution.
worker_config: dict, default=None
Mapping of container image identifier to worker specifications that
are used to create an instance of a
:class:`flowserv.controller.worker.base.ContainerStep` worker.
"""
self.fs = FS(env=service)
self.service = service
self.worker_config = worker_config if worker_config else service.worker_config(
)
logging.info("workers {}".format(self.worker_config))
# The is_async flag controlls the default setting for asynchronous
# execution. If the flag is False all workflow steps will be executed
# in a sequentiall (blocking) manner.
self.is_async = service.get(FLOWSERV_ASYNC)
# Directory for temporary run files.
basedir = service.get(FLOWSERV_BASEDIR)
if basedir is None:
raise err.MissingConfigurationError('API base directory')
logging.info('base dir {}'.format(basedir))
self.runsdir = service.get(FLOWSERV_RUNSDIR,
os.path.join(basedir, DEFAULT_RUNSDIR))
# Dictionary of all running tasks
self.tasks = dict()
# Lock to manage asynchronous access to the task dictionary
self.lock = Lock()
def get_group(self, params: Dict) -> str:
"""Get the user group (submission) identifier from the parameters or
the environment.
Attempts to get the group identifier from the list of parsed parameters.
Assumes the the identifier is given by the parameter ``group``. If no
group identifier is present in the given parameter dictionary an
attempt is made to get the identifier from the environment variable that
is defined for user groups in this context. If that environment variable
is not set an attempt is made to get the value from the environment
variable for the workflow identifier. The latter is done since flowserv
applications are installed with identical workflow and group identifier.
Raises a missing configuration error if the value is not set.
Parameters
----------
params: dict
Dictionary of parsed command-line parameters
Returns
-------
string
"""
# Get the parameter value for the group identifier. The value will
# be set to None if the parameter is defined for a command but was not
# provided by the user.
group_id = params.get('group')
# If the group value is None, attempt to get it from the environment.
if not group_id:
group_id = os.environ.get(self.vars['group'])
# If the group value is still None, attempt to get it from the
# workflow environment variable.
if not group_id:
group_id = os.environ.get(self.vars['workflow'])
# Raise an error if no group identifier was found.
if not group_id:
raise err.MissingConfigurationError(
'submission (group) identifier')
return group_id
def init_backend(api: APIFactory) -> WorkflowController:
"""Create an instance of the workflow engine based on the given configuration
settings. The workflow engine receives a reference to the API factory for
callback operations that modify the global database state.
Parameters
----------
env: dict, default=None
Dictionary that provides access to configuration parameter values.
api: flowserv.service.api.APIFactory
Reference to tha API factory for callbacks that modify the global
database state.
Returns
-------
flowserv.controller.base.WorkflowController
"""
# Create a new instance of the file store based on the configuration in the
# respective environment variables.
module_name = api.get(config.FLOWSERV_BACKEND_MODULE)
class_name = api.get(config.FLOWSERV_BACKEND_CLASS)
# If both environment variables are None return the default controller.
# Otherwise, import the specified module and return an instance of the
# controller class. An error is raised if only one of the two environment
# variables is set.
if module_name is None and class_name is None:
engine = 'flowserv.controller.serial.engine.base.SerialWorkflowEngine'
logging.info('API backend {}'.format(engine))
from flowserv.controller.serial.engine.base import SerialWorkflowEngine
return SerialWorkflowEngine(service=api)
elif module_name is not None and class_name is not None:
logging.info('API backend {}.{}'.format(module_name, class_name))
from importlib import import_module
module = import_module(module_name)
return getattr(module, class_name)(service=api)
raise err.MissingConfigurationError('workflow backend')