def login_user(self, username, password):
"""Authorize a given user and assign an API key for them. If the user
is unknown or the given credentials do not match those in the database
an unknown user error is raised.
Returns the API key that has been associated with the user identifier.
Parameters
----------
username: string
Unique name (i.e., email address) that the user provided when they
registered
password: string
User password specified during registration (in plain text)
Returns
-------
flowserv.model.base.User
Raises
------
flowserv.error.UnknownUserError
"""
# Get the unique user identifier and encrypted password. Raise error
# if user is unknown
query = self.session.query(User)\
.filter(User.name == username)\
.filter(User.active == True) # noqa: E712
user = query.one_or_none()
if user is None:
raise err.UnknownUserError(username)
# Validate that given credentials match the stored user secret
if not pbkdf2_sha256.verify(password, user.secret):
raise err.UnknownUserError(username)
user_id = user.user_id
ttl = dt.datetime.now() + dt.timedelta(seconds=self.token_timeout)
# Check if a valid access token is currently associated with the user.
api_key = user.api_key
if api_key is not None:
expires = dateutil.parser.parse(api_key.expires)
if expires < dt.datetime.now():
# The key has expired. Set a new key value.
api_key.value = util.get_unique_identifier()
api_key.expires = ttl.isoformat()
else:
# Create a new API key for the user and set the expiry date. The
# key expires token_timeout seconds from now.
user.api_key = APIKey(user_id=user_id,
value=util.get_unique_identifier(),
expires=ttl.isoformat())
return user
def create_workflow(api, source, specfile=None):
"""Start a new workflow for a given template."""
return api.workflows().create_workflow(
name=util.get_unique_identifier(),
source=source,
specfile=specfile
)['id']
def create_group(session, workflow_id, users):
"""Create a new workflow group in the database. Expects a workflow
identifier and a list of user identifier. Returns the identifier for the
created group.
Parameters
----------
session: sqlalchemy.orm.session.Session
Database session.
workflow_id: string
Unique workflow identifier.
users: list
List of unique user identifier.
Returns
-------
string
"""
group_id = util.get_unique_identifier()
group = GroupObject(group_id=group_id,
workflow_id=workflow_id,
name=group_id,
owner_id=users[0],
parameters=ParameterIndex(),
workflow_spec=dict())
# Add users as group members.
for user_id in users:
user = session.query(User).filter(User.user_id == user_id).one()
group.members.append(user)
session.add(group)
return group_id
def request_password_reset(self, username):
"""Request a password reset for the user with a given name. Returns
the request identifier that is required as an argument to reset the
password. The result is always going to be the identifier string
independently of whether a user with the given username is registered
or not.
Invalidates all previous password reset requests for the user.
Parameters
----------
username: string
User email that was provided at registration
Returns
-------
string
"""
request_id = util.get_unique_identifier()
# Get user identifier that is associated with the username
query = self.session.query(User)\
.filter(User.name == username)\
.filter(User.active == True) # noqa: E712
user = query.one_or_none()
if user is None:
return request_id
# Create new password reset request. The expiry date for the request is
# calculated using the login timeout
expires = dt.datetime.now() + dt.timedelta(seconds=self.token_timeout)
user.password_request = PasswordRequest(request_id=request_id,
expires=expires.isoformat())
return request_id
def run_tmpdir() -> str:
"""Get path to a temporary workflow run directory.
Returns
-------
string
"""
return util.join('tmp', util.get_unique_identifier())
def __init__(self, identifier: Optional[str] = None):
"""Initialize the unique volume identifier.
If no identifier is provided a unique identifier is generated.
Parameters
----------
identifier: string
Unique identifier.
"""
self.identifier = identifier if identifier is not None else util.get_unique_identifier()
def register_user(self, username, password, verify=False):
"""Create a new user for the given username. Raises an error if a user
with that name already is registered. Returns the internal unique
identifier for the created user.
The verify flag allows to create active or inactive users. An inactive
user cannot login until they have been activated. This option is
intended for scenarios where the user receives an email after they
register that contains a verification/activation link to ensure that
the provided email address is valid.
Parameters
----------
username: string
User email address that is used as the username
password: string
Password used to authenticate the user
verify: bool, optional
Determines whether the created user is active or inactive
Returns
-------
flowserv.model.base.User
Raises
------
flowserv.error.ConstraintViolationError
flowserv.error.DuplicateUserError
"""
# Ensure that the password has at least one (non-space) character
validate_password(password)
# Ensure that the username is not empty and does not contain more than
# 512 characters.
if username is None:
raise err.ConstraintViolationError('missing user name')
username = username.strip()
if username == '' or len(username) > 512:
raise err.ConstraintViolationError('username too long')
# If a user with the given username already exists raise an error
# Get the unique user identifier and encrypted password. Raise error
# if user is unknown
query = self.session.query(User).filter(User.name == username)
user = query.one_or_none()
if user is not None:
raise err.DuplicateUserError(username)
# Insert new user into database after creating an unique user
# identifier and the password hash.
user = User(user_id=util.get_unique_identifier(),
name=username,
secret=pbkdf2_sha256.hash(password.strip()),
active=False if verify else True)
self.session.add(user)
return user
def upload_file(self, group_id: str, file: IOHandle, name: str):
"""Upload a new file for a workflow group. This will create a copy of
the given file in the file store that is associated with the group. The
file will be places in a unique folder inside the groups upload folder.
Raises an error if the given file name is invalid.
Parameters
----------
group_id: string
Unique group identifier
file: flowserv.model.files.base.IOHandle
File object (e.g., uploaded via HTTP request)
name: string
Name of the file
Returns
-------
flowserv.model.base.UploadFile
Raises
------
flowserv.error.ConstraintViolationError
flowserv.error.UnknownWorkflowGroupError
"""
# Get the group object to ensure that the group exists.
group = self.get_group(group_id)
# Ensure that the given file name is valid
constraint.validate_name(name)
# Create a new unique identifier for the file and save the file object
# to the new file path.
file_id = util.get_unique_identifier()
uploaddir = self.fs.group_uploaddir(
workflow_id=group.workflow_id,
group_id=group.group_id
)
# Get file size.
file_size = file.size()
# Attempt to guess the Mime type for the uploaded file from the file
# name.
mime_type, _ = mimetypes.guess_type(url=name)
self.fs.store_files(files=[(file, file_id)], dst=uploaddir)
# Insert information into database and return handle for uploaded file.
fileobj = UploadFile(
file_id=file_id,
created_at=util.utc_now(),
key=os.path.join(uploaddir, file_id),
name=name,
mime_type=mime_type,
size=file_size
)
group.uploads.append(fileobj)
return fileobj
def create_user(api):
"""Register a new user with the API and return the unique user identifier.
Parameters
----------
api: flowserv.service.api.API
Service API manager.
Returns
-------
string
"""
user_name = util.get_unique_identifier()
doc = api.users().register_user(username=user_name,
password=user_name,
verify=False)
return doc['id']
def __init__(self,
identifier: Optional[str] = None,
volume: Optional[str] = None):
"""Initialize the unique worker identifier and the storage volume that
the worker has access to.
Parameters
----------
identifier: string, default=None
Unique worker identifier. If the value is None a new unique identifier
will be generated.
volume: string, default=None
Identifier for the storage volume that the worker has access to.
By default, the worker is expected to have access to the default
volume store for a workflow run.
"""
self.identifier = identifier if identifier is not None else util.get_unique_identifier(
)
self.volume = volume if volume is not None else DEFAULT_STORE
def upload_file(api, group_id, file):
"""Upload an input file for a workflow run. returns the file identifier.
Parameters
----------
api: flowserv.service.api.API
Service API manager.
group_id: string
Unique group identifier.
file: IOHandle
Uploaded file.
Returns
-------
string
"""
return api.uploads().upload_file(group_id=group_id,
file=file,
name=util.get_unique_identifier())['id']
def WorkerSpec(worker_type: str,
identifier: Optional[str] = None,
variables: Optional[Dict] = None,
env: Optional[Dict] = None,
volume: Optional[str] = None) -> Dict:
"""Get a serialization for a worker specification.
Parameters
----------
worker_type: string
Unique worker type identifier.
identifier: string, default=None
Unique worker identifier. If no identifier is given, a new unique
identifier will be generated.
variables: dict, default=None
Mapping with default values for placeholders in command template
strings.
env: dict, default=None
Default settings for environment variables when executing workflow
steps. These settings can get overridden by step-specific settings.
volume: string, default=None
Identifier for the storage volume that the worker has access to.
Returns
-------
dict
"""
# Set optional environment and variables dictionaries if not given.
env = env if env is not None else dict()
variables = variables if variables is not None else dict()
doc = {
WORKER_ID:
identifier if identifier is not None else util.get_unique_identifier(),
'type':
worker_type,
'env': [util.to_kvp(key=k, value=v) for k, v in env.items()],
'variables':
[util.to_kvp(key=k, value=v) for k, v in variables.items()]
}
if volume:
doc['volume'] = volume
return doc
def create_group(api, workflow_id, users=None):
"""Create a new group for the given workflow.
Parameters
----------
api: flowserv.service.api.API
Service API manager.
workflow_id: string
Unique workflow identifier.
users: list(string)
Identifier for group members.
Returns
-------
string
"""
doc = api.groups().create_group(workflow_id=workflow_id,
name=util.get_unique_identifier(),
members=users)
return doc['id']
def init(self) -> DB:
"""Create all tables in the database model schema. This will also
register the default user. The password for the user is a random UUID
since the default user is not expected to login (but be used only in
open access policies).
"""
# Add import for modules that contain ORM definitions.
import flowserv.model.base # noqa: F401
# Drop all tables first before creating them
Base.metadata.drop_all(self._engine)
Base.metadata.create_all(self._engine)
# Create the default user.
with self.session() as session:
from passlib.hash import pbkdf2_sha256
from flowserv.model.base import User
user = User(user_id=config.DEFAULT_USER,
name=config.DEFAULT_USER,
secret=pbkdf2_sha256.hash(
util.get_unique_identifier()),
active=True)
session.add(user)
return self
def create_user(session, active=True):
"""Create a new user in the database. User identifier, name and password
are all the same UUID. Returns the user identifier.
Parameters
----------
session: sqlalchemy.orm.session.Session
Database session.
active: bool, default=True
User activation flag.
Returns
-------
string
"""
user_id = util.get_unique_identifier()
user = User(user_id=user_id,
name=user_id,
secret=pbkdf2_sha256.hash(user_id),
active=active)
session.add(user)
return user_id
def create_run(session, workflow_id, group_id):
"""Create a new group run. Returns the run identifier.
Parameters
----------
session: sqlalchemy.orm.session.Session
Database session.
workflow_id: string
Unique workflow identifier.
group_id: string
Unique group identifier.
Returns
-------
string
"""
run_id = util.get_unique_identifier()
run = RunObject(run_id=run_id,
workflow_id=workflow_id,
group_id=group_id,
state_type=st.STATE_PENDING)
session.add(run)
return run_id
def create_workflow(session, workflow_spec=dict(), result_schema=None):
"""Create a new workflow handle for a given workflow specification. Returns
the workflow identifier.
Parameters
----------
session: sqlalchemy.orm.session.Session
Database session.
workflow_spec: dict, default=dict()
Optional workflow specification.
result_schema: dict, default=None
Optional result schema.
Returns
-------
string
"""
workflow_id = util.get_unique_identifier()
workflow = WorkflowObject(workflow_id=workflow_id,
name=workflow_id,
workflow_spec=workflow_spec,
result_schema=result_schema)
session.add(workflow)
return workflow_id
def _hello_world(api, name=None, description=None, instructions=None):
return api.workflows().workflow_repo.create_workflow(
name=name if name is not None else util.get_unique_identifier(),
description=description,
instructions=instructions,
source=TEMPLATE_DIR)
def create_run(self, workflow=None, group=None, arguments=None, runs=None):
"""Create a new entry for a run that is in pending state. Returns a
handle for the created run.
A run is either created for a group (i.e., a grop submission run) or
for a workflow (i.e., a post-processing run). Only one of the two
parameters is expected to be None.
Parameters
----------
workflow: flowserv.model.base.WorkflowObject, default=None
Workflow handle if this is a post-processing run.
group: flowserv.model.base.GroupObject
Group handle if this is a group sumbission run.
arguments: list
List of argument values for parameters in the template.
runs: list(string), default=None
List of run identifier that define the input for a post-processing
run.
Returns
-------
flowserv.model.base.RunObject
Raises
------
ValueError
flowserv.error.MissingArgumentError
"""
# Ensure that only group or workflow is given.
if workflow is None and group is None:
raise ValueError('missing arguments for workflow or group')
elif workflow is not None and group is not None:
raise ValueError('arguments for workflow or group')
elif group is not None and runs is not None:
raise ValueError('unexpected argument runs')
# Create a unique run identifier.
run_id = util.get_unique_identifier()
# Get workflow and group identifier.
if workflow is None:
workflow_id = group.workflow_id
group_id = group.group_id
else:
workflow_id = workflow.workflow_id
group_id = None
# Return handle for the created run.
run = RunObject(
run_id=run_id,
workflow_id=workflow_id,
group_id=group_id,
arguments=arguments if arguments is not None else list(),
state_type=st.STATE_PENDING
)
self.session.add(run)
# Update the workflow handle if this is a post-processing run.
if workflow is not None:
ranking = list()
for i in range(len(runs)):
ranking.append(WorkflowRankingRun(run_id=runs[i], rank=i))
workflow.postproc_ranking = ranking
workflow.postproc_run_id = run_id
# Commit changes in case run monitors need to access the run state.
self.session.commit()
return run
import shutil
import tempfile
import flowserv.util as util
tmpdir = tempfile.mkdtemp()
with open(os.path.join(tmpdir, 'requirements.txt'), 'wt') as f:
f.write('histore\n')
f.write('scikit-learn\n')
dockerfile = [
'FROM python:3.9', 'COPY requirements.txt /app/requirements.txt',
'WORKDIR /app', 'RUN pip install papermill',
'RUN pip install -r requirements.txt', 'RUN rm -Rf /app', 'WORKDIR /'
]
with open(os.path.join(tmpdir, 'Dockerfile'), 'wt') as f:
for line in dockerfile:
f.write(f'{line}\n')
run_id = util.get_unique_identifier()
client = docker.from_env()
image, logs = client.images.build(path=tmpdir, tag=run_id, nocache=False)
print(image)
for line in logs:
print(line)
client.close()
shutil.rmtree(tmpdir)
