def login_user(self, username, password): """Authorize a given user and assign an API key for them. If the user is unknown or the given credentials do not match those in the database an unknown user error is raised. Returns the API key that has been associated with the user identifier. Parameters ---------- username: string Unique name (i.e., email address) that the user provided when they registered password: string User password specified during registration (in plain text) Returns ------- flowserv.model.base.User Raises ------ flowserv.error.UnknownUserError """ # Get the unique user identifier and encrypted password. Raise error # if user is unknown query = self.session.query(User)\ .filter(User.name == username)\ .filter(User.active == True) # noqa: E712 user = query.one_or_none() if user is None: raise err.UnknownUserError(username) # Validate that given credentials match the stored user secret if not pbkdf2_sha256.verify(password, user.secret): raise err.UnknownUserError(username) user_id = user.user_id ttl = dt.datetime.now() + dt.timedelta(seconds=self.token_timeout) # Check if a valid access token is currently associated with the user. api_key = user.api_key if api_key is not None: expires = dateutil.parser.parse(api_key.expires) if expires < dt.datetime.now(): # The key has expired. Set a new key value. api_key.value = util.get_unique_identifier() api_key.expires = ttl.isoformat() else: # Create a new API key for the user and set the expiry date. The # key expires token_timeout seconds from now. user.api_key = APIKey(user_id=user_id, value=util.get_unique_identifier(), expires=ttl.isoformat()) return user
def create_workflow(api, source, specfile=None): """Start a new workflow for a given template.""" return api.workflows().create_workflow( name=util.get_unique_identifier(), source=source, specfile=specfile )['id']
def create_group(session, workflow_id, users): """Create a new workflow group in the database. Expects a workflow identifier and a list of user identifier. Returns the identifier for the created group. Parameters ---------- session: sqlalchemy.orm.session.Session Database session. workflow_id: string Unique workflow identifier. users: list List of unique user identifier. Returns ------- string """ group_id = util.get_unique_identifier() group = GroupObject(group_id=group_id, workflow_id=workflow_id, name=group_id, owner_id=users[0], parameters=ParameterIndex(), workflow_spec=dict()) # Add users as group members. for user_id in users: user = session.query(User).filter(User.user_id == user_id).one() group.members.append(user) session.add(group) return group_id
def request_password_reset(self, username): """Request a password reset for the user with a given name. Returns the request identifier that is required as an argument to reset the password. The result is always going to be the identifier string independently of whether a user with the given username is registered or not. Invalidates all previous password reset requests for the user. Parameters ---------- username: string User email that was provided at registration Returns ------- string """ request_id = util.get_unique_identifier() # Get user identifier that is associated with the username query = self.session.query(User)\ .filter(User.name == username)\ .filter(User.active == True) # noqa: E712 user = query.one_or_none() if user is None: return request_id # Create new password reset request. The expiry date for the request is # calculated using the login timeout expires = dt.datetime.now() + dt.timedelta(seconds=self.token_timeout) user.password_request = PasswordRequest(request_id=request_id, expires=expires.isoformat()) return request_id
def run_tmpdir() -> str: """Get path to a temporary workflow run directory. Returns ------- string """ return util.join('tmp', util.get_unique_identifier())
def __init__(self, identifier: Optional[str] = None): """Initialize the unique volume identifier. If no identifier is provided a unique identifier is generated. Parameters ---------- identifier: string Unique identifier. """ self.identifier = identifier if identifier is not None else util.get_unique_identifier()
def register_user(self, username, password, verify=False): """Create a new user for the given username. Raises an error if a user with that name already is registered. Returns the internal unique identifier for the created user. The verify flag allows to create active or inactive users. An inactive user cannot login until they have been activated. This option is intended for scenarios where the user receives an email after they register that contains a verification/activation link to ensure that the provided email address is valid. Parameters ---------- username: string User email address that is used as the username password: string Password used to authenticate the user verify: bool, optional Determines whether the created user is active or inactive Returns ------- flowserv.model.base.User Raises ------ flowserv.error.ConstraintViolationError flowserv.error.DuplicateUserError """ # Ensure that the password has at least one (non-space) character validate_password(password) # Ensure that the username is not empty and does not contain more than # 512 characters. if username is None: raise err.ConstraintViolationError('missing user name') username = username.strip() if username == '' or len(username) > 512: raise err.ConstraintViolationError('username too long') # If a user with the given username already exists raise an error # Get the unique user identifier and encrypted password. Raise error # if user is unknown query = self.session.query(User).filter(User.name == username) user = query.one_or_none() if user is not None: raise err.DuplicateUserError(username) # Insert new user into database after creating an unique user # identifier and the password hash. user = User(user_id=util.get_unique_identifier(), name=username, secret=pbkdf2_sha256.hash(password.strip()), active=False if verify else True) self.session.add(user) return user
def upload_file(self, group_id: str, file: IOHandle, name: str): """Upload a new file for a workflow group. This will create a copy of the given file in the file store that is associated with the group. The file will be places in a unique folder inside the groups upload folder. Raises an error if the given file name is invalid. Parameters ---------- group_id: string Unique group identifier file: flowserv.model.files.base.IOHandle File object (e.g., uploaded via HTTP request) name: string Name of the file Returns ------- flowserv.model.base.UploadFile Raises ------ flowserv.error.ConstraintViolationError flowserv.error.UnknownWorkflowGroupError """ # Get the group object to ensure that the group exists. group = self.get_group(group_id) # Ensure that the given file name is valid constraint.validate_name(name) # Create a new unique identifier for the file and save the file object # to the new file path. file_id = util.get_unique_identifier() uploaddir = self.fs.group_uploaddir( workflow_id=group.workflow_id, group_id=group.group_id ) # Get file size. file_size = file.size() # Attempt to guess the Mime type for the uploaded file from the file # name. mime_type, _ = mimetypes.guess_type(url=name) self.fs.store_files(files=[(file, file_id)], dst=uploaddir) # Insert information into database and return handle for uploaded file. fileobj = UploadFile( file_id=file_id, created_at=util.utc_now(), key=os.path.join(uploaddir, file_id), name=name, mime_type=mime_type, size=file_size ) group.uploads.append(fileobj) return fileobj
def create_user(api): """Register a new user with the API and return the unique user identifier. Parameters ---------- api: flowserv.service.api.API Service API manager. Returns ------- string """ user_name = util.get_unique_identifier() doc = api.users().register_user(username=user_name, password=user_name, verify=False) return doc['id']
def __init__(self, identifier: Optional[str] = None, volume: Optional[str] = None): """Initialize the unique worker identifier and the storage volume that the worker has access to. Parameters ---------- identifier: string, default=None Unique worker identifier. If the value is None a new unique identifier will be generated. volume: string, default=None Identifier for the storage volume that the worker has access to. By default, the worker is expected to have access to the default volume store for a workflow run. """ self.identifier = identifier if identifier is not None else util.get_unique_identifier( ) self.volume = volume if volume is not None else DEFAULT_STORE
def upload_file(api, group_id, file): """Upload an input file for a workflow run. returns the file identifier. Parameters ---------- api: flowserv.service.api.API Service API manager. group_id: string Unique group identifier. file: IOHandle Uploaded file. Returns ------- string """ return api.uploads().upload_file(group_id=group_id, file=file, name=util.get_unique_identifier())['id']
def WorkerSpec(worker_type: str, identifier: Optional[str] = None, variables: Optional[Dict] = None, env: Optional[Dict] = None, volume: Optional[str] = None) -> Dict: """Get a serialization for a worker specification. Parameters ---------- worker_type: string Unique worker type identifier. identifier: string, default=None Unique worker identifier. If no identifier is given, a new unique identifier will be generated. variables: dict, default=None Mapping with default values for placeholders in command template strings. env: dict, default=None Default settings for environment variables when executing workflow steps. These settings can get overridden by step-specific settings. volume: string, default=None Identifier for the storage volume that the worker has access to. Returns ------- dict """ # Set optional environment and variables dictionaries if not given. env = env if env is not None else dict() variables = variables if variables is not None else dict() doc = { WORKER_ID: identifier if identifier is not None else util.get_unique_identifier(), 'type': worker_type, 'env': [util.to_kvp(key=k, value=v) for k, v in env.items()], 'variables': [util.to_kvp(key=k, value=v) for k, v in variables.items()] } if volume: doc['volume'] = volume return doc
def create_group(api, workflow_id, users=None): """Create a new group for the given workflow. Parameters ---------- api: flowserv.service.api.API Service API manager. workflow_id: string Unique workflow identifier. users: list(string) Identifier for group members. Returns ------- string """ doc = api.groups().create_group(workflow_id=workflow_id, name=util.get_unique_identifier(), members=users) return doc['id']
def init(self) -> DB: """Create all tables in the database model schema. This will also register the default user. The password for the user is a random UUID since the default user is not expected to login (but be used only in open access policies). """ # Add import for modules that contain ORM definitions. import flowserv.model.base # noqa: F401 # Drop all tables first before creating them Base.metadata.drop_all(self._engine) Base.metadata.create_all(self._engine) # Create the default user. with self.session() as session: from passlib.hash import pbkdf2_sha256 from flowserv.model.base import User user = User(user_id=config.DEFAULT_USER, name=config.DEFAULT_USER, secret=pbkdf2_sha256.hash( util.get_unique_identifier()), active=True) session.add(user) return self
def create_user(session, active=True): """Create a new user in the database. User identifier, name and password are all the same UUID. Returns the user identifier. Parameters ---------- session: sqlalchemy.orm.session.Session Database session. active: bool, default=True User activation flag. Returns ------- string """ user_id = util.get_unique_identifier() user = User(user_id=user_id, name=user_id, secret=pbkdf2_sha256.hash(user_id), active=active) session.add(user) return user_id
def create_run(session, workflow_id, group_id): """Create a new group run. Returns the run identifier. Parameters ---------- session: sqlalchemy.orm.session.Session Database session. workflow_id: string Unique workflow identifier. group_id: string Unique group identifier. Returns ------- string """ run_id = util.get_unique_identifier() run = RunObject(run_id=run_id, workflow_id=workflow_id, group_id=group_id, state_type=st.STATE_PENDING) session.add(run) return run_id
def create_workflow(session, workflow_spec=dict(), result_schema=None): """Create a new workflow handle for a given workflow specification. Returns the workflow identifier. Parameters ---------- session: sqlalchemy.orm.session.Session Database session. workflow_spec: dict, default=dict() Optional workflow specification. result_schema: dict, default=None Optional result schema. Returns ------- string """ workflow_id = util.get_unique_identifier() workflow = WorkflowObject(workflow_id=workflow_id, name=workflow_id, workflow_spec=workflow_spec, result_schema=result_schema) session.add(workflow) return workflow_id
def _hello_world(api, name=None, description=None, instructions=None): return api.workflows().workflow_repo.create_workflow( name=name if name is not None else util.get_unique_identifier(), description=description, instructions=instructions, source=TEMPLATE_DIR)
def create_run(self, workflow=None, group=None, arguments=None, runs=None): """Create a new entry for a run that is in pending state. Returns a handle for the created run. A run is either created for a group (i.e., a grop submission run) or for a workflow (i.e., a post-processing run). Only one of the two parameters is expected to be None. Parameters ---------- workflow: flowserv.model.base.WorkflowObject, default=None Workflow handle if this is a post-processing run. group: flowserv.model.base.GroupObject Group handle if this is a group sumbission run. arguments: list List of argument values for parameters in the template. runs: list(string), default=None List of run identifier that define the input for a post-processing run. Returns ------- flowserv.model.base.RunObject Raises ------ ValueError flowserv.error.MissingArgumentError """ # Ensure that only group or workflow is given. if workflow is None and group is None: raise ValueError('missing arguments for workflow or group') elif workflow is not None and group is not None: raise ValueError('arguments for workflow or group') elif group is not None and runs is not None: raise ValueError('unexpected argument runs') # Create a unique run identifier. run_id = util.get_unique_identifier() # Get workflow and group identifier. if workflow is None: workflow_id = group.workflow_id group_id = group.group_id else: workflow_id = workflow.workflow_id group_id = None # Return handle for the created run. run = RunObject( run_id=run_id, workflow_id=workflow_id, group_id=group_id, arguments=arguments if arguments is not None else list(), state_type=st.STATE_PENDING ) self.session.add(run) # Update the workflow handle if this is a post-processing run. if workflow is not None: ranking = list() for i in range(len(runs)): ranking.append(WorkflowRankingRun(run_id=runs[i], rank=i)) workflow.postproc_ranking = ranking workflow.postproc_run_id = run_id # Commit changes in case run monitors need to access the run state. self.session.commit() return run
import shutil import tempfile import flowserv.util as util tmpdir = tempfile.mkdtemp() with open(os.path.join(tmpdir, 'requirements.txt'), 'wt') as f: f.write('histore\n') f.write('scikit-learn\n') dockerfile = [ 'FROM python:3.9', 'COPY requirements.txt /app/requirements.txt', 'WORKDIR /app', 'RUN pip install papermill', 'RUN pip install -r requirements.txt', 'RUN rm -Rf /app', 'WORKDIR /' ] with open(os.path.join(tmpdir, 'Dockerfile'), 'wt') as f: for line in dockerfile: f.write(f'{line}\n') run_id = util.get_unique_identifier() client = docker.from_env() image, logs = client.images.build(path=tmpdir, tag=run_id, nocache=False) print(image) for line in logs: print(line) client.close() shutil.rmtree(tmpdir)