def __init__( self, created_at, started_at=None, stopped_at=None, messages=None ): """Initialize the timestamps that are associated with the workflow state and the optional error messages. Parameters ---------- created_at: string Timestamp of workflow creation started_at: string Timestamp when the workflow started running stopped_at: string, optional Timestamp when workflow error occurred or the when the workflow was canceled messages: list(string), optional Optional list of error messages """ super(StateError, self).__init__( type_id=STATE_ERROR, created_at=created_at ) self.started_at = started_at if started_at is not None else created_at self.stopped_at = stopped_at if stopped_at is not None else utc_now() self.messages = messages if messages is not None else list()
def test_workflow_leaderboard_serialization(database, tmpdir): """Test serialization of a workflow leaderboard.""" config = Config().basedir(tmpdir) schema = validator('WorkflowLeaderboard') view = WorkflowSerializer() with database.session() as session: manager = WorkflowManager(session=session, fs=FileSystemStore(config)) workflow = manager.create_workflow(source=BENCHMARK_DIR, name='Test', specfile=SPEC_FILE) ts = util.utc_now() ranking = [ RunResult(run_id='0', group_id='1', group_name='A', created_at=ts, started_at=ts, finished_at=ts, values={ 'len': 1, 'count': 10 }) ] doc = view.workflow_leaderboard(workflow, ranking=ranking) schema.validate(doc)
def __init__(self, created_at: str, started_at: str, finished_at: Optional[str] = None, files: Optional[List[str]] = None): """Initialize the timestamps that are associated with the workflow state and the list of created files. Parameters ---------- created_at: string Timestamp of workflow creation started_at: string Timestamp when the workflow started running finished_at: string, optional Timestamp when workflow execution completed files: list(string), default=None Optional list of created files (relative path). """ super(StateSuccess, self).__init__(type_id=STATE_SUCCESS, created_at=created_at) self.started_at = started_at self.finished_at = finished_at if finished_at is not None else utc_now( ) # noqa: E501 self.files = files if files is not None else list()
def test_obsolete_runs(fscls, database, tmpdir): """Test deleting runs that were created before a given date.""" # -- Setup ---------------------------------------------------------------- fs = fscls(env=Config().basedir(tmpdir)) # Create two runs (one SUCCESS and one ERROR) before a timestamp t1 _, _, run_1, _ = success_run(database, fs, tmpdir) _, _, run_2 = error_run(database, fs, ['There were errors']) time.sleep(1) t1 = util.utc_now() # Create another SUCCESS run after timestamp t1 _, _, run_3, _ = success_run(database, fs, tmpdir) # -- Test delete run with state filter ------------------------------------ with database.session() as session: runs = RunManager(session=session, fs=fs) assert runs.delete_obsolete_runs(date=t1, state=st.STATE_ERROR) == 1 # After deleting the error run the two success runs still exist. runs.get_run(run_id=run_1) with pytest.raises(err.UnknownRunError): runs.get_run(run_id=run_2) runs.get_run(run_id=run_3) # -- Test delete all runs prior to a given date --------------------------- with database.session() as session: runs = RunManager(session=session, fs=fs) assert runs.delete_obsolete_runs(date=t1) == 1 # After deleting the run the only one success runs still exist. with pytest.raises(err.UnknownRunError): runs.get_run(run_id=run_1) runs.get_run(run_id=run_3)
def run_success(run_manager, run_id, store, values): """Set given run into success state with the given result data.""" store.store(file=io_file(values), dst=RESULT_FILE_ID) ts = util.utc_now() run_manager.update_run(run_id=run_id, state=st.StateSuccess(created_at=ts, started_at=ts, finished_at=ts, files=[RESULT_FILE_ID]), runstore=store)
def __init__(self, type_id, created_at=None): """Initialize the type identifier and the 'created at' timestamp. Parameters ---------- type_id: string Type identifier """ self.type_id = type_id self.created_at = created_at if created_at is not None else utc_now()
def upload_file(self, group_id: str, file: IOHandle, name: str): """Upload a new file for a workflow group. This will create a copy of the given file in the file store that is associated with the group. The file will be places in a unique folder inside the groups upload folder. Raises an error if the given file name is invalid. Parameters ---------- group_id: string Unique group identifier file: flowserv.model.files.base.IOHandle File object (e.g., uploaded via HTTP request) name: string Name of the file Returns ------- flowserv.model.base.UploadFile Raises ------ flowserv.error.ConstraintViolationError flowserv.error.UnknownWorkflowGroupError """ # Get the group object to ensure that the group exists. group = self.get_group(group_id) # Ensure that the given file name is valid constraint.validate_name(name) # Create a new unique identifier for the file and save the file object # to the new file path. file_id = util.get_unique_identifier() uploaddir = self.fs.group_uploaddir( workflow_id=group.workflow_id, group_id=group.group_id ) # Get file size. file_size = file.size() # Attempt to guess the Mime type for the uploaded file from the file # name. mime_type, _ = mimetypes.guess_type(url=name) self.fs.store_files(files=[(file, file_id)], dst=uploaddir) # Insert information into database and return handle for uploaded file. fileobj = UploadFile( file_id=file_id, created_at=util.utc_now(), key=os.path.join(uploaddir, file_id), name=name, mime_type=mime_type, size=file_size ) group.uploads.append(fileobj) return fileobj
def __init__(self, type_id: str, created_at: Optional[str] = None): """Initialize the type identifier and the 'created at' timestamp. Parameters ---------- type_id: string Workflow state type identifier. created_at: str, default=None Timestamp when the workflow run was first created. """ self.type_id = type_id self.created_at = created_at if created_at is not None else utc_now()
def __init__(self, created_at: str, started_at: Optional[str] = None): """Initialize the timestamps that are associated with the workflow state. Parameters ---------- created_at: string Timestamp of workflow creation started_at: string Timestamp when the workflow started running """ super(StateRunning, self).__init__(type_id=STATE_RUNNING, created_at=created_at) self.started_at = started_at if started_at is not None else utc_now()
def run_success(run_manager, run_id, rundir, values): """Set given run into success state with the given result data.""" os.makedirs(rundir) filename = os.path.join(rundir, RESULT_FILE_ID) util.write_object(filename=filename, obj=values) ts = util.utc_now() run_manager.update_run( run_id=run_id, state=st.StateSuccess( created_at=ts, started_at=ts, finished_at=ts, files=[RESULT_FILE_ID] ), rundir=rundir )
def cancel(self, messages: Optional[List[str]] = None) -> StateCanceled: """Get instance of canceled state for a pending wokflow. Since the workflow did not start to run the started_at timestamp is set to the current time just like the stopped_at timestamp. Parameters ---------- messages: list(string), optional Optional list of messages Returns ------- flowserv.model.workflow.state.StateCanceled """ ts = utc_now() return StateCanceled(created_at=self.created_at, started_at=ts, stopped_at=ts, messages=messages)
def error(self, messages: Optional[List[str]] = None) -> StateError: """Get instance of error state for a pending wokflow. If the exception that caused the workflow execution to terminate is given it will be used to create the list of error messages. Since the workflow did not start to run the started_at timestamp is set to the current time just like the stopped_at timestamp. Parameters ---------- messages: list(string), optional Optional list of error messages Returns ------- flowserv.model.workflow.state.StateError """ ts = utc_now() return StateError(created_at=self.created_at, started_at=ts, stopped_at=ts, messages=messages)
def __init__(self, created_at: str, started_at: Optional[str] = None, stopped_at: Optional[str] = None, messages: Optional[List[str]] = None): """Initialize the timestamps that are associated with the workflow state and the optional messages. Parameters ---------- created_at: string Timestamp of workflow creation started_at: string Timestamp when the workflow started running stopped_at: string, optional Timestamp when workflow was canceled messages: list(string), optional Optional list of messages """ super(StateCanceled, self).__init__(type_id=STATE_CANCELED, created_at=created_at) self.started_at = started_at if started_at is not None else created_at self.stopped_at = stopped_at if stopped_at is not None else utc_now() self.messages = messages if messages is not None else CANCELED