def get_postprocessing_stack(username, stack_id): """ Fetches the Postprocessing Stack with the given ID and username from the postprocessing server :param stack_id: The ID of the stack to fetch :param username: The username of the user who owns this stack :raises NotFoundError: if the requested postprocessing stack is not found :raises UnavailableError: if the Postprocessing service is not reachable :return: Instance of :class:`server.gen.phenopipe_r_pb2.PostprocessingStack` """ r_stub = phenopipe_r_pb2_grpc.PhenopipeRStub(get_r_channel()) try: response = r_stub.GetPostprocessingStack( phenopipe_r_pb2.GetPostprocessingStackRequest(stack_id=stack_id, author=username)) return response.stack except grpc.RpcError as e: if e.code() == grpc.StatusCode.NOT_FOUND: raise NotFoundError(e.details(), "PostprocessingStack") elif e.code() == grpc.StatusCode.UNAVAILABLE: raise UnavailableError("Postprocessing Service") raise # TODO other error options?
def upload_stack(stack): """ Uploads the given Postprocessing Stack to the Postprocessing server via GRPC :param stack: Instance of :class:`server.gen.phenopipe_r_pb2.PostprocessingStack` :return: The ID of the uploaded stack """ r_stub = phenopipe_r_pb2_grpc.PhenopipeRStub(get_r_channel()) try: scripts = [] for index, script in enumerate(stack.scripts): scripts.append(phenopipe_r_pb2.PostprocessingScript(name=script.name, description=script.description, index=index, file=script.file)) stack = phenopipe_r_pb2.PostprocessingStack(name=stack.name, description=stack.description, author=stack.author, scripts=scripts) response = r_stub.UploadPostprocessingStack( phenopipe_r_pb2.UploadPostprocessingStackRequest(stack=stack) ) stack_id = response.stack_id print(stack_id) return stack_id except grpc.RpcError as e: if e.code() == grpc.StatusCode.UNAVAILABLE: raise UnavailableError("Postprocessing Service") elif e.code() == grpc.StatusCode.ALREADY_EXISTS: raise PostprocessingStackAlreadyExistsError(e.details(), e.initial_metadata()[0][1], e.initial_metadata()[1][1])
def get_postprocessing_stacks(username): """ Fetches all available Postprocessing Stacks from the postprocessing server :raises UnavailableError: if the Postprocessing service is not reachable :return: List of instances of :class:`server.gen.phenopipe_r_pb2.PostprocessingStack` """ r_stub = phenopipe_r_pb2_grpc.PhenopipeRStub(get_r_channel()) try: response = r_stub.GetPostprocessingStacks( phenopipe_r_pb2.GetPostprocessingStacksRequest(author=username)) stacks = response.stacks return stacks except grpc.RpcError as e: if e.code() == grpc.StatusCode.UNAVAILABLE: raise UnavailableError("Postprocessing Service") raise # TODO other error options?
def invoke_r_postprocess(experiment_name, postprocess_id, analysis_id, excluded_plant_names, path_to_report, postprocessing_stack_id, postprocessing_stack_name, username, task_key): """ This Methods represents an RQ Job workload. It should be enqueued into the RQ Postprocessing Queue and processed by an according worker Handles the invokation of a postprocess on the postprocessing server and fetches the result information afterwards. The received information is then entered into the database accordingly :param experiment_name: The name of the experiment this postprocess belongs to :param postprocess_id: The ID of this postprocess :param analysis_id: The ID of the analysis to be postprocessed :param excluded_plant_names: The full names (Used as ID by IAP) of all excluded plants :param path_to_report: The path to the export path of the analysis. (The folder where the report file of IAP is saved) :param postprocessing_stack_id: The ID of the stack which should be used for postprocessing :param username: The username of the user issuing this request :return: A dict containing the 'path_to_results', the used 'postprocessing_stack_id' and a timestamps 'started_at' and 'finished_at' (All nested inside the 'response' key) """ print('EXECUTE POSTPROCESS') job = get_current_job() log_store = get_log_store() task = PostprocessingTask.from_key(get_redis_connection(), task_key) log_store.put(job.id, 'Started Postprocessing Job', 0) task.update_message('Started Postprocessing Job') channel = get_grpc_channel() r_stub = phenopipe_r_pb2_grpc.PhenopipeRStub(channel) pipe_stub = phenopipe_pb2_grpc.PhenopipeStub(channel) session = get_session() # TODO get parameters from postrpocess object instead of via function parameters postprocess = session.query(PostprocessModel).get(postprocess_id) try: started_at = datetime.utcnow() postprocess.started_at = started_at session.commit() # TODO pass control group in meta = phenopipe_r_pb2.PostprocessingMetadata( experiment_name=experiment_name, control_treatment_name=postprocess.control_group.treatment) response = r_stub.PostprocessAnalysis( phenopipe_r_pb2.PostprocessRequest( path_to_report=path_to_report, postprocess_stack_id=postprocessing_stack_id, snapshot_hash=postprocess.snapshot_hash, meta=meta, excluded_plant_identifiers=excluded_plant_names)) task.update_message('Started Postprocessing Stack "{}"'.format( postprocessing_stack_name)) log_store.put( job.id, 'Started Postprocessing Stack "{}"'.format( postprocessing_stack_name), 0) remote_job_id = response.job_id request = phenopipe_pb2.WatchJobRequest(job_id=remote_job_id) status = pipe_stub.WatchJob(request) for msg in status: log_store.put(job.id, msg.message.decode('string-escape'), msg.progress) response = r_stub.FetchPostprocessingResult( phenopipe_pb2.FetchJobResultRequest(job_id=remote_job_id)) finished_at = datetime.utcnow() postprocess.finished_at = finished_at task.update_message('Finished Postprocessing Job') log_store.put(job.id, 'Finished Postprocessing Job', 100) postprocess.result_path = response.path_to_results session.commit() return create_return_object( JobType.r_postprocess, analysis_id, { 'path_to_results': response.path_to_results, 'postprocess_stack_id': postprocessing_stack_id, 'started_at': started_at, 'finished_at': finished_at }) except grpc.RpcError as e: session.delete(session.query(PostprocessModel).get(postprocess.id)) session.commit() log_store.put(job.id, e.details(), 0) task.update_message('Postprocessing Job Failed') print(e.details()) raise except DBAPIError as err: # TODO handle this print(err.message) session.rollback() log_store.put(job.id, e.details(), 0) task.update_message('Postprocessing Job Failed') raise