def create_iap_import_sheet(timestamp_id, path): """ Creates the metatdata sheet 'fileimport.xlsx' for IAP at the specified path :param timestamp_id: The ID of the :class:`~server.models.timestamp_model.TimestampModel` instance for which the file should be generated :param path: The destination path to which the 'fileimport.xlsx' should be saved :return:None """ session = get_session() timestamp = session.query(TimestampModel).get(timestamp_id) if timestamp.iap_exp_id is not None: return False wb = Workbook() sheet = wb.active sheet.title = 'meta-data' for row in sheet.iter_rows(min_row=1, max_col=len(_iap_meta_cols), max_row=1): for index, cell in enumerate(row): cell.value = _iap_meta_cols[index] start_date = timestamp.experiment.start_of_experimentation if start_date is None: start_date = timestamp.experiment.start_date row_index = 2 for snap in timestamp.snapshots: images = snap.images.all() for index, row in enumerate( sheet.iter_rows(min_row=row_index, max_col=len(_iap_meta_cols), max_row=row_index + len(images) - 1)): # Assumption based on iap_exp_id check: there are only raw images and no segmented ones image = images[index] row[0].value = image.filename # filename row[1].value = snap.plant.full_name # plant id row[2].value = snap.id # replicate id has to be unique for each plant time_diff = image.created_at - start_date row[3].value = time_diff.days # time row[4].value = 'day' # time unit row[5].value = image.created_at # imaging time row[6].value = snap.measurement_tool row[7].value = snap.camera_position row[8].value = image.angle group = snap.plant.sample_group row[9].value = group.species row[10].value = group.genotype row[11].value = group.variety row[12].value = group.growth_conditions row[13].value = group.treatment row_index += len(images) wb.save(os.path.join(path, 'fileimport.xlsx'))
def invoke_iap_analysis(analysis_id, timestamp_id, username, task_key, experiment_id=None): """ This Methods represents an RQ Job workload. It should be enqueued into the RQ Analysis Queue and processed by an according worker Handles the invocation of data analysis in IAP on the IAP server and fetches the result information afterwards. The received information is then entered into the database accordingly :param analysis_id: The ID of the :class:`~server.models.analysis_model.AnalysisModel` :param timestamp_id: The ID of the :class:`~server.models.timestamp_model.TimestampModel` instance which should be analyzed :param username: The username of the user invoking this job :param analysis_status_id: The ID of the :class:`~server.utils.redis_status_cache.status_object.StatusObject` to which this job belongs :param experiment_id: The IAP ID of this experiment. If this is None the job will assume that the job it depended on has returned the experiment id in its response object with the key 'experiment_id' :return: A dict containing the 'result_id' from IAP, the used 'pipeline_id', 'started_at' and 'finished_at' timestamps. (All nested inside the 'response' key) """ print('EXECUTE ANALYSIS') job = get_current_job() log_store = get_log_store() task = AnalysisTask.from_key(get_redis_connection(), task_key) channel = get_grpc_channel() iap_stub = phenopipe_iap_pb2_grpc.PhenopipeIapStub(channel) pipe_stub = phenopipe_pb2_grpc.PhenopipeStub(channel) if experiment_id is None: experiment_iap_id = job.dependency.result['response'][ 'experiment_id'] # TODO rename experiment_id to experiment_iap_id else: experiment_iap_id = experiment_id log_store.put(job.id, 'Started Analysis Job', 0) task.update_message('Started Analysis Job') session = get_session() # TODO Consider DB errors analysis = session.query(AnalysisModel).get(analysis_id) started_at = datetime.utcnow() analysis.started_at = started_at session.commit() try: response = iap_stub.AnalyzeExperiment( phenopipe_iap_pb2.AnalyzeRequest(experiment_id=experiment_iap_id, pipeline_id=analysis.pipeline_id) ) remote_job_id = response.job_id request = phenopipe_pb2.WatchJobRequest( job_id=remote_job_id ) status = pipe_stub.WatchJob(request) for msg in status: print(msg.message.decode('string-escape')) log_store.put(job.id, msg.message.decode('string-escape'), msg.progress) response = iap_stub.FetchAnalyzeResult( phenopipe_pb2.FetchJobResultRequest(job_id=remote_job_id) ) finished_at = datetime.utcnow() analysis.iap_id = response.result_id analysis.finished_at = finished_at session.commit() log_store.put(job.id, 'Finished Analysis Job', 100) task.update_message('Finished Analysis Job') return create_return_object(JobType.iap_analysis, timestamp_id, {'result_id': response.result_id, 'started_at': started_at, 'finished_at': finished_at, 'pipeline_id': analysis.pipeline_id}) except grpc.RpcError as e: session.delete(session.query(AnalysisModel).get(analysis.id)) session.commit() log_store.put(job.id, e.details(), 0) task.update_message('Analysis Job Failed') print(e.details()) raise
def invoke_iap_export(timestamp_id, output_path, username, shared_folder_map, task_key, analysis_iap_id=None): """ This Methods represents an RQ Job workload. It should be enqueued into the RQ Analysis Queue and processed by an according worker Handles the invokation of data export of an IAP analysis on the IAP server and fetches the result information afterwards. The received information is then entered into the database accordingly :param timestamp_id: The ID of the :class:`~server.models.timestamp_model.TimestampModel` instance to which the data belongs :param output_path: The path, as SMB URL, where the data should be exported to :param username: The username of the user invoking this job :param analysis_status_id: The ID of the :class:`~server.utils.redis_status_cache.status_object.StatusObject` to which this job belongs :param shared_folder_map: A dict containing a mapping between SMB URLs and local paths representing the corresponding mount points :param analysis_iap_id: The IAP ID of the analysis on the IAP server :return: a dict containing the 'analysis_id' for which the data has been exported and the 'path' to which the results have been exported. (All nested inside the 'response' key) """ print('EXECUTE EXPORT') job = get_current_job() log_store = get_log_store() task = AnalysisTask.from_key(get_redis_connection(), task_key) channel = get_grpc_channel() iap_stub = phenopipe_iap_pb2_grpc.PhenopipeIapStub(channel) pipe_stub = phenopipe_pb2_grpc.PhenopipeStub(channel) if analysis_iap_id is None: analysis_iap_id = job.dependency.result['response']['result_id'] else: analysis_iap_id = analysis_iap_id log_store.put(job.id, 'Started Export Job', 0) task.update_message('Started Export Job') try: response = iap_stub.ExportExperiment( phenopipe_iap_pb2.ExportRequest(experiment_id=analysis_iap_id, destination_path=output_path) ) remote_job_id = response.job_id request = phenopipe_pb2.WatchJobRequest( job_id=remote_job_id ) status = pipe_stub.WatchJob(request) for msg in status: print(msg.message.decode('string-escape')) log_store.put(job.id, msg.message.decode('string-escape'), msg.progress) response = iap_stub.FetchExportResult( phenopipe_pb2.FetchJobResultRequest(job_id=remote_job_id) ) session = get_session() analysis = session.query(AnalysisModel) \ .filter(AnalysisModel.timestamp_id == timestamp_id) \ .filter(AnalysisModel.iap_id == analysis_iap_id) \ .one() log_store.put(job.id, 'Received Results. Started to parse and add information', 90) task.update_message('Received Results. Started to parse and add information') image_path = get_local_path_from_smb(response.image_path, shared_folder_map) print('Image Path: {}'.format(image_path)) # TODO handle DB errors for image_name in os.listdir(image_path): print('Image Name: {}'.format(image_name)) # Extract information from filename snapshot_id, _, new_filename = image_name.partition('_') _, _, angle = os.path.splitext(image_name)[0].rpartition('_') img = ImageModel(snapshot_id, response.image_path, new_filename, angle, 'segmented') session.add(img) # rename file and remove the snapshot id os.rename(os.path.join(image_path, image_name), os.path.join(image_path, new_filename)) analysis.export_path = response.path analysis.exported_at = datetime.utcnow() session.commit() log_store.put(job.id, 'Finished Export Job', 100) task.update_message('Finished Export Job') return create_return_object(JobType.iap_export, timestamp_id, {'analysis_id': analysis.id, 'path': response.path}) except grpc.RpcError as e: log_store.put(job.id, e.details(), 0) task.update_message('Export Job Failed') print(e.details()) raise
def invoke_iap_import(timestamp_id, experiment_name, coordinator, scientist, local_path, path, username, task_key): """ This Methods represents an RQ Job workload. It should be enqueued into the RQ Analysis Queue and processed by an according worker Handles the invokation of data import into IAP on the IAP server and fetches the result information afterwards. The received information is then entered into the database accordingly :param timestamp_id: The ID of the :class:`~server.models.timestamp_model.TimestampModel` instance which should be imported :param experiment_name: The name of the experiment to import :param coordinator: The name of the experiment coordinator :param scientist: The name of the scientist carrying out the experiment :param local_path: The path to the data on the local system :param path: The SMB url representing the location of the data :param username: The username of the user invoking this job :param task_key: The redis key of the :class:`~server.modules.analysis.analysis_task.AnalysisTask` to which this job belongs :return: A dict containing the 'experiment_id' (nested in the 'response' key) returned by IAP """ print('EXECUTE IMPORT') job = get_current_job() log_store = get_log_store() task = AnalysisTask.from_key(get_redis_connection(), task_key) channel = get_grpc_channel() iap_stub = phenopipe_iap_pb2_grpc.PhenopipeIapStub(channel) pipe_stub = phenopipe_pb2_grpc.PhenopipeStub(channel) log_store.put(job.id, 'Started Import Job', 0) task.update_message('Started Import Job') log_store.put(job.id, 'Create Metadata File') task.update_message('Create Metadata File') create_iap_import_sheet(timestamp_id, local_path) log_store.put(job.id, 'Metadata File Created') task.update_message('Metadata File Created') try: log_store.put(job.id, 'Import data into IAP') task.update_message('Import data into IAP') import time time.sleep(30) response = iap_stub.ImportExperiment( phenopipe_iap_pb2.ImportRequest(path=path, experiment_name=experiment_name, coordinator_name=coordinator, user_name=scientist) ) remote_job_id = response.job_id print(remote_job_id) request = phenopipe_pb2.WatchJobRequest( job_id=remote_job_id ) status = pipe_stub.WatchJob(request) for msg in status: print(msg.message.decode('string-escape')) log_store.put(job.id, msg.message.decode('string-escape'), msg.progress) response = iap_stub.FetchImportResult( phenopipe_pb2.FetchJobResultRequest(job_id=remote_job_id) ) session = get_session() timestamp = session.query(TimestampModel).get(timestamp_id) timestamp.iap_exp_id = response.experiment_id session.commit() log_store.put(job.id, 'Finished Import Job', 100) task.update_message('Finished Import Job') return create_return_object(JobType.iap_import, timestamp_id, {'experiment_id': response.experiment_id}) except grpc.RpcError as e: if e.code() == grpc.StatusCode.ALREADY_EXISTS: session = get_session() timestamp = session.query(TimestampModel).get(timestamp_id) timestamp.iap_exp_id = e.initial_metadata()[0][1] session.commit() return create_return_object(JobType.iap_import, timestamp_id, {'experiment_id': timestamp.iap_exp_id}) else: task.update_message('Import Job Failed') log_store.put(job.id, e.details(), 0) print(e.details()) raise