def invoke_iap_export(timestamp_id, output_path, username, shared_folder_map, task_key, analysis_iap_id=None):
"""
This Methods represents an RQ Job workload. It should be enqueued into the RQ Analysis Queue and processed by an according worker
Handles the invokation of data export of an IAP analysis on the IAP server and fetches the result information afterwards.
The received information is then entered into the database accordingly
:param timestamp_id: The ID of the :class:`~server.models.timestamp_model.TimestampModel` instance to which the data belongs
:param output_path: The path, as SMB URL, where the data should be exported to
:param username: The username of the user invoking this job
:param analysis_status_id: The ID of the :class:`~server.utils.redis_status_cache.status_object.StatusObject` to which this job belongs
:param shared_folder_map: A dict containing a mapping between SMB URLs and local paths representing the corresponding mount points
:param analysis_iap_id: The IAP ID of the analysis on the IAP server
:return: a dict containing the 'analysis_id' for which the data has been exported
and the 'path' to which the results have been exported. (All nested inside the 'response' key)
"""
print('EXECUTE EXPORT')
job = get_current_job()
log_store = get_log_store()
task = AnalysisTask.from_key(get_redis_connection(), task_key)
channel = get_grpc_channel()
iap_stub = phenopipe_iap_pb2_grpc.PhenopipeIapStub(channel)
pipe_stub = phenopipe_pb2_grpc.PhenopipeStub(channel)
if analysis_iap_id is None:
analysis_iap_id = job.dependency.result['response']['result_id']
else:
analysis_iap_id = analysis_iap_id
log_store.put(job.id, 'Started Export Job', 0)
task.update_message('Started Export Job')
try:
response = iap_stub.ExportExperiment(
phenopipe_iap_pb2.ExportRequest(experiment_id=analysis_iap_id, destination_path=output_path)
)
remote_job_id = response.job_id
request = phenopipe_pb2.WatchJobRequest(
job_id=remote_job_id
)
status = pipe_stub.WatchJob(request)
for msg in status:
print(msg.message.decode('string-escape'))
log_store.put(job.id, msg.message.decode('string-escape'), msg.progress)
response = iap_stub.FetchExportResult(
phenopipe_pb2.FetchJobResultRequest(job_id=remote_job_id)
)
session = get_session()
analysis = session.query(AnalysisModel) \
.filter(AnalysisModel.timestamp_id == timestamp_id) \
.filter(AnalysisModel.iap_id == analysis_iap_id) \
.one()
log_store.put(job.id, 'Received Results. Started to parse and add information', 90)
task.update_message('Received Results. Started to parse and add information')
image_path = get_local_path_from_smb(response.image_path, shared_folder_map)
print('Image Path: {}'.format(image_path))
# TODO handle DB errors
for image_name in os.listdir(image_path):
print('Image Name: {}'.format(image_name))
# Extract information from filename
snapshot_id, _, new_filename = image_name.partition('_')
_, _, angle = os.path.splitext(image_name)[0].rpartition('_')
img = ImageModel(snapshot_id, response.image_path, new_filename, angle, 'segmented')
session.add(img)
# rename file and remove the snapshot id
os.rename(os.path.join(image_path, image_name), os.path.join(image_path, new_filename))
analysis.export_path = response.path
analysis.exported_at = datetime.utcnow()
session.commit()
log_store.put(job.id, 'Finished Export Job', 100)
task.update_message('Finished Export Job')
return create_return_object(JobType.iap_export, timestamp_id,
{'analysis_id': analysis.id, 'path': response.path})
except grpc.RpcError as e:
log_store.put(job.id, e.details(), 0)
task.update_message('Export Job Failed')
print(e.details())
raise
def invoke_iap_import(timestamp_id, experiment_name, coordinator, scientist, local_path, path, username,
task_key):
"""
This Methods represents an RQ Job workload. It should be enqueued into the RQ Analysis Queue and processed by an according worker
Handles the invokation of data import into IAP on the IAP server and fetches the result information afterwards.
The received information is then entered into the database accordingly
:param timestamp_id: The ID of the :class:`~server.models.timestamp_model.TimestampModel` instance which should be imported
:param experiment_name: The name of the experiment to import
:param coordinator: The name of the experiment coordinator
:param scientist: The name of the scientist carrying out the experiment
:param local_path: The path to the data on the local system
:param path: The SMB url representing the location of the data
:param username: The username of the user invoking this job
:param task_key: The redis key of the :class:`~server.modules.analysis.analysis_task.AnalysisTask` to which this job belongs
:return: A dict containing the 'experiment_id' (nested in the 'response' key) returned by IAP
"""
print('EXECUTE IMPORT')
job = get_current_job()
log_store = get_log_store()
task = AnalysisTask.from_key(get_redis_connection(), task_key)
channel = get_grpc_channel()
iap_stub = phenopipe_iap_pb2_grpc.PhenopipeIapStub(channel)
pipe_stub = phenopipe_pb2_grpc.PhenopipeStub(channel)
log_store.put(job.id, 'Started Import Job', 0)
task.update_message('Started Import Job')
log_store.put(job.id, 'Create Metadata File')
task.update_message('Create Metadata File')
create_iap_import_sheet(timestamp_id, local_path)
log_store.put(job.id, 'Metadata File Created')
task.update_message('Metadata File Created')
try:
log_store.put(job.id, 'Import data into IAP')
task.update_message('Import data into IAP')
import time
time.sleep(30)
response = iap_stub.ImportExperiment(
phenopipe_iap_pb2.ImportRequest(path=path, experiment_name=experiment_name,
coordinator_name=coordinator,
user_name=scientist)
)
remote_job_id = response.job_id
print(remote_job_id)
request = phenopipe_pb2.WatchJobRequest(
job_id=remote_job_id
)
status = pipe_stub.WatchJob(request)
for msg in status:
print(msg.message.decode('string-escape'))
log_store.put(job.id, msg.message.decode('string-escape'), msg.progress)
response = iap_stub.FetchImportResult(
phenopipe_pb2.FetchJobResultRequest(job_id=remote_job_id)
)
session = get_session()
timestamp = session.query(TimestampModel).get(timestamp_id)
timestamp.iap_exp_id = response.experiment_id
session.commit()
log_store.put(job.id, 'Finished Import Job', 100)
task.update_message('Finished Import Job')
return create_return_object(JobType.iap_import, timestamp_id, {'experiment_id': response.experiment_id})
except grpc.RpcError as e:
if e.code() == grpc.StatusCode.ALREADY_EXISTS:
session = get_session()
timestamp = session.query(TimestampModel).get(timestamp_id)
timestamp.iap_exp_id = e.initial_metadata()[0][1]
session.commit()
return create_return_object(JobType.iap_import, timestamp_id, {'experiment_id': timestamp.iap_exp_id})
else:
task.update_message('Import Job Failed')
log_store.put(job.id, e.details(), 0)
print(e.details())
raise
def invoke_iap_analysis(analysis_id, timestamp_id, username, task_key,
experiment_id=None):
"""
This Methods represents an RQ Job workload. It should be enqueued into the RQ Analysis Queue and processed by an according worker
Handles the invocation of data analysis in IAP on the IAP server and fetches the result information afterwards.
The received information is then entered into the database accordingly
:param analysis_id: The ID of the :class:`~server.models.analysis_model.AnalysisModel`
:param timestamp_id: The ID of the :class:`~server.models.timestamp_model.TimestampModel` instance which should be analyzed
:param username: The username of the user invoking this job
:param analysis_status_id: The ID of the :class:`~server.utils.redis_status_cache.status_object.StatusObject` to which this job belongs
:param experiment_id: The IAP ID of this experiment. If this is None the job will assume that the job it depended on
has returned the experiment id in its response object with the key 'experiment_id'
:return: A dict containing the 'result_id' from IAP, the used 'pipeline_id', 'started_at' and 'finished_at' timestamps.
(All nested inside the 'response' key)
"""
print('EXECUTE ANALYSIS')
job = get_current_job()
log_store = get_log_store()
task = AnalysisTask.from_key(get_redis_connection(), task_key)
channel = get_grpc_channel()
iap_stub = phenopipe_iap_pb2_grpc.PhenopipeIapStub(channel)
pipe_stub = phenopipe_pb2_grpc.PhenopipeStub(channel)
if experiment_id is None:
experiment_iap_id = job.dependency.result['response'][
'experiment_id'] # TODO rename experiment_id to experiment_iap_id
else:
experiment_iap_id = experiment_id
log_store.put(job.id, 'Started Analysis Job', 0)
task.update_message('Started Analysis Job')
session = get_session()
# TODO Consider DB errors
analysis = session.query(AnalysisModel).get(analysis_id)
started_at = datetime.utcnow()
analysis.started_at = started_at
session.commit()
try:
response = iap_stub.AnalyzeExperiment(
phenopipe_iap_pb2.AnalyzeRequest(experiment_id=experiment_iap_id, pipeline_id=analysis.pipeline_id)
)
remote_job_id = response.job_id
request = phenopipe_pb2.WatchJobRequest(
job_id=remote_job_id
)
status = pipe_stub.WatchJob(request)
for msg in status:
print(msg.message.decode('string-escape'))
log_store.put(job.id, msg.message.decode('string-escape'), msg.progress)
response = iap_stub.FetchAnalyzeResult(
phenopipe_pb2.FetchJobResultRequest(job_id=remote_job_id)
)
finished_at = datetime.utcnow()
analysis.iap_id = response.result_id
analysis.finished_at = finished_at
session.commit()
log_store.put(job.id, 'Finished Analysis Job', 100)
task.update_message('Finished Analysis Job')
return create_return_object(JobType.iap_analysis, timestamp_id,
{'result_id': response.result_id, 'started_at': started_at,
'finished_at': finished_at, 'pipeline_id': analysis.pipeline_id})
except grpc.RpcError as e:
session.delete(session.query(AnalysisModel).get(analysis.id))
session.commit()
log_store.put(job.id, e.details(), 0)
task.update_message('Analysis Job Failed')
print(e.details())
raise
def invoke_r_postprocess(experiment_name, postprocess_id, analysis_id,
excluded_plant_names, path_to_report,
postprocessing_stack_id, postprocessing_stack_name,
username, task_key):
"""
This Methods represents an RQ Job workload. It should be enqueued into the RQ Postprocessing Queue and processed by an according worker
Handles the invokation of a postprocess on the postprocessing server and fetches the result information afterwards.
The received information is then entered into the database accordingly
:param experiment_name: The name of the experiment this postprocess belongs to
:param postprocess_id: The ID of this postprocess
:param analysis_id: The ID of the analysis to be postprocessed
:param excluded_plant_names: The full names (Used as ID by IAP) of all excluded plants
:param path_to_report: The path to the export path of the analysis. (The folder where the report file of IAP is saved)
:param postprocessing_stack_id: The ID of the stack which should be used for postprocessing
:param username: The username of the user issuing this request
:return: A dict containing the 'path_to_results', the used 'postprocessing_stack_id' and a timestamps 'started_at' and 'finished_at'
(All nested inside the 'response' key)
"""
print('EXECUTE POSTPROCESS')
job = get_current_job()
log_store = get_log_store()
task = PostprocessingTask.from_key(get_redis_connection(), task_key)
log_store.put(job.id, 'Started Postprocessing Job', 0)
task.update_message('Started Postprocessing Job')
channel = get_grpc_channel()
r_stub = phenopipe_r_pb2_grpc.PhenopipeRStub(channel)
pipe_stub = phenopipe_pb2_grpc.PhenopipeStub(channel)
session = get_session()
# TODO get parameters from postrpocess object instead of via function parameters
postprocess = session.query(PostprocessModel).get(postprocess_id)
try:
started_at = datetime.utcnow()
postprocess.started_at = started_at
session.commit()
# TODO pass control group in
meta = phenopipe_r_pb2.PostprocessingMetadata(
experiment_name=experiment_name,
control_treatment_name=postprocess.control_group.treatment)
response = r_stub.PostprocessAnalysis(
phenopipe_r_pb2.PostprocessRequest(
path_to_report=path_to_report,
postprocess_stack_id=postprocessing_stack_id,
snapshot_hash=postprocess.snapshot_hash,
meta=meta,
excluded_plant_identifiers=excluded_plant_names))
task.update_message('Started Postprocessing Stack "{}"'.format(
postprocessing_stack_name))
log_store.put(
job.id, 'Started Postprocessing Stack "{}"'.format(
postprocessing_stack_name), 0)
remote_job_id = response.job_id
request = phenopipe_pb2.WatchJobRequest(job_id=remote_job_id)
status = pipe_stub.WatchJob(request)
for msg in status:
log_store.put(job.id, msg.message.decode('string-escape'),
msg.progress)
response = r_stub.FetchPostprocessingResult(
phenopipe_pb2.FetchJobResultRequest(job_id=remote_job_id))
finished_at = datetime.utcnow()
postprocess.finished_at = finished_at
task.update_message('Finished Postprocessing Job')
log_store.put(job.id, 'Finished Postprocessing Job', 100)
postprocess.result_path = response.path_to_results
session.commit()
return create_return_object(
JobType.r_postprocess, analysis_id, {
'path_to_results': response.path_to_results,
'postprocess_stack_id': postprocessing_stack_id,
'started_at': started_at,
'finished_at': finished_at
})
except grpc.RpcError as e:
session.delete(session.query(PostprocessModel).get(postprocess.id))
session.commit()
log_store.put(job.id, e.details(), 0)
task.update_message('Postprocessing Job Failed')
print(e.details())
raise
except DBAPIError as err:
# TODO handle this
print(err.message)
session.rollback()
log_store.put(job.id, e.details(), 0)
task.update_message('Postprocessing Job Failed')
raise