def upload_pipeline(pipeline, username):
"""
Uploads the given pipeline file to the Analyis/IAP server
:param pipeline: the pipeline file object
:raises UnavailableError: if the Analysis/IAP service is not reachable
:raises PipelineAlreadyExistsError: if a pipeline with the same name is already present on the server
:raises InvalidPipelineError: if the pipeline file is malformed
:raises InternalError: if the Analysis/IAP server could not process the request although the user input was correct
:return: True if successful
"""
grpc_ip = current_app.config['ANALYSIS_SERVER_IP']
grpc_port = current_app.config['ANALYSIS_SERVER_GRPC_PORT']
channel = grpc.insecure_channel('{}:{}'.format(grpc_ip, str(grpc_port)))
iap_stub = phenopipe_iap_pb2_grpc.PhenopipeIapStub(channel)
try:
response = iap_stub.UploadPipeline(
phenopipe_iap_pb2.UploadPipelineRequest(file=pipeline.stream.read(), author=username)
)
return response.success
except grpc.RpcError as e:
if e.code() == grpc.StatusCode.UNAVAILABLE:
raise UnavailableError("Analysis Service")
elif e.code() == grpc.StatusCode.ALREADY_EXISTS:
raise PipelineAlreadyExistsError(e.details(), e.initial_metadata()[0][1])
elif e.code() == grpc.StatusCode.INVALID_ARGUMENT:
raise InvalidPipelineError(e.details())
elif e.code() == grpc.StatusCode.INTERNAL:
raise InternalError(e.details())
raise # TODO other error options? --> internal server error? if list of possible grpc errors is exhausted we don't know what error occured
def get_iap_pipeline(username, pipeline_id):
"""
Fetches the IAP Analysis pipeline with the given id by author identified by the username from the Analysis/IAP server
:param username the username of the pipeline author
:param pipeline_id the id of the pipeline to fetch
:raises NotFoundError: if the requested pipeline is not found
:raises UnavailableError: if the Analysis/IAP service is not reachable
:return: The fetched pipeline
"""
grpc_ip = current_app.config['ANALYSIS_SERVER_IP']
grpc_port = current_app.config['ANALYSIS_SERVER_GRPC_PORT']
channel = grpc.insecure_channel('{}:{}'.format(grpc_ip, str(grpc_port)))
iap_stub = phenopipe_iap_pb2_grpc.PhenopipeIapStub(channel)
try:
response = iap_stub.GetPipeline(
phenopipe_iap_pb2.GetPipelineRequest(id=pipeline_id, author=username)
)
return response.pipeline
except grpc.RpcError as e:
if e.code() == grpc.StatusCode.NOT_FOUND:
raise NotFoundError(e.details(), "Pipeline")
elif e.code() == grpc.StatusCode.UNAVAILABLE:
raise UnavailableError("Analysis Service")
raise # TODO other error options? --> internal server error? if list of possible grpc errors is exhausted we don't know what error occured
def delete_pipeline(pipeline_id):
"""
Deletes the specified pipeline form the Analysis/IAP server
:param pipeline_id: The id which identifies the pipeline
:return: True if successful
"""
grpc_ip = current_app.config['ANALYSIS_SERVER_IP']
grpc_port = current_app.config['ANALYSIS_SERVER_GRPC_PORT']
channel = grpc.insecure_channel('{}:{}'.format(grpc_ip, str(grpc_port)))
iap_stub = phenopipe_iap_pb2_grpc.PhenopipeIapStub(channel)
try:
response = iap_stub.DeletePipeline(
phenopipe_iap_pb2.DeletePipelineRequest(id=pipeline_id))
return response.success
except grpc.RpcError as e:
# TODO raise meaningful exception to return to user
print(e.details())
def get_iap_pipelines(username):
"""
Fetches all available IAP Analysis pipelines from the Analysis/IAP server
:raises UnavailableError: if the Analysis/IAP service is not reachable
:return: A list of available pipelines
"""
grpc_ip = current_app.config['ANALYSIS_SERVER_IP']
grpc_port = current_app.config['ANALYSIS_SERVER_GRPC_PORT']
channel = grpc.insecure_channel('{}:{}'.format(grpc_ip, str(grpc_port)))
iap_stub = phenopipe_iap_pb2_grpc.PhenopipeIapStub(channel)
try:
response = iap_stub.GetPipelines(
phenopipe_iap_pb2.GetPipelinesRequest(author=username))
return response.pipelines
except grpc.RpcError as e:
if e.code() == grpc.StatusCode.UNAVAILABLE:
raise UnavailableError("Analysis Service")
raise # TODO other error options? --> internal server error? if list of possible grpc errors is exhausted we don't know what error occured
def invoke_iap_export(timestamp_id, output_path, username, shared_folder_map, task_key, analysis_iap_id=None):
"""
This Methods represents an RQ Job workload. It should be enqueued into the RQ Analysis Queue and processed by an according worker
Handles the invokation of data export of an IAP analysis on the IAP server and fetches the result information afterwards.
The received information is then entered into the database accordingly
:param timestamp_id: The ID of the :class:`~server.models.timestamp_model.TimestampModel` instance to which the data belongs
:param output_path: The path, as SMB URL, where the data should be exported to
:param username: The username of the user invoking this job
:param analysis_status_id: The ID of the :class:`~server.utils.redis_status_cache.status_object.StatusObject` to which this job belongs
:param shared_folder_map: A dict containing a mapping between SMB URLs and local paths representing the corresponding mount points
:param analysis_iap_id: The IAP ID of the analysis on the IAP server
:return: a dict containing the 'analysis_id' for which the data has been exported
and the 'path' to which the results have been exported. (All nested inside the 'response' key)
"""
print('EXECUTE EXPORT')
job = get_current_job()
log_store = get_log_store()
task = AnalysisTask.from_key(get_redis_connection(), task_key)
channel = get_grpc_channel()
iap_stub = phenopipe_iap_pb2_grpc.PhenopipeIapStub(channel)
pipe_stub = phenopipe_pb2_grpc.PhenopipeStub(channel)
if analysis_iap_id is None:
analysis_iap_id = job.dependency.result['response']['result_id']
else:
analysis_iap_id = analysis_iap_id
log_store.put(job.id, 'Started Export Job', 0)
task.update_message('Started Export Job')
try:
response = iap_stub.ExportExperiment(
phenopipe_iap_pb2.ExportRequest(experiment_id=analysis_iap_id, destination_path=output_path)
)
remote_job_id = response.job_id
request = phenopipe_pb2.WatchJobRequest(
job_id=remote_job_id
)
status = pipe_stub.WatchJob(request)
for msg in status:
print(msg.message.decode('string-escape'))
log_store.put(job.id, msg.message.decode('string-escape'), msg.progress)
response = iap_stub.FetchExportResult(
phenopipe_pb2.FetchJobResultRequest(job_id=remote_job_id)
)
session = get_session()
analysis = session.query(AnalysisModel) \
.filter(AnalysisModel.timestamp_id == timestamp_id) \
.filter(AnalysisModel.iap_id == analysis_iap_id) \
.one()
log_store.put(job.id, 'Received Results. Started to parse and add information', 90)
task.update_message('Received Results. Started to parse and add information')
image_path = get_local_path_from_smb(response.image_path, shared_folder_map)
print('Image Path: {}'.format(image_path))
# TODO handle DB errors
for image_name in os.listdir(image_path):
print('Image Name: {}'.format(image_name))
# Extract information from filename
snapshot_id, _, new_filename = image_name.partition('_')
_, _, angle = os.path.splitext(image_name)[0].rpartition('_')
img = ImageModel(snapshot_id, response.image_path, new_filename, angle, 'segmented')
session.add(img)
# rename file and remove the snapshot id
os.rename(os.path.join(image_path, image_name), os.path.join(image_path, new_filename))
analysis.export_path = response.path
analysis.exported_at = datetime.utcnow()
session.commit()
log_store.put(job.id, 'Finished Export Job', 100)
task.update_message('Finished Export Job')
return create_return_object(JobType.iap_export, timestamp_id,
{'analysis_id': analysis.id, 'path': response.path})
except grpc.RpcError as e:
log_store.put(job.id, e.details(), 0)
task.update_message('Export Job Failed')
print(e.details())
raise
def invoke_iap_analysis(analysis_id, timestamp_id, username, task_key,
experiment_id=None):
"""
This Methods represents an RQ Job workload. It should be enqueued into the RQ Analysis Queue and processed by an according worker
Handles the invocation of data analysis in IAP on the IAP server and fetches the result information afterwards.
The received information is then entered into the database accordingly
:param analysis_id: The ID of the :class:`~server.models.analysis_model.AnalysisModel`
:param timestamp_id: The ID of the :class:`~server.models.timestamp_model.TimestampModel` instance which should be analyzed
:param username: The username of the user invoking this job
:param analysis_status_id: The ID of the :class:`~server.utils.redis_status_cache.status_object.StatusObject` to which this job belongs
:param experiment_id: The IAP ID of this experiment. If this is None the job will assume that the job it depended on
has returned the experiment id in its response object with the key 'experiment_id'
:return: A dict containing the 'result_id' from IAP, the used 'pipeline_id', 'started_at' and 'finished_at' timestamps.
(All nested inside the 'response' key)
"""
print('EXECUTE ANALYSIS')
job = get_current_job()
log_store = get_log_store()
task = AnalysisTask.from_key(get_redis_connection(), task_key)
channel = get_grpc_channel()
iap_stub = phenopipe_iap_pb2_grpc.PhenopipeIapStub(channel)
pipe_stub = phenopipe_pb2_grpc.PhenopipeStub(channel)
if experiment_id is None:
experiment_iap_id = job.dependency.result['response'][
'experiment_id'] # TODO rename experiment_id to experiment_iap_id
else:
experiment_iap_id = experiment_id
log_store.put(job.id, 'Started Analysis Job', 0)
task.update_message('Started Analysis Job')
session = get_session()
# TODO Consider DB errors
analysis = session.query(AnalysisModel).get(analysis_id)
started_at = datetime.utcnow()
analysis.started_at = started_at
session.commit()
try:
response = iap_stub.AnalyzeExperiment(
phenopipe_iap_pb2.AnalyzeRequest(experiment_id=experiment_iap_id, pipeline_id=analysis.pipeline_id)
)
remote_job_id = response.job_id
request = phenopipe_pb2.WatchJobRequest(
job_id=remote_job_id
)
status = pipe_stub.WatchJob(request)
for msg in status:
print(msg.message.decode('string-escape'))
log_store.put(job.id, msg.message.decode('string-escape'), msg.progress)
response = iap_stub.FetchAnalyzeResult(
phenopipe_pb2.FetchJobResultRequest(job_id=remote_job_id)
)
finished_at = datetime.utcnow()
analysis.iap_id = response.result_id
analysis.finished_at = finished_at
session.commit()
log_store.put(job.id, 'Finished Analysis Job', 100)
task.update_message('Finished Analysis Job')
return create_return_object(JobType.iap_analysis, timestamp_id,
{'result_id': response.result_id, 'started_at': started_at,
'finished_at': finished_at, 'pipeline_id': analysis.pipeline_id})
except grpc.RpcError as e:
session.delete(session.query(AnalysisModel).get(analysis.id))
session.commit()
log_store.put(job.id, e.details(), 0)
task.update_message('Analysis Job Failed')
print(e.details())
raise
def invoke_iap_import(timestamp_id, experiment_name, coordinator, scientist, local_path, path, username,
task_key):
"""
This Methods represents an RQ Job workload. It should be enqueued into the RQ Analysis Queue and processed by an according worker
Handles the invokation of data import into IAP on the IAP server and fetches the result information afterwards.
The received information is then entered into the database accordingly
:param timestamp_id: The ID of the :class:`~server.models.timestamp_model.TimestampModel` instance which should be imported
:param experiment_name: The name of the experiment to import
:param coordinator: The name of the experiment coordinator
:param scientist: The name of the scientist carrying out the experiment
:param local_path: The path to the data on the local system
:param path: The SMB url representing the location of the data
:param username: The username of the user invoking this job
:param task_key: The redis key of the :class:`~server.modules.analysis.analysis_task.AnalysisTask` to which this job belongs
:return: A dict containing the 'experiment_id' (nested in the 'response' key) returned by IAP
"""
print('EXECUTE IMPORT')
job = get_current_job()
log_store = get_log_store()
task = AnalysisTask.from_key(get_redis_connection(), task_key)
channel = get_grpc_channel()
iap_stub = phenopipe_iap_pb2_grpc.PhenopipeIapStub(channel)
pipe_stub = phenopipe_pb2_grpc.PhenopipeStub(channel)
log_store.put(job.id, 'Started Import Job', 0)
task.update_message('Started Import Job')
log_store.put(job.id, 'Create Metadata File')
task.update_message('Create Metadata File')
create_iap_import_sheet(timestamp_id, local_path)
log_store.put(job.id, 'Metadata File Created')
task.update_message('Metadata File Created')
try:
log_store.put(job.id, 'Import data into IAP')
task.update_message('Import data into IAP')
import time
time.sleep(30)
response = iap_stub.ImportExperiment(
phenopipe_iap_pb2.ImportRequest(path=path, experiment_name=experiment_name,
coordinator_name=coordinator,
user_name=scientist)
)
remote_job_id = response.job_id
print(remote_job_id)
request = phenopipe_pb2.WatchJobRequest(
job_id=remote_job_id
)
status = pipe_stub.WatchJob(request)
for msg in status:
print(msg.message.decode('string-escape'))
log_store.put(job.id, msg.message.decode('string-escape'), msg.progress)
response = iap_stub.FetchImportResult(
phenopipe_pb2.FetchJobResultRequest(job_id=remote_job_id)
)
session = get_session()
timestamp = session.query(TimestampModel).get(timestamp_id)
timestamp.iap_exp_id = response.experiment_id
session.commit()
log_store.put(job.id, 'Finished Import Job', 100)
task.update_message('Finished Import Job')
return create_return_object(JobType.iap_import, timestamp_id, {'experiment_id': response.experiment_id})
except grpc.RpcError as e:
if e.code() == grpc.StatusCode.ALREADY_EXISTS:
session = get_session()
timestamp = session.query(TimestampModel).get(timestamp_id)
timestamp.iap_exp_id = e.initial_metadata()[0][1]
session.commit()
return create_return_object(JobType.iap_import, timestamp_id, {'experiment_id': timestamp.iap_exp_id})
else:
task.update_message('Import Job Failed')
log_store.put(job.id, e.details(), 0)
print(e.details())
raise