def submit_dummies(self, timestamp_id, pipeline_id, username):
task_name = 'Dummy Task'
task_description = 'Dummy Task for testing'
task = AnalysisTask(self._connection, timestamp_id, pipeline_id,
self._rq_queue.name, task_name, task_description)
task.message = "Analysis Task enqueued"
task.save()
self._connection.sadd(self._get_tracking_set_key(username), task.key)
import_job = self._rq_queue.enqueue_call(
dummy_job, ('import dummy', task.key),
result_ttl=-1,
ttl=-1,
timeout=self._timeout_import,
description='Dummy for import task')
task.import_job = import_job
analysis_job = self._rq_queue.enqueue_call(
dummy_job, ('analysis dummy', task.key),
result_ttl=-1,
ttl=-1,
timeout=self._timeout_import,
description='Dummy for analysis task',
depends_on=import_job)
task.analysis_job = analysis_job
export_job = self._rq_queue.enqueue_call(
dummy_job, ('export dummy', task.key),
result_ttl=-1,
ttl=-1,
timeout=self._timeout_import,
description='Dummy for export task',
depends_on=analysis_job)
task.export_job = export_job
task.save()
return task, None
def dummy_job(name, task_key):
print('EXECUTE DUMMY')
job = get_current_job()
log_store = get_log_store()
task = AnalysisTask.from_key(get_redis_connection(), task_key)
log_store.put(job.id, 'Started Dummy Job ({})'.format(name), 0)
task.update_message('Started Dummy Job ({})'.format(name))
status = [('{} {}'.format('Status', str(i)), i) for i in range(0, 10)]
for msg, progress in status:
log_store.put(job.id, msg, progress)
log_store.put(job.id, 'Finished Dummy Job ({})'.format(name), 100)
task.update_message('Finished Dummy Job ({})'.format(name))
return 'whuiii'
def submit_task(self, timestamp, experiment_name, coordinator, scientist,
input_path, output_path, pipeline, username, local_path):
"""
Creates and Enqueues the necessary background jobs to import, analyze and export the data of the given timestamp
:param timestamp: The timestamp to be analyzed
:param experiment_name: The name of the experiment the timestamp belongs to
:param coordinator: The name of the experiment coordinator
:param scientist: The name of the scientist conducting this experiment
:param input_path: The path, as SMB URL, where the input data (images) are stored
:param output_path: The path, as SMB URL, where the exported data should be stored
:param pipeline: Instance of an IAP Pipeline object which should be used for analysis
:param username: The username of the User requesting this process
:param local_path: The path to the data on the local system
"""
analysis, created = AnalysisModel.get_or_create(
timestamp.id, pipeline.id)
if created:
db.session.commit()
task_name = 'Analyse timestamp data with IAP'
task_description = 'Full IAP Analysis for experiment "{}" at timestamp(Date:{}) with pipeline "{}".'.format(
experiment_name,
timestamp.created_at.strftime("%a %b %d %H:%M:%S UTC %Y"),
pipeline.name)
task = AnalysisTask(self._connection, timestamp.id, pipeline.id,
self._rq_queue.name, task_name,
task_description)
task.message = "Analysis Task enqueued"
task.save()
self._connection.sadd(self._get_tracking_set_key(username),
task.key)
# Lock to ensure only one import job is executed for multiple pipelines
self._lock.acquire()
import_job = None
if timestamp.iap_exp_id is None:
import_job_id = self._get_import_job_id(timestamp.id)
if import_job_id is None:
description = 'Import Image data of experiment "{}" at timestamp [{}] to IAP'.format(
experiment_name,
timestamp.created_at.strftime(
"%a %b %d %H:%M:%S UTC %Y"))
import_job = self._rq_queue.enqueue_call(
invoke_iap_import,
(timestamp.id, experiment_name, coordinator, scientist,
local_path, input_path, username, task.key),
result_ttl=-1,
ttl=-1,
timeout=self._timeout_import,
description=description,
meta={'name': 'import_job'})
self._connection.set(
self._get_import_job_id_key(timestamp.id),
str(import_job.id))
else:
import_job = self._rq_queue.fetch_job(import_job_id)
task.import_job = import_job
self._lock.release()
description = 'Analyse the data of timestamp [{}] with the pipeline "{}" in IAP'.format(
timestamp.created_at.strftime("%a %b %d %H:%M:%S UTC %Y"),
pipeline.name)
analysis_job = self._rq_queue.enqueue_call(
invoke_iap_analysis, (analysis.id, timestamp.id, username,
task.key, timestamp.iap_exp_id),
result_ttl=-1,
ttl=-1,
timeout=self._timeout_analysis,
description=description,
meta={'name': 'analysis_job'},
depends_on=import_job)
task.analysis_job = analysis_job
shared_folder_map = current_app.config['SHARED_FOLDER_MAP']
description = 'Export the IAP results for timestamp [{}] for further use'.format(
timestamp.created_at.strftime("%a %b %d %H:%M:%S UTC %Y"))
export_job = self._rq_queue.enqueue_call(
invoke_iap_export, (timestamp.id, output_path, username,
shared_folder_map, task.key),
result_ttl=-1,
ttl=-1,
timeout=self._timeout_export,
description=description,
meta={'name': 'export_job'},
depends_on=analysis_job)
task.export_job = export_job
task.save()
return task, analysis
else:
if analysis.finished_at is None:
return AnalysisTask.from_key(
self._connection,
AnalysisTask.key_for(analysis.timestamp_id,
analysis.pipeline_id)), analysis
else:
raise AlreadyFinishedError(
AnalysisModel, analysis.id,
'The requested analysis has already been processed')
def fetch_all_tasks(self, username):
tasks = []
keys = self.fetch_all_task_keys(username)
for key in keys:
tasks.append(AnalysisTask.from_key(self._connection, key))
return tasks
def invoke_iap_export(timestamp_id, output_path, username, shared_folder_map, task_key, analysis_iap_id=None):
"""
This Methods represents an RQ Job workload. It should be enqueued into the RQ Analysis Queue and processed by an according worker
Handles the invokation of data export of an IAP analysis on the IAP server and fetches the result information afterwards.
The received information is then entered into the database accordingly
:param timestamp_id: The ID of the :class:`~server.models.timestamp_model.TimestampModel` instance to which the data belongs
:param output_path: The path, as SMB URL, where the data should be exported to
:param username: The username of the user invoking this job
:param analysis_status_id: The ID of the :class:`~server.utils.redis_status_cache.status_object.StatusObject` to which this job belongs
:param shared_folder_map: A dict containing a mapping between SMB URLs and local paths representing the corresponding mount points
:param analysis_iap_id: The IAP ID of the analysis on the IAP server
:return: a dict containing the 'analysis_id' for which the data has been exported
and the 'path' to which the results have been exported. (All nested inside the 'response' key)
"""
print('EXECUTE EXPORT')
job = get_current_job()
log_store = get_log_store()
task = AnalysisTask.from_key(get_redis_connection(), task_key)
channel = get_grpc_channel()
iap_stub = phenopipe_iap_pb2_grpc.PhenopipeIapStub(channel)
pipe_stub = phenopipe_pb2_grpc.PhenopipeStub(channel)
if analysis_iap_id is None:
analysis_iap_id = job.dependency.result['response']['result_id']
else:
analysis_iap_id = analysis_iap_id
log_store.put(job.id, 'Started Export Job', 0)
task.update_message('Started Export Job')
try:
response = iap_stub.ExportExperiment(
phenopipe_iap_pb2.ExportRequest(experiment_id=analysis_iap_id, destination_path=output_path)
)
remote_job_id = response.job_id
request = phenopipe_pb2.WatchJobRequest(
job_id=remote_job_id
)
status = pipe_stub.WatchJob(request)
for msg in status:
print(msg.message.decode('string-escape'))
log_store.put(job.id, msg.message.decode('string-escape'), msg.progress)
response = iap_stub.FetchExportResult(
phenopipe_pb2.FetchJobResultRequest(job_id=remote_job_id)
)
session = get_session()
analysis = session.query(AnalysisModel) \
.filter(AnalysisModel.timestamp_id == timestamp_id) \
.filter(AnalysisModel.iap_id == analysis_iap_id) \
.one()
log_store.put(job.id, 'Received Results. Started to parse and add information', 90)
task.update_message('Received Results. Started to parse and add information')
image_path = get_local_path_from_smb(response.image_path, shared_folder_map)
print('Image Path: {}'.format(image_path))
# TODO handle DB errors
for image_name in os.listdir(image_path):
print('Image Name: {}'.format(image_name))
# Extract information from filename
snapshot_id, _, new_filename = image_name.partition('_')
_, _, angle = os.path.splitext(image_name)[0].rpartition('_')
img = ImageModel(snapshot_id, response.image_path, new_filename, angle, 'segmented')
session.add(img)
# rename file and remove the snapshot id
os.rename(os.path.join(image_path, image_name), os.path.join(image_path, new_filename))
analysis.export_path = response.path
analysis.exported_at = datetime.utcnow()
session.commit()
log_store.put(job.id, 'Finished Export Job', 100)
task.update_message('Finished Export Job')
return create_return_object(JobType.iap_export, timestamp_id,
{'analysis_id': analysis.id, 'path': response.path})
except grpc.RpcError as e:
log_store.put(job.id, e.details(), 0)
task.update_message('Export Job Failed')
print(e.details())
raise
def invoke_iap_analysis(analysis_id, timestamp_id, username, task_key,
experiment_id=None):
"""
This Methods represents an RQ Job workload. It should be enqueued into the RQ Analysis Queue and processed by an according worker
Handles the invocation of data analysis in IAP on the IAP server and fetches the result information afterwards.
The received information is then entered into the database accordingly
:param analysis_id: The ID of the :class:`~server.models.analysis_model.AnalysisModel`
:param timestamp_id: The ID of the :class:`~server.models.timestamp_model.TimestampModel` instance which should be analyzed
:param username: The username of the user invoking this job
:param analysis_status_id: The ID of the :class:`~server.utils.redis_status_cache.status_object.StatusObject` to which this job belongs
:param experiment_id: The IAP ID of this experiment. If this is None the job will assume that the job it depended on
has returned the experiment id in its response object with the key 'experiment_id'
:return: A dict containing the 'result_id' from IAP, the used 'pipeline_id', 'started_at' and 'finished_at' timestamps.
(All nested inside the 'response' key)
"""
print('EXECUTE ANALYSIS')
job = get_current_job()
log_store = get_log_store()
task = AnalysisTask.from_key(get_redis_connection(), task_key)
channel = get_grpc_channel()
iap_stub = phenopipe_iap_pb2_grpc.PhenopipeIapStub(channel)
pipe_stub = phenopipe_pb2_grpc.PhenopipeStub(channel)
if experiment_id is None:
experiment_iap_id = job.dependency.result['response'][
'experiment_id'] # TODO rename experiment_id to experiment_iap_id
else:
experiment_iap_id = experiment_id
log_store.put(job.id, 'Started Analysis Job', 0)
task.update_message('Started Analysis Job')
session = get_session()
# TODO Consider DB errors
analysis = session.query(AnalysisModel).get(analysis_id)
started_at = datetime.utcnow()
analysis.started_at = started_at
session.commit()
try:
response = iap_stub.AnalyzeExperiment(
phenopipe_iap_pb2.AnalyzeRequest(experiment_id=experiment_iap_id, pipeline_id=analysis.pipeline_id)
)
remote_job_id = response.job_id
request = phenopipe_pb2.WatchJobRequest(
job_id=remote_job_id
)
status = pipe_stub.WatchJob(request)
for msg in status:
print(msg.message.decode('string-escape'))
log_store.put(job.id, msg.message.decode('string-escape'), msg.progress)
response = iap_stub.FetchAnalyzeResult(
phenopipe_pb2.FetchJobResultRequest(job_id=remote_job_id)
)
finished_at = datetime.utcnow()
analysis.iap_id = response.result_id
analysis.finished_at = finished_at
session.commit()
log_store.put(job.id, 'Finished Analysis Job', 100)
task.update_message('Finished Analysis Job')
return create_return_object(JobType.iap_analysis, timestamp_id,
{'result_id': response.result_id, 'started_at': started_at,
'finished_at': finished_at, 'pipeline_id': analysis.pipeline_id})
except grpc.RpcError as e:
session.delete(session.query(AnalysisModel).get(analysis.id))
session.commit()
log_store.put(job.id, e.details(), 0)
task.update_message('Analysis Job Failed')
print(e.details())
raise
def invoke_iap_import(timestamp_id, experiment_name, coordinator, scientist, local_path, path, username,
task_key):
"""
This Methods represents an RQ Job workload. It should be enqueued into the RQ Analysis Queue and processed by an according worker
Handles the invokation of data import into IAP on the IAP server and fetches the result information afterwards.
The received information is then entered into the database accordingly
:param timestamp_id: The ID of the :class:`~server.models.timestamp_model.TimestampModel` instance which should be imported
:param experiment_name: The name of the experiment to import
:param coordinator: The name of the experiment coordinator
:param scientist: The name of the scientist carrying out the experiment
:param local_path: The path to the data on the local system
:param path: The SMB url representing the location of the data
:param username: The username of the user invoking this job
:param task_key: The redis key of the :class:`~server.modules.analysis.analysis_task.AnalysisTask` to which this job belongs
:return: A dict containing the 'experiment_id' (nested in the 'response' key) returned by IAP
"""
print('EXECUTE IMPORT')
job = get_current_job()
log_store = get_log_store()
task = AnalysisTask.from_key(get_redis_connection(), task_key)
channel = get_grpc_channel()
iap_stub = phenopipe_iap_pb2_grpc.PhenopipeIapStub(channel)
pipe_stub = phenopipe_pb2_grpc.PhenopipeStub(channel)
log_store.put(job.id, 'Started Import Job', 0)
task.update_message('Started Import Job')
log_store.put(job.id, 'Create Metadata File')
task.update_message('Create Metadata File')
create_iap_import_sheet(timestamp_id, local_path)
log_store.put(job.id, 'Metadata File Created')
task.update_message('Metadata File Created')
try:
log_store.put(job.id, 'Import data into IAP')
task.update_message('Import data into IAP')
import time
time.sleep(30)
response = iap_stub.ImportExperiment(
phenopipe_iap_pb2.ImportRequest(path=path, experiment_name=experiment_name,
coordinator_name=coordinator,
user_name=scientist)
)
remote_job_id = response.job_id
print(remote_job_id)
request = phenopipe_pb2.WatchJobRequest(
job_id=remote_job_id
)
status = pipe_stub.WatchJob(request)
for msg in status:
print(msg.message.decode('string-escape'))
log_store.put(job.id, msg.message.decode('string-escape'), msg.progress)
response = iap_stub.FetchImportResult(
phenopipe_pb2.FetchJobResultRequest(job_id=remote_job_id)
)
session = get_session()
timestamp = session.query(TimestampModel).get(timestamp_id)
timestamp.iap_exp_id = response.experiment_id
session.commit()
log_store.put(job.id, 'Finished Import Job', 100)
task.update_message('Finished Import Job')
return create_return_object(JobType.iap_import, timestamp_id, {'experiment_id': response.experiment_id})
except grpc.RpcError as e:
if e.code() == grpc.StatusCode.ALREADY_EXISTS:
session = get_session()
timestamp = session.query(TimestampModel).get(timestamp_id)
timestamp.iap_exp_id = e.initial_metadata()[0][1]
session.commit()
return create_return_object(JobType.iap_import, timestamp_id, {'experiment_id': timestamp.iap_exp_id})
else:
task.update_message('Import Job Failed')
log_store.put(job.id, e.details(), 0)
print(e.details())
raise
def resolve_analysis_task(self, args, context, info):
_, key = from_global_id(args.get('id'))
task = AnalysisTask.from_key(redis_db, key)
return Task.from_task_object(task)