def submit_dummies(self, timestamp_id, pipeline_id, username):
task_name = 'Dummy Task'
task_description = 'Dummy Task for testing'
task = AnalysisTask(self._connection, timestamp_id, pipeline_id,
self._rq_queue.name, task_name, task_description)
task.message = "Analysis Task enqueued"
task.save()
self._connection.sadd(self._get_tracking_set_key(username), task.key)
import_job = self._rq_queue.enqueue_call(
dummy_job, ('import dummy', task.key),
result_ttl=-1,
ttl=-1,
timeout=self._timeout_import,
description='Dummy for import task')
task.import_job = import_job
analysis_job = self._rq_queue.enqueue_call(
dummy_job, ('analysis dummy', task.key),
result_ttl=-1,
ttl=-1,
timeout=self._timeout_import,
description='Dummy for analysis task',
depends_on=import_job)
task.analysis_job = analysis_job
export_job = self._rq_queue.enqueue_call(
dummy_job, ('export dummy', task.key),
result_ttl=-1,
ttl=-1,
timeout=self._timeout_import,
description='Dummy for export task',
depends_on=analysis_job)
task.export_job = export_job
task.save()
return task, None
def submit_task(self, timestamp, experiment_name, coordinator, scientist,
input_path, output_path, pipeline, username, local_path):
"""
Creates and Enqueues the necessary background jobs to import, analyze and export the data of the given timestamp
:param timestamp: The timestamp to be analyzed
:param experiment_name: The name of the experiment the timestamp belongs to
:param coordinator: The name of the experiment coordinator
:param scientist: The name of the scientist conducting this experiment
:param input_path: The path, as SMB URL, where the input data (images) are stored
:param output_path: The path, as SMB URL, where the exported data should be stored
:param pipeline: Instance of an IAP Pipeline object which should be used for analysis
:param username: The username of the User requesting this process
:param local_path: The path to the data on the local system
"""
analysis, created = AnalysisModel.get_or_create(
timestamp.id, pipeline.id)
if created:
db.session.commit()
task_name = 'Analyse timestamp data with IAP'
task_description = 'Full IAP Analysis for experiment "{}" at timestamp(Date:{}) with pipeline "{}".'.format(
experiment_name,
timestamp.created_at.strftime("%a %b %d %H:%M:%S UTC %Y"),
pipeline.name)
task = AnalysisTask(self._connection, timestamp.id, pipeline.id,
self._rq_queue.name, task_name,
task_description)
task.message = "Analysis Task enqueued"
task.save()
self._connection.sadd(self._get_tracking_set_key(username),
task.key)
# Lock to ensure only one import job is executed for multiple pipelines
self._lock.acquire()
import_job = None
if timestamp.iap_exp_id is None:
import_job_id = self._get_import_job_id(timestamp.id)
if import_job_id is None:
description = 'Import Image data of experiment "{}" at timestamp [{}] to IAP'.format(
experiment_name,
timestamp.created_at.strftime(
"%a %b %d %H:%M:%S UTC %Y"))
import_job = self._rq_queue.enqueue_call(
invoke_iap_import,
(timestamp.id, experiment_name, coordinator, scientist,
local_path, input_path, username, task.key),
result_ttl=-1,
ttl=-1,
timeout=self._timeout_import,
description=description,
meta={'name': 'import_job'})
self._connection.set(
self._get_import_job_id_key(timestamp.id),
str(import_job.id))
else:
import_job = self._rq_queue.fetch_job(import_job_id)
task.import_job = import_job
self._lock.release()
description = 'Analyse the data of timestamp [{}] with the pipeline "{}" in IAP'.format(
timestamp.created_at.strftime("%a %b %d %H:%M:%S UTC %Y"),
pipeline.name)
analysis_job = self._rq_queue.enqueue_call(
invoke_iap_analysis, (analysis.id, timestamp.id, username,
task.key, timestamp.iap_exp_id),
result_ttl=-1,
ttl=-1,
timeout=self._timeout_analysis,
description=description,
meta={'name': 'analysis_job'},
depends_on=import_job)
task.analysis_job = analysis_job
shared_folder_map = current_app.config['SHARED_FOLDER_MAP']
description = 'Export the IAP results for timestamp [{}] for further use'.format(
timestamp.created_at.strftime("%a %b %d %H:%M:%S UTC %Y"))
export_job = self._rq_queue.enqueue_call(
invoke_iap_export, (timestamp.id, output_path, username,
shared_folder_map, task.key),
result_ttl=-1,
ttl=-1,
timeout=self._timeout_export,
description=description,
meta={'name': 'export_job'},
depends_on=analysis_job)
task.export_job = export_job
task.save()
return task, analysis
else:
if analysis.finished_at is None:
return AnalysisTask.from_key(
self._connection,
AnalysisTask.key_for(analysis.timestamp_id,
analysis.pipeline_id)), analysis
else:
raise AlreadyFinishedError(
AnalysisModel, analysis.id,
'The requested analysis has already been processed')