def test_delete(self):
resource = Resource(**self.test_resource_data)
resource.save()
retr_resource = Resource.objects.get(name="testresource.jpg")
retr_resource.delete()
retr_resource2 = Resource.objects.filter(name="testresource.jpg")
self.assertFalse(retr_resource2.exists())
def run(self, runjob_id):
"""
Code here are run asynchronously in Celery thread.
To prevent re-creating a deleted object, any write to database should use
one of the following:
+ `queryset.update()`
+ `obj.save(update_fields=[...])`
+ `obj.file_field.save(..., save=False)` + `obj.save(update_fields=['file_field'])`
instead of:
+ `obj.save()`
+ `obj.file_field.save(..., save=True)`
"""
runjob = RunJob.objects.get(uuid=runjob_id)
settings = self._settings(runjob)
inputs = self._inputs(runjob)
with self.tempdir() as temp_dir:
outputs = self._outputs(runjob)
# build argument for run_my_task and mapping dictionary
arg_outputs = {}
temppath_map = {} # retains where originally assigned paths are from... prevent jobs changing them
for opt_name, output_list in outputs.iteritems():
if opt_name not in arg_outputs:
arg_outputs[opt_name] = []
for output in output_list:
if output['is_list'] is False:
output_res_tempname = str(uuid.uuid4())
output_res_temppath = os.path.join(temp_dir, output_res_tempname)
arg_outputs[opt_name].append({
'resource_path': output_res_temppath,
'resource_type': output['resource_type']
})
output['resource_temp_path'] = output_res_temppath
temppath_map[output_res_temppath] = output
else:
# create a folder for them
output_res_tempname = str(uuid.uuid4())
output_res_tempfolder = os.path.join(temp_dir, output_res_tempname) + os.sep
os.mkdir(output_res_tempfolder)
arg_outputs[opt_name].append({
'resource_folder': output_res_tempfolder,
'resource_type': output['resource_type']
})
output['resource_temp_folder'] = output_res_tempfolder
temppath_map[output_res_tempfolder] = output
retval = self.run_my_task(inputs, settings, arg_outputs)
if isinstance(retval, self.WAITING_FOR_INPUT):
settings.update(retval.settings_update)
runjob.status = task_status.WAITING_FOR_INPUT
runjob.job_settings = settings
runjob.error_summary = None
runjob.error_details = None
runjob.celery_task_id = None
runjob.save(update_fields=['status', 'job_settings', 'error_summary', 'error_details', 'celery_task_id'])
# Send an email to owner of WorkflowRun
wfrun_id = RunJob.objects.filter(pk=runjob_id).values_list('workflow_run__uuid', flat=True)[0]
workflowrun = WorkflowRun.objects.get(uuid=wfrun_id)
user = WorkflowRun.objects.get(uuid=wfrun_id).creator
if not rodan_settings.TEST:
if user.email and rodan_settings.EMAIL_USE and user.user_preference.send_email:
subject = "Workflow Run '{0}' is waiting for user input".format(workflowrun.name)
body = "A workflow run you started is waiting for user input.\n\n"
body = body + "Name: {0}\n".format(workflowrun.name)
body = body + "Description: {0}".format(workflowrun.description)
to = [user.email]
registry.tasks['rodan.core.send_email'].apply_async((subject, body, to))
return 'WAITING FOR INPUT'
else:
# ensure the runjob did not produce any error
try:
err = self.error_details
if len(self.error_details) > 0:
raise RuntimeError(self.error_details)
except AttributeError:
pass
# ensure the job has produced all output files
for opt_name, output_list in outputs.iteritems():
for output in output_list:
if output['is_list'] is False:
if not os.path.isfile(output['resource_temp_path']):
raise RuntimeError("The job did not produce the output file for {0}".format(opt_name))
else:
files = [f for f in os.listdir(output['resource_temp_folder']) if os.path.isfile(os.path.join(output['resource_temp_folder'], f))]
if len(files) == 0:
raise RuntimeError("The job did not produce any output files for the resource list for {0}".format(opt_name))
# save outputs
for temppath, output in temppath_map.iteritems():
if output['is_list'] is False:
with open(temppath, 'rb') as f:
resource = Output.objects.get(uuid=output['uuid']).resource
resource.resource_file.save(temppath, File(f), save=False) # Django will resolve the path according to upload_to
resource.save(update_fields=['resource_file'])
#registry.tasks['rodan.core.create_thumbnails'].run(resource.uuid.hex) # call synchronously
#registry.tasks['rodan.core.create_diva'].run(resource.uuid.hex) # call synchronously
else:
files = [ff for ff in os.listdir(output['resource_temp_folder']) if os.path.isfile(os.path.join(output['resource_temp_folder'], f))]
files.sort() # alphabetical order
resourcelist = Output.objects.get(uuid=output['uuid']).resource_list
for index, ff in enumerate(files):
with open(os.path.join(output['resource_temp_folder'], ff), 'rb') as f:
resource = Resource(
project=resourcelist.project,
resource_type=resourcelist.resource_type,
name=ff,
description="Order #{0} in ResourceList {1}".format(index, resourcelist.name),
origin=resourcelist.origin
)
resource.save()
resource.resource_file.save(ff, File(f), save=False) # Django will resolve the path according to upload_to
resource.save(update_fields=['resource_file'])
#registry.tasks['rodan.core.create_thumbnails'].run(resource.uuid.hex) # call synchronously
#registry.tasks['rodan.core.create_diva'].run(resource.uuid.hex) # call synchronously
resourcelist.resources.add(resource)
runjob.status = task_status.FINISHED
runjob.error_summary = None
runjob.error_details = None
runjob.celery_task_id = None
runjob.save(update_fields=['status', 'error_summary', 'error_details', 'celery_task_id'])
# Call master task.
master_task = registry.tasks['rodan.core.master_task']
wfrun_id = str(runjob.workflow_run.uuid)
mt_retval = master_task.run(wfrun_id)
return "FINISHED | master_task: {0}".format(mt_retval)
def test_save_runjob_result(self):
resource = Resource(**self.test_resource_data)
resource.save()
retr_resource = Resource.objects.get(name="testresource.jpg")
self.assertEqual(retr_resource.name, resource.name)
def run(self, runjob_id):
"""
Code here are run asynchronously in Celery thread.
To prevent re-creating a deleted object, any write to database should use
one of the following:
+ `queryset.update()`
+ `obj.save(update_fields=[...])`
+ `obj.file_field.save(..., save=False)` + `obj.save(update_fields=['file_field'])`
instead of:
+ `obj.save()`
+ `obj.file_field.save(..., save=True)`
"""
runjob = RunJob.objects.get(uuid=runjob_id)
settings = self._settings(runjob)
inputs = self._inputs(runjob)
with self.tempdir() as temp_dir:
outputs = self._outputs(runjob)
# build argument for run_my_task and mapping dictionary
arg_outputs = {}
temppath_map = {
} # retains where originally assigned paths are from... prevent jobs changing them
for opt_name, output_list in outputs.items():
if opt_name not in arg_outputs:
arg_outputs[opt_name] = []
for output in output_list:
if output['is_list'] is False:
output_res_tempname = str(uuid.uuid4())
output_res_temppath = os.path.join(
temp_dir, output_res_tempname)
arg_outputs[opt_name].append({
'resource_path':
output_res_temppath,
'resource_type':
output['resource_type']
})
output['resource_temp_path'] = output_res_temppath
temppath_map[output_res_temppath] = output
else:
# create a folder for them
output_res_tempname = str(uuid.uuid4())
output_res_tempfolder = os.path.join(
temp_dir, output_res_tempname) + os.sep
os.mkdir(output_res_tempfolder)
arg_outputs[opt_name].append({
'resource_folder':
output_res_tempfolder,
'resource_type':
output['resource_type']
})
output['resource_temp_folder'] = output_res_tempfolder
temppath_map[output_res_tempfolder] = output
retval = self.run_my_task(inputs, settings, arg_outputs)
if isinstance(retval, self.WAITING_FOR_INPUT):
settings.update(retval.settings_update)
runjob.status = task_status.WAITING_FOR_INPUT
runjob.job_settings = settings
runjob.error_summary = None
runjob.error_details = None
runjob.celery_task_id = None
runjob.save(update_fields=[
'status', 'job_settings', 'error_summary', 'error_details',
'celery_task_id'
])
# Send an email to owner of WorkflowRun
wfrun_id = RunJob.objects.filter(pk=runjob_id).values_list(
'workflow_run__uuid', flat=True)[0]
workflowrun = WorkflowRun.objects.get(uuid=wfrun_id)
user = WorkflowRun.objects.get(uuid=wfrun_id).creator
if not rodan_settings.TEST:
if user.email and rodan_settings.EMAIL_USE and user.user_preference.send_email:
subject = "Workflow Run '{0}' is waiting for user input".format(
workflowrun.name)
body = "A workflow run you started is waiting for user input.\n\n"
body = body + "Name: {0}\n".format(workflowrun.name)
body = body + "Description: {0}".format(
workflowrun.description)
to = [user.email]
registry.tasks['rodan.core.send_email'].apply_async(
(subject, body, to))
# registry.tasks['rodan.core.send_email'].apply_async((subject, body, to), queue="celery")
return 'WAITING FOR INPUT'
else:
# ensure the runjob did not produce any error
try:
err = self.error_details
if len(self.error_details) > 0:
raise RuntimeError(self.error_details)
except AttributeError:
pass
# ensure the job has produced all output files
for opt_name, output_list in outputs.items():
for output in output_list:
if output['is_list'] is False:
if not os.path.isfile(
output['resource_temp_path']):
raise RuntimeError(
"The job did not produce the output file for {0}"
.format(opt_name))
else:
files = [
f for f in os.listdir(
output['resource_temp_folder'])
if os.path.isfile(
os.path.join(
output['resource_temp_folder'], f))
]
if len(files) == 0:
raise RuntimeError(
"The job did not produce any output files for the resource list for {0}"
.format(opt_name))
# save outputs
for temppath, output in temppath_map.items():
if output['is_list'] is False:
with open(temppath, 'rb') as f:
resource = Output.objects.get(
uuid=output['uuid']).resource
resource.resource_file.save(
temppath, File(f), save=False
) # Django will resolve the path according to upload_to
resource.save(update_fields=['resource_file'])
if resource.resource_type.mimetype.startswith(
'image'):
# registry.tasks['rodan.core.create_thumbnails'].run(resource.uuid.hex) # call synchronously
# registry.tasks['rodan.core.create_diva'].run(resource.uuid.hex) # call synchronously
registry.tasks['rodan.core.create_diva'].si(
resource.uuid.hex).apply_async(
queue="celery") # call asynchronously
else:
files = [
ff for ff in os.listdir(
output['resource_temp_folder'])
if os.path.isfile(
os.path.join(output['resource_temp_folder'],
f))
]
files.sort() # alphabetical order
resourcelist = Output.objects.get(
uuid=output['uuid']).resource_list
for index, ff in enumerate(files):
with open(
os.path.join(
output['resource_temp_folder'], ff),
'rb') as f:
resource = Resource(
project=resourcelist.project,
resource_type=resourcelist.resource_type,
name=ff,
description="Order #{0} in ResourceList {1}"
.format(index, resourcelist.name),
origin=resourcelist.origin)
resource.save()
resource.resource_file.save(
ff, File(f), save=False
) # Django will resolve the path according to upload_to
resource.save(update_fields=['resource_file'])
if resource.resource_type.mimetype.startswith(
'image'):
#registry.tasks['rodan.core.create_thumbnails'].run(resource.uuid.hex) # call synchronously
registry.tasks[
'rodan.core.create_diva'].run(
resource.uuid.hex
) # call synchronously
# registry.tasks['rodan.core.create_diva'].si(resource.uuid.hex).apply_async(queue="celery") # call synchronously
resourcelist.resources.add(resource)
runjob.status = task_status.FINISHED
runjob.error_summary = None
runjob.error_details = None
runjob.celery_task_id = None
runjob.save(update_fields=[
'status', 'error_summary', 'error_details',
'celery_task_id'
])
# Call master task.
master_task = registry.tasks['rodan.core.master_task']
wfrun_id = str(runjob.workflow_run.uuid)
mt_retval = master_task.si(wfrun_id).apply_async(
queue="celery")
return "FINISHED | master_task: {0}".format(mt_retval)
