Ejemplo n.º 1
0
    def test_delete(self):
        resource = Resource(**self.test_resource_data)
        resource.save()

        retr_resource = Resource.objects.get(name="testresource.jpg")
        retr_resource.delete()

        retr_resource2 = Resource.objects.filter(name="testresource.jpg")
        self.assertFalse(retr_resource2.exists())
Ejemplo n.º 2
0
    def run(self, runjob_id):
        """
        Code here are run asynchronously in Celery thread.

        To prevent re-creating a deleted object, any write to database should use
        one of the following:
        + `queryset.update()`
        + `obj.save(update_fields=[...])`
        + `obj.file_field.save(..., save=False)` + `obj.save(update_fields=['file_field'])`

        instead of:
        + `obj.save()`
        + `obj.file_field.save(..., save=True)`
        """
        runjob = RunJob.objects.get(uuid=runjob_id)
        settings = self._settings(runjob)
        inputs = self._inputs(runjob)

        with self.tempdir() as temp_dir:
            outputs = self._outputs(runjob)

            # build argument for run_my_task and mapping dictionary
            arg_outputs = {}
            temppath_map = {}   # retains where originally assigned paths are from... prevent jobs changing them

            for opt_name, output_list in outputs.iteritems():
                if opt_name not in arg_outputs:
                    arg_outputs[opt_name] = []
                for output in output_list:
                    if output['is_list'] is False:
                        output_res_tempname = str(uuid.uuid4())
                        output_res_temppath = os.path.join(temp_dir, output_res_tempname)
                        arg_outputs[opt_name].append({
                            'resource_path': output_res_temppath,
                            'resource_type': output['resource_type']
                        })
                        output['resource_temp_path'] = output_res_temppath
                        temppath_map[output_res_temppath] = output
                    else:
                        # create a folder for them
                        output_res_tempname = str(uuid.uuid4())
                        output_res_tempfolder = os.path.join(temp_dir, output_res_tempname) + os.sep
                        os.mkdir(output_res_tempfolder)
                        arg_outputs[opt_name].append({
                            'resource_folder': output_res_tempfolder,
                            'resource_type': output['resource_type']
                        })
                        output['resource_temp_folder'] = output_res_tempfolder
                        temppath_map[output_res_tempfolder] = output

            retval = self.run_my_task(inputs, settings, arg_outputs)

            if isinstance(retval, self.WAITING_FOR_INPUT):
                settings.update(retval.settings_update)

                runjob.status = task_status.WAITING_FOR_INPUT
                runjob.job_settings = settings
                runjob.error_summary = None
                runjob.error_details = None
                runjob.celery_task_id = None
                runjob.save(update_fields=['status', 'job_settings', 'error_summary', 'error_details', 'celery_task_id'])

                # Send an email to owner of WorkflowRun
                wfrun_id = RunJob.objects.filter(pk=runjob_id).values_list('workflow_run__uuid', flat=True)[0]
                workflowrun = WorkflowRun.objects.get(uuid=wfrun_id)
                user = WorkflowRun.objects.get(uuid=wfrun_id).creator
                if not rodan_settings.TEST:
                    if user.email and rodan_settings.EMAIL_USE and user.user_preference.send_email:
                        subject = "Workflow Run '{0}' is waiting for user input".format(workflowrun.name)
                        body = "A workflow run you started is waiting for user input.\n\n"
                        body = body + "Name: {0}\n".format(workflowrun.name)
                        body = body + "Description: {0}".format(workflowrun.description)
                        to = [user.email]
                        registry.tasks['rodan.core.send_email'].apply_async((subject, body, to))

                    return 'WAITING FOR INPUT'
            else:
                # ensure the runjob did not produce any error
                try:
                    err = self.error_details
                    if len(self.error_details) > 0:
                        raise RuntimeError(self.error_details)
                except AttributeError:
                    pass

                # ensure the job has produced all output files
                for opt_name, output_list in outputs.iteritems():
                    for output in output_list:
                        if output['is_list'] is False:
                            if not os.path.isfile(output['resource_temp_path']):
                                raise RuntimeError("The job did not produce the output file for {0}".format(opt_name))
                        else:
                            files = [f for f in os.listdir(output['resource_temp_folder']) if os.path.isfile(os.path.join(output['resource_temp_folder'], f))]
                            if len(files) == 0:
                                raise RuntimeError("The job did not produce any output files for the resource list for {0}".format(opt_name))

                # save outputs
                for temppath, output in temppath_map.iteritems():
                    if output['is_list'] is False:
                        with open(temppath, 'rb') as f:
                            resource = Output.objects.get(uuid=output['uuid']).resource
                            resource.resource_file.save(temppath, File(f), save=False) # Django will resolve the path according to upload_to
                            resource.save(update_fields=['resource_file'])
                            #registry.tasks['rodan.core.create_thumbnails'].run(resource.uuid.hex) # call synchronously
                            #registry.tasks['rodan.core.create_diva'].run(resource.uuid.hex) # call synchronously
                    else:
                        files = [ff for ff in os.listdir(output['resource_temp_folder']) if os.path.isfile(os.path.join(output['resource_temp_folder'], f))]
                        files.sort()  # alphabetical order

                        resourcelist = Output.objects.get(uuid=output['uuid']).resource_list
                        for index, ff in enumerate(files):
                            with open(os.path.join(output['resource_temp_folder'], ff), 'rb') as f:
                                resource = Resource(
                                    project=resourcelist.project,
                                    resource_type=resourcelist.resource_type,
                                    name=ff,
                                    description="Order #{0} in ResourceList {1}".format(index, resourcelist.name),
                                    origin=resourcelist.origin
                                )
                                resource.save()
                                resource.resource_file.save(ff, File(f), save=False) # Django will resolve the path according to upload_to
                                resource.save(update_fields=['resource_file'])
                                #registry.tasks['rodan.core.create_thumbnails'].run(resource.uuid.hex) # call synchronously
                                #registry.tasks['rodan.core.create_diva'].run(resource.uuid.hex) # call synchronously
                            resourcelist.resources.add(resource)

                runjob.status = task_status.FINISHED
                runjob.error_summary = None
                runjob.error_details = None
                runjob.celery_task_id = None
                runjob.save(update_fields=['status', 'error_summary', 'error_details', 'celery_task_id'])

                # Call master task.
                master_task = registry.tasks['rodan.core.master_task']
                wfrun_id = str(runjob.workflow_run.uuid)
                mt_retval = master_task.run(wfrun_id)
                return "FINISHED  |  master_task: {0}".format(mt_retval)
Ejemplo n.º 3
0
    def test_save_runjob_result(self):
        resource = Resource(**self.test_resource_data)
        resource.save()

        retr_resource = Resource.objects.get(name="testresource.jpg")
        self.assertEqual(retr_resource.name, resource.name)
Ejemplo n.º 4
0
    def run(self, runjob_id):
        """
        Code here are run asynchronously in Celery thread.

        To prevent re-creating a deleted object, any write to database should use
        one of the following:
        + `queryset.update()`
        + `obj.save(update_fields=[...])`
        + `obj.file_field.save(..., save=False)` + `obj.save(update_fields=['file_field'])`

        instead of:
        + `obj.save()`
        + `obj.file_field.save(..., save=True)`
        """
        runjob = RunJob.objects.get(uuid=runjob_id)
        settings = self._settings(runjob)
        inputs = self._inputs(runjob)

        with self.tempdir() as temp_dir:
            outputs = self._outputs(runjob)

            # build argument for run_my_task and mapping dictionary
            arg_outputs = {}
            temppath_map = {
            }  # retains where originally assigned paths are from... prevent jobs changing them

            for opt_name, output_list in outputs.items():
                if opt_name not in arg_outputs:
                    arg_outputs[opt_name] = []
                for output in output_list:
                    if output['is_list'] is False:
                        output_res_tempname = str(uuid.uuid4())
                        output_res_temppath = os.path.join(
                            temp_dir, output_res_tempname)
                        arg_outputs[opt_name].append({
                            'resource_path':
                            output_res_temppath,
                            'resource_type':
                            output['resource_type']
                        })
                        output['resource_temp_path'] = output_res_temppath
                        temppath_map[output_res_temppath] = output
                    else:
                        # create a folder for them
                        output_res_tempname = str(uuid.uuid4())
                        output_res_tempfolder = os.path.join(
                            temp_dir, output_res_tempname) + os.sep
                        os.mkdir(output_res_tempfolder)
                        arg_outputs[opt_name].append({
                            'resource_folder':
                            output_res_tempfolder,
                            'resource_type':
                            output['resource_type']
                        })
                        output['resource_temp_folder'] = output_res_tempfolder
                        temppath_map[output_res_tempfolder] = output

            retval = self.run_my_task(inputs, settings, arg_outputs)

            if isinstance(retval, self.WAITING_FOR_INPUT):
                settings.update(retval.settings_update)

                runjob.status = task_status.WAITING_FOR_INPUT
                runjob.job_settings = settings
                runjob.error_summary = None
                runjob.error_details = None
                runjob.celery_task_id = None
                runjob.save(update_fields=[
                    'status', 'job_settings', 'error_summary', 'error_details',
                    'celery_task_id'
                ])

                # Send an email to owner of WorkflowRun
                wfrun_id = RunJob.objects.filter(pk=runjob_id).values_list(
                    'workflow_run__uuid', flat=True)[0]
                workflowrun = WorkflowRun.objects.get(uuid=wfrun_id)
                user = WorkflowRun.objects.get(uuid=wfrun_id).creator
                if not rodan_settings.TEST:
                    if user.email and rodan_settings.EMAIL_USE and user.user_preference.send_email:
                        subject = "Workflow Run '{0}' is waiting for user input".format(
                            workflowrun.name)
                        body = "A workflow run you started is waiting for user input.\n\n"
                        body = body + "Name: {0}\n".format(workflowrun.name)
                        body = body + "Description: {0}".format(
                            workflowrun.description)
                        to = [user.email]
                        registry.tasks['rodan.core.send_email'].apply_async(
                            (subject, body, to))
                        # registry.tasks['rodan.core.send_email'].apply_async((subject, body, to), queue="celery")

                    return 'WAITING FOR INPUT'
            else:
                # ensure the runjob did not produce any error
                try:
                    err = self.error_details
                    if len(self.error_details) > 0:
                        raise RuntimeError(self.error_details)
                except AttributeError:
                    pass

                # ensure the job has produced all output files
                for opt_name, output_list in outputs.items():
                    for output in output_list:
                        if output['is_list'] is False:
                            if not os.path.isfile(
                                    output['resource_temp_path']):
                                raise RuntimeError(
                                    "The job did not produce the output file for {0}"
                                    .format(opt_name))
                        else:
                            files = [
                                f for f in os.listdir(
                                    output['resource_temp_folder'])
                                if os.path.isfile(
                                    os.path.join(
                                        output['resource_temp_folder'], f))
                            ]
                            if len(files) == 0:
                                raise RuntimeError(
                                    "The job did not produce any output files for the resource list for {0}"
                                    .format(opt_name))

                # save outputs
                for temppath, output in temppath_map.items():
                    if output['is_list'] is False:
                        with open(temppath, 'rb') as f:
                            resource = Output.objects.get(
                                uuid=output['uuid']).resource
                            resource.resource_file.save(
                                temppath, File(f), save=False
                            )  # Django will resolve the path according to upload_to
                            resource.save(update_fields=['resource_file'])
                            if resource.resource_type.mimetype.startswith(
                                    'image'):
                                # registry.tasks['rodan.core.create_thumbnails'].run(resource.uuid.hex) # call synchronously
                                # registry.tasks['rodan.core.create_diva'].run(resource.uuid.hex) # call synchronously
                                registry.tasks['rodan.core.create_diva'].si(
                                    resource.uuid.hex).apply_async(
                                        queue="celery")  # call asynchronously
                    else:
                        files = [
                            ff for ff in os.listdir(
                                output['resource_temp_folder'])
                            if os.path.isfile(
                                os.path.join(output['resource_temp_folder'],
                                             f))
                        ]
                        files.sort()  # alphabetical order

                        resourcelist = Output.objects.get(
                            uuid=output['uuid']).resource_list
                        for index, ff in enumerate(files):
                            with open(
                                    os.path.join(
                                        output['resource_temp_folder'], ff),
                                    'rb') as f:
                                resource = Resource(
                                    project=resourcelist.project,
                                    resource_type=resourcelist.resource_type,
                                    name=ff,
                                    description="Order #{0} in ResourceList {1}"
                                    .format(index, resourcelist.name),
                                    origin=resourcelist.origin)
                                resource.save()
                                resource.resource_file.save(
                                    ff, File(f), save=False
                                )  # Django will resolve the path according to upload_to
                                resource.save(update_fields=['resource_file'])
                                if resource.resource_type.mimetype.startswith(
                                        'image'):
                                    #registry.tasks['rodan.core.create_thumbnails'].run(resource.uuid.hex) # call synchronously
                                    registry.tasks[
                                        'rodan.core.create_diva'].run(
                                            resource.uuid.hex
                                        )  # call synchronously
                                    # registry.tasks['rodan.core.create_diva'].si(resource.uuid.hex).apply_async(queue="celery") # call synchronously
                            resourcelist.resources.add(resource)

                runjob.status = task_status.FINISHED
                runjob.error_summary = None
                runjob.error_details = None
                runjob.celery_task_id = None
                runjob.save(update_fields=[
                    'status', 'error_summary', 'error_details',
                    'celery_task_id'
                ])

                # Call master task.
                master_task = registry.tasks['rodan.core.master_task']
                wfrun_id = str(runjob.workflow_run.uuid)
                mt_retval = master_task.si(wfrun_id).apply_async(
                    queue="celery")
                return "FINISHED  |  master_task: {0}".format(mt_retval)