Esempio n. 1
0
def get_task_instance(job_name):
    if job_name not in REGISTERED_JOBS:
        return None

    job_info = REGISTERED_JOBS[job_name]

    if not isinstance(job_info, dict) or 'type' not in job_info:
        raise ImproperlyConfigured('NC_REGISTERED_JOBS configuration is invalid.')

    if job_info['type'] == 'task':
        class_path = job_info.get('task')
        if not class_path:
            raise ImproperlyConfigured('Registered job {} does not specify a task.'.format(job_name))

        try:
            module_name, class_name = class_path.rsplit('.', 1)
            module = import_module(module_name)
            cls = getattr(module, class_name)
        except (ImportError, ValueError, AttributeError):
            raise ImproperlyConfigured('{} is not a valid task.'.format(class_path))

        return cls()
    elif job_info['type'] == 'workflow':
        path = job_info.get('path')
        if not path or not os.path.isfile(path):
            raise ImproperlyConfigured('The workflow {} does not exist.'.format(path))

        with open(path, 'r') as f:
            return Workflow.from_json(f.read())
    else:
        raise ImproperlyConfigured('Invalid job type: {}'.format(job_info['type']))
Esempio n. 2
0
    def test_map_reduce_workflow(self):
        with open(os.path.join(TEST_DATA_DIR, "map_reduce_workflow.json"), "r") as f:
            workflow = Workflow.from_json(f.read())

        arr_1 = numpy.arange(10)
        arr_2 = numpy.arange(10, 20)
        arr_3 = numpy.arange(20, 30)
        expected = sum([x / float(numpy.max(x)) for x in [arr_1, arr_2, arr_3]])
        result = workflow(arrays_in=[arr_1, arr_2, arr_3])
        array_out = result["array_out"]

        assert is_ndarray(array_out)
        assert (array_out == expected).all()
Esempio n. 3
0
    def test_map_reduce_workflow(self):
        with open(os.path.join(TEST_DATA_DIR, 'map_reduce_workflow.json'),
                  'r') as f:
            workflow = Workflow.from_json(f.read())

        arr_1 = numpy.arange(10)
        arr_2 = numpy.arange(10, 20)
        arr_3 = numpy.arange(20, 30)
        expected = sum(
            [x / float(numpy.max(x)) for x in [arr_1, arr_2, arr_3]])
        result = workflow(arrays_in=[arr_1, arr_2, arr_3])
        array_out = result['array_out']

        assert is_ndarray(array_out)
        assert (array_out == expected).all()
Esempio n. 4
0
    def test_simple_workflow_deserialization(self, simple_workflow):
        serialized = simple_workflow.to_json()
        workflow = Workflow.from_json(serialized)

        assert json.loads(workflow.to_json()) == json.loads(serialized)
Esempio n. 5
0
    def test_simple_workflow_deserialization(self, simple_workflow):
        serialized = simple_workflow.to_json()
        workflow = Workflow.from_json(serialized)

        assert json.loads(workflow.to_json()) == json.loads(serialized)