def get_task_instance(job_name):
if job_name not in REGISTERED_JOBS:
return None
job_info = REGISTERED_JOBS[job_name]
if not isinstance(job_info, dict) or 'type' not in job_info:
raise ImproperlyConfigured('NC_REGISTERED_JOBS configuration is invalid.')
if job_info['type'] == 'task':
class_path = job_info.get('task')
if not class_path:
raise ImproperlyConfigured('Registered job {} does not specify a task.'.format(job_name))
try:
module_name, class_name = class_path.rsplit('.', 1)
module = import_module(module_name)
cls = getattr(module, class_name)
except (ImportError, ValueError, AttributeError):
raise ImproperlyConfigured('{} is not a valid task.'.format(class_path))
return cls()
elif job_info['type'] == 'workflow':
path = job_info.get('path')
if not path or not os.path.isfile(path):
raise ImproperlyConfigured('The workflow {} does not exist.'.format(path))
with open(path, 'r') as f:
return Workflow.from_json(f.read())
else:
raise ImproperlyConfigured('Invalid job type: {}'.format(job_info['type']))
def test_map_reduce_workflow(self):
with open(os.path.join(TEST_DATA_DIR, "map_reduce_workflow.json"), "r") as f:
workflow = Workflow.from_json(f.read())
arr_1 = numpy.arange(10)
arr_2 = numpy.arange(10, 20)
arr_3 = numpy.arange(20, 30)
expected = sum([x / float(numpy.max(x)) for x in [arr_1, arr_2, arr_3]])
result = workflow(arrays_in=[arr_1, arr_2, arr_3])
array_out = result["array_out"]
assert is_ndarray(array_out)
assert (array_out == expected).all()
def test_map_reduce_workflow(self):
with open(os.path.join(TEST_DATA_DIR, 'map_reduce_workflow.json'),
'r') as f:
workflow = Workflow.from_json(f.read())
arr_1 = numpy.arange(10)
arr_2 = numpy.arange(10, 20)
arr_3 = numpy.arange(20, 30)
expected = sum(
[x / float(numpy.max(x)) for x in [arr_1, arr_2, arr_3]])
result = workflow(arrays_in=[arr_1, arr_2, arr_3])
array_out = result['array_out']
assert is_ndarray(array_out)
assert (array_out == expected).all()
def test_simple_workflow_deserialization(self, simple_workflow):
serialized = simple_workflow.to_json()
workflow = Workflow.from_json(serialized)
assert json.loads(workflow.to_json()) == json.loads(serialized)
def test_simple_workflow_deserialization(self, simple_workflow):
serialized = simple_workflow.to_json()
workflow = Workflow.from_json(serialized)
assert json.loads(workflow.to_json()) == json.loads(serialized)