def simple_workflow():
class SumTask(Task):
name = 'test:sum'
inputs = [
IntParameter('int1', required=True),
IntParameter('int2', required=True)
]
outputs = [IntParameter('sum')]
def execute(self, int1, int2):
return int1 + int2
workflow = Workflow()
workflow.inputs = [
IntParameter('int1', required=True),
IntParameter('int2', required=True),
IntParameter('int3', required=True)
]
workflow.outputs = [IntParameter('total')]
workflow.add_node('sum_1', SumTask(), {
'int1': ('input', 'int1'),
'int2': ('input', 'int2')
})
workflow.add_node('sum_2', SumTask(), {
'int1': ('input', 'int3'),
'int2': ('dependency', ('sum_1', 'sum'))
})
workflow.map_output('sum_2', 'sum', 'total')
return workflow
def simple_workflow():
class SumTask(Task):
name = "test:sum"
inputs = [IntParameter("int1", required=True), IntParameter("int2", required=True)]
outputs = [IntParameter("sum")]
def execute(self, int1, int2):
return int1 + int2
workflow = Workflow()
workflow.inputs = [
IntParameter("int1", required=True),
IntParameter("int2", required=True),
IntParameter("int3", required=True),
]
workflow.outputs = [IntParameter("total")]
workflow.add_node("sum_1", SumTask(), {"int1": ("input", "int1"), "int2": ("input", "int2")})
workflow.add_node("sum_2", SumTask(), {"int1": ("input", "int3"), "int2": ("dependency", ("sum_1", "sum"))})
workflow.map_output("sum_2", "sum", "total")
return workflow
def get_task_instance(job_name):
if job_name not in REGISTERED_JOBS:
return None
job_info = REGISTERED_JOBS[job_name]
if not isinstance(job_info, dict) or 'type' not in job_info:
raise ImproperlyConfigured('NC_REGISTERED_JOBS configuration is invalid.')
if job_info['type'] == 'task':
class_path = job_info.get('task')
if not class_path:
raise ImproperlyConfigured('Registered job {} does not specify a task.'.format(job_name))
try:
module_name, class_name = class_path.rsplit('.', 1)
module = import_module(module_name)
cls = getattr(module, class_name)
except (ImportError, ValueError, AttributeError):
raise ImproperlyConfigured('{} is not a valid task.'.format(class_path))
return cls()
elif job_info['type'] == 'workflow':
path = job_info.get('path')
if not path or not os.path.isfile(path):
raise ImproperlyConfigured('The workflow {} does not exist.'.format(path))
with open(path, 'r') as f:
return Workflow.from_json(f.read())
else:
raise ImproperlyConfigured('Invalid job type: {}'.format(job_info['type']))
def test_map_reduce_workflow(self):
with open(os.path.join(TEST_DATA_DIR, "map_reduce_workflow.json"), "r") as f:
workflow = Workflow.from_json(f.read())
arr_1 = numpy.arange(10)
arr_2 = numpy.arange(10, 20)
arr_3 = numpy.arange(20, 30)
expected = sum([x / float(numpy.max(x)) for x in [arr_1, arr_2, arr_3]])
result = workflow(arrays_in=[arr_1, arr_2, arr_3])
array_out = result["array_out"]
assert is_ndarray(array_out)
assert (array_out == expected).all()
def test_map_reduce_workflow(self):
with open(os.path.join(TEST_DATA_DIR, 'map_reduce_workflow.json'),
'r') as f:
workflow = Workflow.from_json(f.read())
arr_1 = numpy.arange(10)
arr_2 = numpy.arange(10, 20)
arr_3 = numpy.arange(20, 30)
expected = sum(
[x / float(numpy.max(x)) for x in [arr_1, arr_2, arr_3]])
result = workflow(arrays_in=[arr_1, arr_2, arr_3])
array_out = result['array_out']
assert is_ndarray(array_out)
assert (array_out == expected).all()
def test_simple_workflow_deserialization(self, simple_workflow):
serialized = simple_workflow.to_json()
workflow = Workflow.from_json(serialized)
assert json.loads(workflow.to_json()) == json.loads(serialized)
def test_simple_workflow_deserialization(self, simple_workflow):
serialized = simple_workflow.to_json()
workflow = Workflow.from_json(serialized)
assert json.loads(workflow.to_json()) == json.loads(serialized)
