def testSingleParallelInputWithGatherMissingData(self):
channel = 'a_parallel_channel'
step_run_input = getInputWithPartialTree(
mode='gather(2)', channel_name=channel, group=0)
run = Run.objects.create(is_leaf=True)
run.inputs.add(step_run_input)
t = InputCalculator(run)
input_sets = t.get_input_sets()
self.assertEqual(len(input_sets), 0) #Nothing ready
def testSingleParallelInputWithGatherMissingData(self):
channel = 'a_parallel_channel'
step_run_input = getInputWithPartialTree(mode='gather(2)',
channel_name=channel,
group=0)
input_nodes = [step_run_input]
t = InputCalculator(input_nodes, channel, [])
input_sets = t.get_input_sets()
self.assertEqual(len(input_sets), 0) #Nothing ready
def testGetTargetDataPathNoGather(self):
channel = 'channel0'
input1 = RunInput.objects.create(channel=channel,
group=0,
mode='no_gather',
type='string')
string_data = 'input1_data'
data_object = DataObject.get_by_value(string_data, 'string')
input_data_path = [(0, 2), (1, 2), (3, 5)]
input1.add_data_object(input_data_path, data_object)
target_data_path = InputCalculator._gather(
input_data_path, InputCalculator._get_gather_depth(input1))
self.assertTrue(are_paths_equal(target_data_path, input_data_path))
def testSingleParallelInputWithGather(self):
channel = 'a_parallel_channel'
step_run_input = getInputWithFullTree(mode='gather(2)',
channel_name=channel,
group=0)
input_nodes = [step_run_input]
t = InputCalculator(input_nodes, channel, [])
input_sets = t.get_input_sets()
self.assertEqual(len(input_sets), 1)
self.assertEqual(input_sets[0].data_path, [])
input_items = input_sets[0].input_items
self.assertEqual(len(input_items), 1)
self.assertEqual(input_items[0].channel, channel)
self.assertEqual(input_items[0].data_node.substitution_value,
['i', 'a', 'm', 'r', 'o', 'b', 'o', 't'])
def testSingleScalarInput(self):
channel = 'a_scalar_channel'
step_run_input = getScalarInput(
mode='no_gather', channel_name=channel, group=0)
run = Run.objects.create(is_leaf=True)
run.inputs.add(step_run_input)
t = InputCalculator(run)
input_sets = t.get_input_sets()
self.assertEqual(len(input_sets), 1)
self.assertEqual(input_sets[0].data_path, [])
input_items = input_sets[0].input_items
self.assertEqual(len(input_items), 1)
self.assertEqual(input_items[0].channel, channel)
self.assertEqual(input_items[0].data_node.data_object.substitution_value,
scalar_input_text)
def testSingleParallelInputWithGather(self):
channel = 'a_parallel_channel'
step_run_input = getInputWithFullTree(
mode='gather(2)', channel_name=channel, group=0)
run = Run.objects.create(is_leaf=True)
run.inputs.add(step_run_input)
t = InputCalculator(run)
input_sets = t.get_input_sets()
self.assertEqual(len(input_sets), 1)
self.assertEqual(input_sets[0].data_path, [])
input_items = input_sets[0].input_items
self.assertEqual(len(input_items), 1)
self.assertEqual(input_items[0].channel, channel)
self.assertEqual(input_items[0].data_node.substitution_value,
['i','a','m','r','o','b','o','t'])
def testSingleParallelInput(self):
channel = 'a_parallel_channel'
step_run_input = getInputWithPartialTree(
mode='no_gather', channel_name=channel, group=0)
run = Run.objects.create(is_leaf=True)
run.inputs.add(step_run_input)
t = InputCalculator(run)
input_sets = t.get_input_sets()
self.assertEqual(len(input_sets), 3)
self.assertTrue(are_paths_equal(input_sets[0].data_path, [(0,3),(0,1)]))
input_items = input_sets[0].input_items
self.assertEqual(len(input_items), 1)
self.assertEqual(input_items[0].channel, channel)
self.assertEqual(input_items[0].data_node.data_object.substitution_value,
'i')
def testSingleParallelInput(self):
channel = 'a_parallel_channel'
step_run_input = getInputWithPartialTree(mode='no_gather',
channel_name=channel,
group=0)
input_nodes = [step_run_input]
t = InputCalculator(input_nodes, channel, [])
input_sets = t.get_input_sets()
self.assertEqual(len(input_sets), 3)
self.assertTrue(
are_paths_equal(input_sets[0].data_path, [(0, 3), (0, 1)]))
input_items = input_sets[0].input_items
self.assertEqual(len(input_items), 1)
self.assertEqual(input_items[0].channel, channel)
self.assertEqual(
input_items[0].data_node.data_object.substitution_value, 'i')
def testGetTargetDataPathGatherNExceedDepth(self):
gather_depth = 4
channel = 'channel1'
input1 = getInputWithFullTree(mode='gather(%s)' % gather_depth,
channel_name=channel,
group=0)
# We are using a gather depth greater than the tree height.
# This should gather the full tree and not issue error or warning.
t = InputCalculator([input1], channel, [])
input_sets = t.get_input_sets()
self.assertEqual(len(input_sets), 1)
input_items = input_sets[0].input_items
self.assertEqual(len(input_items), 1)
data_node = input_items[0].data_node
self.assertEqual(data_node.substitution_value,
['i', 'a', 'm', 'r', 'o', 'b', 'o', 't'])
def testSingleScalarInput(self):
channel = 'a_scalar_channel'
step_run_input = getScalarInput(mode='no_gather',
channel_name=channel,
group=0)
input_nodes = [step_run_input]
t = InputCalculator(input_nodes, channel, [])
input_sets = t.get_input_sets()
self.assertEqual(len(input_sets), 1)
self.assertEqual(input_sets[0].data_path, [])
input_items = input_sets[0].input_items
self.assertEqual(len(input_items), 1)
self.assertEqual(input_items[0].channel, channel)
self.assertEqual(
input_items[0].data_node.data_object.substitution_value,
scalar_input_text)
def testTwoInputsDifferentGroups(self):
channel = 'channel1'
input1 = getInputWithPartialTree(mode='no_gather',
channel_name=channel,
group=0)
channel2 = 'channel2'
input2 = getScalarInput(mode='no_gather',
channel_name=channel2,
group=1)
t = InputCalculator([input1, input2], channel2, [])
input_sets = t.get_input_sets()
self.assertEqual(len(input_sets), 3)
self.assertTrue(
are_paths_equal(input_sets[0].data_path, [(0, 3), (0, 1)]))
input_items = input_sets[0].input_items
self.assertEqual(len(input_items), 2)
def testTwoInputsDifferentGroups(self):
channel = 'channel1'
input1 = getInputWithPartialTree(
mode='no_gather', channel_name=channel, group=0)
channel2 = 'channel2'
input2 = getScalarInput(
mode='no_gather', channel_name=channel2, group=1)
run = Run.objects.create(is_leaf=True)
run.inputs.add(input1)
run.inputs.add(input2)
t = InputCalculator(run)
input_sets = t.get_input_sets()
self.assertEqual(len(input_sets), 3)
self.assertTrue(are_paths_equal(input_sets[0].data_path, [(0,3),(0,1)]))
input_items = input_sets[0].input_items
self.assertEqual(len(input_items), 2)
def testTwoParallelInputsDifferentGroups(self):
channel = 'channel1'
input1 = getInputWithPartialTree(
mode='no_gather', channel_name=channel, group=0)
channel2 = 'channel2'
input2 = getInputWithFullTree(
mode='no_gather', channel_name=channel2, group=1)
run = Run.objects.create(is_leaf=True)
run.inputs.add(input1)
run.inputs.add(input2)
t = InputCalculator(run)
input_sets = t.get_input_sets()
self.assertEqual(len(input_sets), 3*8)
self.assertTrue(are_paths_equal(
input_sets[0].data_path, [(0,3),(0,1),(0,3),(0,1)]))
self.assertTrue(are_paths_equal(
input_sets[8].data_path, [(2,3),(0,5),(0,3),(0,1)]))
input_items = input_sets[0].input_items
self.assertEqual(len(input_items), 2)
# Now reverse order of groups
channel = 'channel1'
input1 = getInputWithPartialTree(
mode='no_gather', channel_name=channel, group=1)
channel2 = 'channel2'
input2 = getInputWithFullTree(
mode='no_gather', channel_name=channel2, group=0)
run = Run.objects.create(is_leaf=True)
run.inputs.add(input1)
run.inputs.add(input2)
t = InputCalculator(run)
input_sets = t.get_input_sets()
self.assertEqual(len(input_sets), 8*3)
self.assertTrue(are_paths_equal(
input_sets[0].data_path, [(0,3),(0,1),(0,3),(0,1)]))
self.assertTrue(are_paths_equal(
input_sets[4].data_path, [(1,3),(0,2),(2,3),(0,5)]))
input_items = input_sets[0].input_items
self.assertEqual(len(input_items), 2)
def testTwoParallelInputsDifferentGroupsBothGather(self):
channel = 'channel1'
input1 = getInputWithPartialTree(mode='gather',
channel_name=channel,
group=0)
channel2 = 'channel2'
input2 = getInputWithFullTree(mode='gather',
channel_name=channel2,
group=1)
t = InputCalculator([input1, input2], channel2, [])
input_sets = t.get_input_sets()
self.assertEqual(len(input_sets), 1 * 3)
self.assertTrue(
are_paths_equal(input_sets[0].data_path, [(0, 3), (0, 3)]))
self.assertTrue(
are_paths_equal(input_sets[1].data_path, [(0, 3), (1, 3)]))
input_items = input_sets[0].input_items
self.assertEqual(len(input_items), 2)
def push(self, channel, data_path):
"""Called when new data is available at the given data_path
on the given channel. This will trigger creation of new tasks if 1)
other input data for those tasks is available, and 2) the task with
that data_path was not already created previously.
"""
if get_setting('TEST_NO_CREATE_TASK'):
return
if not self.is_leaf:
return
for input_set in InputCalculator(self.inputs.all(), channel, data_path)\
.get_input_sets():
self._push_input_set(input_set)
def testSimple(self):
with self.settings(TEST_DISABLE_ASYNC_DELAY=True,
TEST_NO_CREATE_TASK=True):
run = request_run_from_template_file(os.path.join(
os.path.dirname(__file__), '..', '..', 'test', 'fixtures',
'simple', 'simple.yaml'),
word_in='puppy')
sets = InputCalculator(run.inputs.all(), 'word_in', [])\
.get_input_sets()
self.assertEqual(len(sets), 1)
self.assertEqual(sets[0].data_path, [])
input_items = [item for item in sets[0]]
self.assertEqual(len(input_items), 1)
self.assertEqual(input_items[0].channel, 'word_in')
def testTwoParallelInputsDifferentGroups(self):
channel = 'channel1'
input1 = getInputWithPartialTree(mode='no_gather',
channel_name=channel,
group=0)
channel2 = 'channel2'
input2 = getInputWithFullTree(mode='no_gather',
channel_name=channel2,
group=1)
t = InputCalculator([input1, input2], channel2, [])
input_sets = t.get_input_sets()
self.assertEqual(len(input_sets), 3 * 8)
self.assertTrue(
are_paths_equal(input_sets[0].data_path, [(0, 3), (0, 1), (0, 3),
(0, 1)]))
self.assertTrue(
are_paths_equal(input_sets[8].data_path, [(2, 3), (0, 5), (0, 3),
(0, 1)]))
input_items = input_sets[0].input_items
self.assertEqual(len(input_items), 2)
# Now reverse order of groups
channel = 'channel1'
input1 = getInputWithPartialTree(mode='no_gather',
channel_name=channel,
group=1)
channel2 = 'channel2'
input2 = getInputWithFullTree(mode='no_gather',
channel_name=channel2,
group=0)
t = InputCalculator([input1, input2], channel2, [])
input_sets = t.get_input_sets()
self.assertEqual(len(input_sets), 8 * 3)
self.assertTrue(
are_paths_equal(input_sets[0].data_path, [(0, 3), (0, 1), (0, 3),
(0, 1)]))
self.assertTrue(
are_paths_equal(input_sets[4].data_path, [(1, 3), (0, 2), (2, 3),
(0, 5)]))
input_items = input_sets[0].input_items
self.assertEqual(len(input_items), 2)