def testSingleParallelInputWithGatherMissingData(self): channel = 'a_parallel_channel' step_run_input = getInputWithPartialTree( mode='gather(2)', channel_name=channel, group=0) run = Run.objects.create(is_leaf=True) run.inputs.add(step_run_input) t = InputCalculator(run) input_sets = t.get_input_sets() self.assertEqual(len(input_sets), 0) #Nothing ready
def testSingleParallelInputWithGatherMissingData(self): channel = 'a_parallel_channel' step_run_input = getInputWithPartialTree(mode='gather(2)', channel_name=channel, group=0) input_nodes = [step_run_input] t = InputCalculator(input_nodes, channel, []) input_sets = t.get_input_sets() self.assertEqual(len(input_sets), 0) #Nothing ready
def testGetTargetDataPathNoGather(self): channel = 'channel0' input1 = RunInput.objects.create(channel=channel, group=0, mode='no_gather', type='string') string_data = 'input1_data' data_object = DataObject.get_by_value(string_data, 'string') input_data_path = [(0, 2), (1, 2), (3, 5)] input1.add_data_object(input_data_path, data_object) target_data_path = InputCalculator._gather( input_data_path, InputCalculator._get_gather_depth(input1)) self.assertTrue(are_paths_equal(target_data_path, input_data_path))
def testSingleParallelInputWithGather(self): channel = 'a_parallel_channel' step_run_input = getInputWithFullTree(mode='gather(2)', channel_name=channel, group=0) input_nodes = [step_run_input] t = InputCalculator(input_nodes, channel, []) input_sets = t.get_input_sets() self.assertEqual(len(input_sets), 1) self.assertEqual(input_sets[0].data_path, []) input_items = input_sets[0].input_items self.assertEqual(len(input_items), 1) self.assertEqual(input_items[0].channel, channel) self.assertEqual(input_items[0].data_node.substitution_value, ['i', 'a', 'm', 'r', 'o', 'b', 'o', 't'])
def testSingleScalarInput(self): channel = 'a_scalar_channel' step_run_input = getScalarInput( mode='no_gather', channel_name=channel, group=0) run = Run.objects.create(is_leaf=True) run.inputs.add(step_run_input) t = InputCalculator(run) input_sets = t.get_input_sets() self.assertEqual(len(input_sets), 1) self.assertEqual(input_sets[0].data_path, []) input_items = input_sets[0].input_items self.assertEqual(len(input_items), 1) self.assertEqual(input_items[0].channel, channel) self.assertEqual(input_items[0].data_node.data_object.substitution_value, scalar_input_text)
def testSingleParallelInputWithGather(self): channel = 'a_parallel_channel' step_run_input = getInputWithFullTree( mode='gather(2)', channel_name=channel, group=0) run = Run.objects.create(is_leaf=True) run.inputs.add(step_run_input) t = InputCalculator(run) input_sets = t.get_input_sets() self.assertEqual(len(input_sets), 1) self.assertEqual(input_sets[0].data_path, []) input_items = input_sets[0].input_items self.assertEqual(len(input_items), 1) self.assertEqual(input_items[0].channel, channel) self.assertEqual(input_items[0].data_node.substitution_value, ['i','a','m','r','o','b','o','t'])
def testSingleParallelInput(self): channel = 'a_parallel_channel' step_run_input = getInputWithPartialTree( mode='no_gather', channel_name=channel, group=0) run = Run.objects.create(is_leaf=True) run.inputs.add(step_run_input) t = InputCalculator(run) input_sets = t.get_input_sets() self.assertEqual(len(input_sets), 3) self.assertTrue(are_paths_equal(input_sets[0].data_path, [(0,3),(0,1)])) input_items = input_sets[0].input_items self.assertEqual(len(input_items), 1) self.assertEqual(input_items[0].channel, channel) self.assertEqual(input_items[0].data_node.data_object.substitution_value, 'i')
def testSingleParallelInput(self): channel = 'a_parallel_channel' step_run_input = getInputWithPartialTree(mode='no_gather', channel_name=channel, group=0) input_nodes = [step_run_input] t = InputCalculator(input_nodes, channel, []) input_sets = t.get_input_sets() self.assertEqual(len(input_sets), 3) self.assertTrue( are_paths_equal(input_sets[0].data_path, [(0, 3), (0, 1)])) input_items = input_sets[0].input_items self.assertEqual(len(input_items), 1) self.assertEqual(input_items[0].channel, channel) self.assertEqual( input_items[0].data_node.data_object.substitution_value, 'i')
def testGetTargetDataPathGatherNExceedDepth(self): gather_depth = 4 channel = 'channel1' input1 = getInputWithFullTree(mode='gather(%s)' % gather_depth, channel_name=channel, group=0) # We are using a gather depth greater than the tree height. # This should gather the full tree and not issue error or warning. t = InputCalculator([input1], channel, []) input_sets = t.get_input_sets() self.assertEqual(len(input_sets), 1) input_items = input_sets[0].input_items self.assertEqual(len(input_items), 1) data_node = input_items[0].data_node self.assertEqual(data_node.substitution_value, ['i', 'a', 'm', 'r', 'o', 'b', 'o', 't'])
def testSingleScalarInput(self): channel = 'a_scalar_channel' step_run_input = getScalarInput(mode='no_gather', channel_name=channel, group=0) input_nodes = [step_run_input] t = InputCalculator(input_nodes, channel, []) input_sets = t.get_input_sets() self.assertEqual(len(input_sets), 1) self.assertEqual(input_sets[0].data_path, []) input_items = input_sets[0].input_items self.assertEqual(len(input_items), 1) self.assertEqual(input_items[0].channel, channel) self.assertEqual( input_items[0].data_node.data_object.substitution_value, scalar_input_text)
def testTwoInputsDifferentGroups(self): channel = 'channel1' input1 = getInputWithPartialTree(mode='no_gather', channel_name=channel, group=0) channel2 = 'channel2' input2 = getScalarInput(mode='no_gather', channel_name=channel2, group=1) t = InputCalculator([input1, input2], channel2, []) input_sets = t.get_input_sets() self.assertEqual(len(input_sets), 3) self.assertTrue( are_paths_equal(input_sets[0].data_path, [(0, 3), (0, 1)])) input_items = input_sets[0].input_items self.assertEqual(len(input_items), 2)
def testTwoInputsDifferentGroups(self): channel = 'channel1' input1 = getInputWithPartialTree( mode='no_gather', channel_name=channel, group=0) channel2 = 'channel2' input2 = getScalarInput( mode='no_gather', channel_name=channel2, group=1) run = Run.objects.create(is_leaf=True) run.inputs.add(input1) run.inputs.add(input2) t = InputCalculator(run) input_sets = t.get_input_sets() self.assertEqual(len(input_sets), 3) self.assertTrue(are_paths_equal(input_sets[0].data_path, [(0,3),(0,1)])) input_items = input_sets[0].input_items self.assertEqual(len(input_items), 2)
def testTwoParallelInputsDifferentGroups(self): channel = 'channel1' input1 = getInputWithPartialTree( mode='no_gather', channel_name=channel, group=0) channel2 = 'channel2' input2 = getInputWithFullTree( mode='no_gather', channel_name=channel2, group=1) run = Run.objects.create(is_leaf=True) run.inputs.add(input1) run.inputs.add(input2) t = InputCalculator(run) input_sets = t.get_input_sets() self.assertEqual(len(input_sets), 3*8) self.assertTrue(are_paths_equal( input_sets[0].data_path, [(0,3),(0,1),(0,3),(0,1)])) self.assertTrue(are_paths_equal( input_sets[8].data_path, [(2,3),(0,5),(0,3),(0,1)])) input_items = input_sets[0].input_items self.assertEqual(len(input_items), 2) # Now reverse order of groups channel = 'channel1' input1 = getInputWithPartialTree( mode='no_gather', channel_name=channel, group=1) channel2 = 'channel2' input2 = getInputWithFullTree( mode='no_gather', channel_name=channel2, group=0) run = Run.objects.create(is_leaf=True) run.inputs.add(input1) run.inputs.add(input2) t = InputCalculator(run) input_sets = t.get_input_sets() self.assertEqual(len(input_sets), 8*3) self.assertTrue(are_paths_equal( input_sets[0].data_path, [(0,3),(0,1),(0,3),(0,1)])) self.assertTrue(are_paths_equal( input_sets[4].data_path, [(1,3),(0,2),(2,3),(0,5)])) input_items = input_sets[0].input_items self.assertEqual(len(input_items), 2)
def testTwoParallelInputsDifferentGroupsBothGather(self): channel = 'channel1' input1 = getInputWithPartialTree(mode='gather', channel_name=channel, group=0) channel2 = 'channel2' input2 = getInputWithFullTree(mode='gather', channel_name=channel2, group=1) t = InputCalculator([input1, input2], channel2, []) input_sets = t.get_input_sets() self.assertEqual(len(input_sets), 1 * 3) self.assertTrue( are_paths_equal(input_sets[0].data_path, [(0, 3), (0, 3)])) self.assertTrue( are_paths_equal(input_sets[1].data_path, [(0, 3), (1, 3)])) input_items = input_sets[0].input_items self.assertEqual(len(input_items), 2)
def push(self, channel, data_path): """Called when new data is available at the given data_path on the given channel. This will trigger creation of new tasks if 1) other input data for those tasks is available, and 2) the task with that data_path was not already created previously. """ if get_setting('TEST_NO_CREATE_TASK'): return if not self.is_leaf: return for input_set in InputCalculator(self.inputs.all(), channel, data_path)\ .get_input_sets(): self._push_input_set(input_set)
def testSimple(self): with self.settings(TEST_DISABLE_ASYNC_DELAY=True, TEST_NO_CREATE_TASK=True): run = request_run_from_template_file(os.path.join( os.path.dirname(__file__), '..', '..', 'test', 'fixtures', 'simple', 'simple.yaml'), word_in='puppy') sets = InputCalculator(run.inputs.all(), 'word_in', [])\ .get_input_sets() self.assertEqual(len(sets), 1) self.assertEqual(sets[0].data_path, []) input_items = [item for item in sets[0]] self.assertEqual(len(input_items), 1) self.assertEqual(input_items[0].channel, 'word_in')
def testTwoParallelInputsDifferentGroups(self): channel = 'channel1' input1 = getInputWithPartialTree(mode='no_gather', channel_name=channel, group=0) channel2 = 'channel2' input2 = getInputWithFullTree(mode='no_gather', channel_name=channel2, group=1) t = InputCalculator([input1, input2], channel2, []) input_sets = t.get_input_sets() self.assertEqual(len(input_sets), 3 * 8) self.assertTrue( are_paths_equal(input_sets[0].data_path, [(0, 3), (0, 1), (0, 3), (0, 1)])) self.assertTrue( are_paths_equal(input_sets[8].data_path, [(2, 3), (0, 5), (0, 3), (0, 1)])) input_items = input_sets[0].input_items self.assertEqual(len(input_items), 2) # Now reverse order of groups channel = 'channel1' input1 = getInputWithPartialTree(mode='no_gather', channel_name=channel, group=1) channel2 = 'channel2' input2 = getInputWithFullTree(mode='no_gather', channel_name=channel2, group=0) t = InputCalculator([input1, input2], channel2, []) input_sets = t.get_input_sets() self.assertEqual(len(input_sets), 8 * 3) self.assertTrue( are_paths_equal(input_sets[0].data_path, [(0, 3), (0, 1), (0, 3), (0, 1)])) self.assertTrue( are_paths_equal(input_sets[4].data_path, [(1, 3), (0, 2), (2, 3), (0, 5)])) input_items = input_sets[0].input_items self.assertEqual(len(input_items), 2)