def test_run_phase_internal_calls_multisubmit_phase1(self):
        # Make sure that the correct calls are made to methods stubbed out by
        # SystemTestsCompareTwoFake (when runs succeed), when we have a
        # multi-submit test, in the first phase

        # Setup
        run_one_suffix = 'base'
        run_two_suffix = 'run2'
        case1root, _ = self.get_caseroots()
        case1 = CaseFake(case1root)
        mytest = SystemTestsCompareTwoFake(case1=case1,
                                           run_one_suffix=run_one_suffix,
                                           run_two_suffix=run_two_suffix,
                                           multisubmit=True)
        # RESUBMIT=1 signals first phase
        case1.set_value("RESUBMIT", 1)

        # Exercise
        mytest.run()

        # Verify
        expected_calls = [
            Call(METHOD_case_one_custom_prerun_action, {}),
            Call(METHOD_run_indv, {
                'suffix': run_one_suffix,
                'CASEROOT': case1root
            }),
            Call(METHOD_case_one_custom_postrun_action, {}),
        ]
        self.assertEqual(expected_calls, mytest.log)

        # Also verify that comparison is NOT called:
        compare_phase_name = self.get_compare_phase_name(mytest)
        self.assertEqual(test_status.TEST_PEND_STATUS,
                         mytest._test_status.get_status(compare_phase_name))
    def test_setup(self):
        # Ensure that test setup properly sets up case 1 and case 2

        # Setup
        case1root = os.path.join(self.tempdir, 'case1')
        case1 = CaseFake(case1root)
        case1.set_value('var_preset', 'preset_value')

        # Exercise
        mytest = SystemTestsCompareTwoFake(case1)

        # Verify
        # Make sure that pre-existing values in case1 are copied to case2 (via
        # clone)
        self.assertEqual('preset_value',
                         mytest._case2.get_value('var_preset'))

        # Make sure that _common_setup is called for both
        self.assertEqual('common_val',
                         mytest._case1.get_value('var_set_in_common_setup'))
        self.assertEqual('common_val',
                         mytest._case2.get_value('var_set_in_common_setup'))

        # Make sure that _case_one_setup and _case_two_setup are called
        # appropriately
        self.assertEqual('case1val',
                         mytest._case1.get_value('var_set_in_setup'))
        self.assertEqual('case2val',
                         mytest._case2.get_value('var_set_in_setup'))
    def test_run_phase_internal_calls_multisubmit_phase2(self):
        # Make sure that the correct calls are made to methods stubbed out by
        # SystemTestsCompareTwoFake (when runs succeed), when we have a
        # multi-submit test, in the second phase

        # Setup
        run_one_suffix = 'base'
        run_two_suffix = 'run2'
        case1root, case2root = self.get_caseroots()
        case1 = CaseFake(case1root)
        mytest = SystemTestsCompareTwoFake(
            case1 = case1,
            run_one_suffix = run_one_suffix,
            run_two_suffix = run_two_suffix,
            multisubmit = True)
        # RESUBMIT=0 signals second phase
        case1.set_value("RESUBMIT", 0)

        # Exercise
        mytest.run()

        # Verify
        expected_calls = [
            Call(METHOD_case_two_custom_prerun_action, {}),
            Call(METHOD_run_indv,
                 {'suffix': run_two_suffix, 'CASEROOT': case2root}),
            Call(METHOD_case_two_custom_postrun_action, {}),
            Call(METHOD_link_to_case2_output, {}),
            Call(METHOD_component_compare_test,
                {'suffix1': run_one_suffix, 'suffix2': run_two_suffix})
        ]
        self.assertEqual(expected_calls, mytest.log)
Exemple #4
0
    def test_setup(self):
        # Ensure that test setup properly sets up case 1 and case 2

        # Setup
        case1root = os.path.join(self.tempdir, 'case1')
        case1 = CaseFake(case1root)
        case1.set_value('var_preset', 'preset_value')

        # Exercise
        mytest = SystemTestsCompareTwoFake(case1)

        # Verify
        # Make sure that pre-existing values in case1 are copied to case2 (via
        # clone)
        self.assertEqual('preset_value', mytest._case2.get_value('var_preset'))

        # Make sure that _common_setup is called for both
        self.assertEqual('common_val',
                         mytest._case1.get_value('var_set_in_common_setup'))
        self.assertEqual('common_val',
                         mytest._case2.get_value('var_set_in_common_setup'))

        # Make sure that _case_one_setup and _case_two_setup are called
        # appropriately
        self.assertEqual('case1val',
                         mytest._case1.get_value('var_set_in_setup'))
        self.assertEqual('case2val',
                         mytest._case2.get_value('var_set_in_setup'))
    def test_run_phase_internal_calls_multisubmit_phase1(self):
        # Make sure that the correct calls are made to methods stubbed out by
        # SystemTestsCompareTwoFake (when runs succeed), when we have a
        # multi-submit test, in the first phase

        # Setup
        run_one_suffix = 'base'
        run_two_suffix = 'run2'
        case1root, _ = self.get_caseroots()
        case1 = CaseFake(case1root)
        mytest = SystemTestsCompareTwoFake(
            case1 = case1,
            run_one_suffix = run_one_suffix,
            run_two_suffix = run_two_suffix,
            multisubmit = True)
        # RESUBMIT=1 signals first phase
        case1.set_value("RESUBMIT", 1)

        # Exercise
        mytest.run()

        # Verify
        expected_calls = [
            Call(METHOD_case_one_custom_prerun_action, {}),
            Call(METHOD_run_indv,
                 {'suffix': run_one_suffix, 'CASEROOT': case1root}),
            Call(METHOD_case_one_custom_postrun_action, {}),
        ]
        self.assertEqual(expected_calls, mytest.log)

        # Also verify that comparison is NOT called:
        compare_phase_name = self.get_compare_phase_name(mytest)
        self.assertEqual(test_status.TEST_PEND_STATUS, mytest._test_status.get_status(compare_phase_name))
    def test_setup_separate_builds_sharedlibroot(self):
        # If we're using separate_builds, the two cases should still use
        # the same sharedlibroot

        # Setup
        case1root, _ = self.get_caseroots()
        case1 = CaseFake(case1root)
        case1.set_value("SHAREDLIBROOT",
                        os.path.join(case1root, "sharedlibroot"))

        # Exercise
        mytest = SystemTestsCompareTwoFake(case1, separate_builds=True)

        # Verify
        self.assertEqual(case1.get_value("SHAREDLIBROOT"),
                         mytest._case2.get_value("SHAREDLIBROOT"))
    def test_setup_separate_builds_sharedlibroot(self):
        # If we're using separate_builds, the two cases should still use
        # the same sharedlibroot

        # Setup
        case1root, _ = self.get_caseroots()
        case1 = CaseFake(case1root)
        case1.set_value("SHAREDLIBROOT", os.path.join(case1root, "sharedlibroot"))

        # Exercise
        mytest = SystemTestsCompareTwoFake(case1,
                                           separate_builds = True)

        # Verify
        self.assertEqual(case1.get_value("SHAREDLIBROOT"),
                         mytest._case2.get_value("SHAREDLIBROOT"))
    def test_run_phase_internal_calls(self):
        # Make sure that the correct calls are made to methods stubbed out by
        # SystemTestsCompareTwoFake (when runs succeed)
        #
        # The point of this is: A number of methods called from the run_phase
        # method are stubbed out in the Fake test implementation, because their
        # actions are awkward in these unit tests. But we still want to make
        # sure that those methods actually got called correctly.

        # Setup
        run_one_suffix = 'base'
        run_two_suffix = 'run2'
        case1root, case2root = self.get_caseroots()
        case1 = CaseFake(case1root)
        mytest = SystemTestsCompareTwoFake(case1,
            run_one_suffix = run_one_suffix,
            run_two_suffix = run_two_suffix)

        # Exercise
        mytest.run()

        # Verify
        expected_calls = [
            Call(METHOD_case_one_custom_prerun_action, {}),
            Call(METHOD_run_indv,
                 {'suffix': run_one_suffix, 'CASEROOT': case1root}),
            Call(METHOD_case_one_custom_postrun_action, {}),
            Call(METHOD_case_two_custom_prerun_action, {}),
            Call(METHOD_run_indv,
                 {'suffix': run_two_suffix, 'CASEROOT': case2root}),
            Call(METHOD_case_two_custom_postrun_action, {}),
            Call(METHOD_link_to_case2_output, {})
        ]
        self.assertEqual(expected_calls, mytest.log)
    def test_create_clone(self):
        # Setup
        old_caseroot = os.path.join(self.tempdir, "oldcase")
        oldcase = CaseFake(old_caseroot)
        oldcase.set_value("foo", "bar")

        # Exercise
        new_caseroot = os.path.join(self.tempdir, "newcase")
        clone = oldcase.create_clone(new_caseroot)

        # Verify
        self.assertEqual("bar", clone.get_value("foo"))
        self.assertEqual("newcase", clone.get_value("CASE"))
        self.assertEqual("newcase", clone.get_value("CASEBASEID"))
        self.assertEqual(new_caseroot, clone.get_value("CASEROOT"))
        self.assertEqual(os.path.join(new_caseroot, "run"),
                         clone.get_value("RUNDIR"))
Exemple #10
0
    def test_create_clone(self):
        # Setup
        old_caseroot = os.path.join(self.tempdir, 'oldcase')
        oldcase = CaseFake(old_caseroot)
        oldcase.set_value('foo', 'bar')

        # Exercise
        new_caseroot = os.path.join(self.tempdir, 'newcase')
        clone = oldcase.create_clone(new_caseroot)

        # Verify
        self.assertEqual('bar', clone.get_value('foo'))
        self.assertEqual('newcase', clone.get_value('CASE'))
        self.assertEqual('newcase', clone.get_value('CASEBASEID'))
        self.assertEqual(new_caseroot, clone.get_value('CASEROOT'))
        self.assertEqual(os.path.join(new_caseroot, 'run'),
                         clone.get_value('RUNDIR'))
Exemple #11
0
    def test_create_clone(self):
        # Setup
        old_caseroot = os.path.join(self.tempdir, 'oldcase')
        oldcase = CaseFake(old_caseroot)
        oldcase.set_value('foo', 'bar')

        # Exercise
        new_caseroot = os.path.join(self.tempdir, 'newcase')
        clone = oldcase.create_clone(new_caseroot)

        # Verify
        self.assertEqual('bar', clone.get_value('foo'))
        self.assertEqual('newcase', clone.get_value('CASE'))
        self.assertEqual('newcase', clone.get_value('CASEBASEID'))
        self.assertEqual(new_caseroot, clone.get_value('CASEROOT'))
        self.assertEqual(os.path.join(new_caseroot, 'run'),
                         clone.get_value('RUNDIR'))
Exemple #12
0
    def test_run_phase_internal_calls_multisubmit_phase2(self):
        # Make sure that the correct calls are made to methods stubbed out by
        # SystemTestsCompareTwoFake (when runs succeed), when we have a
        # multi-submit test, in the second phase

        # Setup
        run_one_suffix = "base"
        run_two_suffix = "run2"
        case1root, case2root = self.get_caseroots()
        case1 = CaseFake(case1root)
        mytest = SystemTestsCompareTwoFake(
            case1=case1,
            run_one_suffix=run_one_suffix,
            run_two_suffix=run_two_suffix,
            multisubmit=True,
            compare_should_pass=True,
        )
        # RESUBMIT=0 signals second phase
        case1.set_value("RESUBMIT", 0)

        # Exercise
        mytest.run()

        # Verify
        expected_calls = [
            Call(METHOD_case_two_custom_prerun_action, {}),
            Call(METHOD_run_indv, {
                "suffix": run_two_suffix,
                "CASEROOT": case2root
            }),
            Call(METHOD_case_two_custom_postrun_action, {}),
            Call(METHOD_link_to_case2_output, {}),
        ]
        self.assertEqual(expected_calls, mytest.log)

        # Also verify that comparison is called:
        compare_phase_name = self.get_compare_phase_name(mytest)
        self.assertEqual(
            test_status.TEST_PASS_STATUS,
            mytest._test_status.get_status(compare_phase_name),
        )
    def setup_test_and_directories(self, casename1, run2_suffix):
        """
        Returns test object
        """

        case1root = os.path.join(self.tempdir, casename1)
        case1 = CaseFake(case1root)
        mytest = SystemTestsCompareTwoFake(case1, run_two_suffix=run2_suffix)
        mytest._case1.make_rundir()  #pylint: disable=maybe-no-member
        mytest._case2.make_rundir()  #pylint: disable=maybe-no-member

        return mytest
Exemple #14
0
    def test_run2_fails(self):
        # Make sure that a failure in run2 is reported correctly

        # Setup
        case1root = os.path.join(self.tempdir, 'case1')
        case1 = CaseFake(case1root)
        mytest = SystemTestsCompareTwoFake(case1, run_two_should_pass=False)

        # Exercise
        mytest.run()

        # Verify
        self.assertEqual(test_status.TEST_FAIL_STATUS,
                         mytest._test_status.get_status(test_status.RUN_PHASE))
Exemple #15
0
    def test_run_phase_passes(self):
        # Make sure the run phase behaves properly when all runs succeed.

        # Setup
        case1root = os.path.join(self.tempdir, 'case1')
        case1 = CaseFake(case1root)
        mytest = SystemTestsCompareTwoFake(case1)

        # Exercise
        mytest.run()

        # Verify
        self.assertEqual(test_status.TEST_PASS_STATUS,
                         mytest._test_status.get_status(test_status.RUN_PHASE))
    def test_compare_fails(self):
        # Make sure that a failure in the comparison is reported correctly

        # Setup
        case1root = os.path.join(self.tempdir, 'case1')
        case1 = CaseFake(case1root)
        mytest = SystemTestsCompareTwoFake(case1, compare_should_pass=False)

        # Exercise
        mytest.run()

        # Verify
        compare_phase_name = self.get_compare_phase_name(mytest)
        self.assertEqual(test_status.TEST_FAIL_STATUS,
                         mytest._test_status.get_status(compare_phase_name))
Exemple #17
0
    def test_setup_error(self):
        # If there is an error in setup, an exception should be raised and the
        # case2 directory should be removed

        # Setup
        case1root = os.path.join(self.tempdir, 'case1')
        case1 = CaseFake(case1root)

        # Exercise
        with self.assertRaises(Exception):
            SystemTestsCompareTwoFake(case1,
                                      run_two_suffix='test',
                                      case2setup_raises_exception=True)

        # Verify
        self.assertFalse(os.path.exists(os.path.join(case1root, 'case1.test')))
Exemple #18
0
    def test_run1_fails(self):
        # Make sure that a failure in run1 is reported correctly

        # Setup
        case1root = os.path.join(self.tempdir, "case1")
        case1 = CaseFake(case1root)
        mytest = SystemTestsCompareTwoFake(case1, run_one_should_pass=False)

        # Exercise
        try:
            mytest.run()
        except Exception:
            pass

        # Verify
        self.assertEqual(
            test_status.TEST_FAIL_STATUS,
            mytest._test_status.get_status(test_status.RUN_PHASE),
        )
Exemple #19
0
    def test_run_phase_internal_calls(self):
        # Make sure that the correct calls are made to methods stubbed out by
        # SystemTestsCompareTwoFake (when runs succeed)
        #
        # The point of this is: A number of methods called from the run_phase
        # method are stubbed out in the Fake test implementation, because their
        # actions are awkward in these unit tests. But we still want to make
        # sure that those methods actually got called correctly.

        # Setup
        run_one_suffix = 'base'
        run_two_suffix = 'run2'
        casename = 'mytest'
        case1root = os.path.join(self.tempdir, casename)
        case1 = CaseFake(case1root)
        mytest = SystemTestsCompareTwoFake(case1,
                                           run_one_suffix=run_one_suffix,
                                           run_two_suffix=run_two_suffix)

        # Exercise
        mytest.run()

        # Verify
        expected_calls = [
            Call(METHOD_run_indv, {
                'suffix': run_one_suffix,
                'CASE': casename
            }),
            Call(
                METHOD_run_indv, {
                    'suffix': run_two_suffix,
                    'CASE': '{}.{}'.format(casename, run_two_suffix)
                }),
            Call(METHOD_link_to_case2_output, {}),
            Call(METHOD_component_compare_test, {
                'suffix1': run_one_suffix,
                'suffix2': run_two_suffix
            })
        ]
        self.assertEqual(expected_calls, mytest.log)
Exemple #20
0
    def test_setup_case2_exists(self):
        # If case2 already exists, then setup code should not be called

        # Setup
        case1root = os.path.join(self.tempdir, 'case1')
        case1 = CaseFake(case1root)
        os.makedirs(os.path.join(case1root, 'case1.test'))

        # Exercise
        mytest = SystemTestsCompareTwoFake(case1, run_two_suffix='test')

        # Verify:

        # Make sure that case2 object is set (i.e., that it doesn't remain None)
        self.assertEqual('case1.test', mytest._case2.get_value('CASE'))

        # Variables set in various setup methods should not be set
        # (In the real world - i.e., outside of this unit testing fakery - these
        # values would be set when the Case objects are created.)
        self.assertIsNone(mytest._case1.get_value('var_set_in_common_setup'))
        self.assertIsNone(mytest._case2.get_value('var_set_in_common_setup'))
        self.assertIsNone(mytest._case1.get_value('var_set_in_setup'))
        self.assertIsNone(mytest._case2.get_value('var_set_in_setup'))
Exemple #21
0
 def _case_from_existing_caseroot(self, caseroot):
     """
     Returns a CaseFake object instead of a Case object
     """
     return CaseFake(caseroot, create_case_root=False)
    def test_internal_calls_multisubmit_failed_state(self):
        run_one_suffix = 'base'
        run_two_suffix = 'run2'
        case1root, _ = self.get_caseroots()
        case1 = CaseFake(case1root)

        def _set_initial_test_values(x):
            x.set_value('RESUBMIT', 1)

        case1.set_initial_test_values = functools.partial(
            _set_initial_test_values, case1)

        # Standard first phase
        case1.set_value('IS_FIRST_RUN', True)
        case1.set_value('RESUBMIT', 1)

        mytest = SystemTestsCompareTwoFake(case1=case1,
                                           run_one_suffix=run_one_suffix,
                                           run_two_suffix=run_two_suffix,
                                           multisubmit=True)

        mytest.run()

        expected_calls = [
            Call(METHOD_case_one_custom_prerun_action, {}),
            Call(METHOD_run_indv, {
                'CASEROOT': case1root,
                'suffix': 'base'
            }),
            Call(METHOD_case_one_custom_postrun_action, {})
        ]

        self.assertEqual(expected_calls, mytest.log)

        # Emulate a rerun ensure phase 1 still runs
        case1.set_value('IS_FIRST_RUN', True)
        case1.set_value('RESUBMIT', 0)

        # Reset the log
        mytest.log = []

        mytest.run()

        expected_calls = [
            Call(METHOD_case_one_custom_prerun_action, {}),
            Call(METHOD_run_indv, {
                'CASEROOT': case1root,
                'suffix': 'base'
            }),
            Call(METHOD_case_one_custom_postrun_action, {})
        ]

        self.assertEqual(expected_calls, mytest.log)