def test_run_phase_internal_calls_multisubmit_phase1(self):
# Make sure that the correct calls are made to methods stubbed out by
# SystemTestsCompareTwoFake (when runs succeed), when we have a
# multi-submit test, in the first phase
# Setup
run_one_suffix = 'base'
run_two_suffix = 'run2'
case1root, _ = self.get_caseroots()
case1 = CaseFake(case1root)
mytest = SystemTestsCompareTwoFake(case1=case1,
run_one_suffix=run_one_suffix,
run_two_suffix=run_two_suffix,
multisubmit=True)
# RESUBMIT=1 signals first phase
case1.set_value("RESUBMIT", 1)
# Exercise
mytest.run()
# Verify
expected_calls = [
Call(METHOD_case_one_custom_prerun_action, {}),
Call(METHOD_run_indv, {
'suffix': run_one_suffix,
'CASEROOT': case1root
}),
Call(METHOD_case_one_custom_postrun_action, {}),
]
self.assertEqual(expected_calls, mytest.log)
# Also verify that comparison is NOT called:
compare_phase_name = self.get_compare_phase_name(mytest)
self.assertEqual(test_status.TEST_PEND_STATUS,
mytest._test_status.get_status(compare_phase_name))
def test_setup(self):
# Ensure that test setup properly sets up case 1 and case 2
# Setup
case1root = os.path.join(self.tempdir, 'case1')
case1 = CaseFake(case1root)
case1.set_value('var_preset', 'preset_value')
# Exercise
mytest = SystemTestsCompareTwoFake(case1)
# Verify
# Make sure that pre-existing values in case1 are copied to case2 (via
# clone)
self.assertEqual('preset_value',
mytest._case2.get_value('var_preset'))
# Make sure that _common_setup is called for both
self.assertEqual('common_val',
mytest._case1.get_value('var_set_in_common_setup'))
self.assertEqual('common_val',
mytest._case2.get_value('var_set_in_common_setup'))
# Make sure that _case_one_setup and _case_two_setup are called
# appropriately
self.assertEqual('case1val',
mytest._case1.get_value('var_set_in_setup'))
self.assertEqual('case2val',
mytest._case2.get_value('var_set_in_setup'))
def test_run_phase_internal_calls_multisubmit_phase2(self):
# Make sure that the correct calls are made to methods stubbed out by
# SystemTestsCompareTwoFake (when runs succeed), when we have a
# multi-submit test, in the second phase
# Setup
run_one_suffix = 'base'
run_two_suffix = 'run2'
case1root, case2root = self.get_caseroots()
case1 = CaseFake(case1root)
mytest = SystemTestsCompareTwoFake(
case1 = case1,
run_one_suffix = run_one_suffix,
run_two_suffix = run_two_suffix,
multisubmit = True)
# RESUBMIT=0 signals second phase
case1.set_value("RESUBMIT", 0)
# Exercise
mytest.run()
# Verify
expected_calls = [
Call(METHOD_case_two_custom_prerun_action, {}),
Call(METHOD_run_indv,
{'suffix': run_two_suffix, 'CASEROOT': case2root}),
Call(METHOD_case_two_custom_postrun_action, {}),
Call(METHOD_link_to_case2_output, {}),
Call(METHOD_component_compare_test,
{'suffix1': run_one_suffix, 'suffix2': run_two_suffix})
]
self.assertEqual(expected_calls, mytest.log)
def test_setup(self):
# Ensure that test setup properly sets up case 1 and case 2
# Setup
case1root = os.path.join(self.tempdir, 'case1')
case1 = CaseFake(case1root)
case1.set_value('var_preset', 'preset_value')
# Exercise
mytest = SystemTestsCompareTwoFake(case1)
# Verify
# Make sure that pre-existing values in case1 are copied to case2 (via
# clone)
self.assertEqual('preset_value', mytest._case2.get_value('var_preset'))
# Make sure that _common_setup is called for both
self.assertEqual('common_val',
mytest._case1.get_value('var_set_in_common_setup'))
self.assertEqual('common_val',
mytest._case2.get_value('var_set_in_common_setup'))
# Make sure that _case_one_setup and _case_two_setup are called
# appropriately
self.assertEqual('case1val',
mytest._case1.get_value('var_set_in_setup'))
self.assertEqual('case2val',
mytest._case2.get_value('var_set_in_setup'))
def test_run_phase_internal_calls_multisubmit_phase1(self):
# Make sure that the correct calls are made to methods stubbed out by
# SystemTestsCompareTwoFake (when runs succeed), when we have a
# multi-submit test, in the first phase
# Setup
run_one_suffix = 'base'
run_two_suffix = 'run2'
case1root, _ = self.get_caseroots()
case1 = CaseFake(case1root)
mytest = SystemTestsCompareTwoFake(
case1 = case1,
run_one_suffix = run_one_suffix,
run_two_suffix = run_two_suffix,
multisubmit = True)
# RESUBMIT=1 signals first phase
case1.set_value("RESUBMIT", 1)
# Exercise
mytest.run()
# Verify
expected_calls = [
Call(METHOD_case_one_custom_prerun_action, {}),
Call(METHOD_run_indv,
{'suffix': run_one_suffix, 'CASEROOT': case1root}),
Call(METHOD_case_one_custom_postrun_action, {}),
]
self.assertEqual(expected_calls, mytest.log)
# Also verify that comparison is NOT called:
compare_phase_name = self.get_compare_phase_name(mytest)
self.assertEqual(test_status.TEST_PEND_STATUS, mytest._test_status.get_status(compare_phase_name))
def test_setup_separate_builds_sharedlibroot(self):
# If we're using separate_builds, the two cases should still use
# the same sharedlibroot
# Setup
case1root, _ = self.get_caseroots()
case1 = CaseFake(case1root)
case1.set_value("SHAREDLIBROOT",
os.path.join(case1root, "sharedlibroot"))
# Exercise
mytest = SystemTestsCompareTwoFake(case1, separate_builds=True)
# Verify
self.assertEqual(case1.get_value("SHAREDLIBROOT"),
mytest._case2.get_value("SHAREDLIBROOT"))
def test_setup_separate_builds_sharedlibroot(self):
# If we're using separate_builds, the two cases should still use
# the same sharedlibroot
# Setup
case1root, _ = self.get_caseroots()
case1 = CaseFake(case1root)
case1.set_value("SHAREDLIBROOT", os.path.join(case1root, "sharedlibroot"))
# Exercise
mytest = SystemTestsCompareTwoFake(case1,
separate_builds = True)
# Verify
self.assertEqual(case1.get_value("SHAREDLIBROOT"),
mytest._case2.get_value("SHAREDLIBROOT"))
def test_run_phase_internal_calls(self):
# Make sure that the correct calls are made to methods stubbed out by
# SystemTestsCompareTwoFake (when runs succeed)
#
# The point of this is: A number of methods called from the run_phase
# method are stubbed out in the Fake test implementation, because their
# actions are awkward in these unit tests. But we still want to make
# sure that those methods actually got called correctly.
# Setup
run_one_suffix = 'base'
run_two_suffix = 'run2'
case1root, case2root = self.get_caseroots()
case1 = CaseFake(case1root)
mytest = SystemTestsCompareTwoFake(case1,
run_one_suffix = run_one_suffix,
run_two_suffix = run_two_suffix)
# Exercise
mytest.run()
# Verify
expected_calls = [
Call(METHOD_case_one_custom_prerun_action, {}),
Call(METHOD_run_indv,
{'suffix': run_one_suffix, 'CASEROOT': case1root}),
Call(METHOD_case_one_custom_postrun_action, {}),
Call(METHOD_case_two_custom_prerun_action, {}),
Call(METHOD_run_indv,
{'suffix': run_two_suffix, 'CASEROOT': case2root}),
Call(METHOD_case_two_custom_postrun_action, {}),
Call(METHOD_link_to_case2_output, {})
]
self.assertEqual(expected_calls, mytest.log)
def test_create_clone(self):
# Setup
old_caseroot = os.path.join(self.tempdir, "oldcase")
oldcase = CaseFake(old_caseroot)
oldcase.set_value("foo", "bar")
# Exercise
new_caseroot = os.path.join(self.tempdir, "newcase")
clone = oldcase.create_clone(new_caseroot)
# Verify
self.assertEqual("bar", clone.get_value("foo"))
self.assertEqual("newcase", clone.get_value("CASE"))
self.assertEqual("newcase", clone.get_value("CASEBASEID"))
self.assertEqual(new_caseroot, clone.get_value("CASEROOT"))
self.assertEqual(os.path.join(new_caseroot, "run"),
clone.get_value("RUNDIR"))
def test_create_clone(self):
# Setup
old_caseroot = os.path.join(self.tempdir, 'oldcase')
oldcase = CaseFake(old_caseroot)
oldcase.set_value('foo', 'bar')
# Exercise
new_caseroot = os.path.join(self.tempdir, 'newcase')
clone = oldcase.create_clone(new_caseroot)
# Verify
self.assertEqual('bar', clone.get_value('foo'))
self.assertEqual('newcase', clone.get_value('CASE'))
self.assertEqual('newcase', clone.get_value('CASEBASEID'))
self.assertEqual(new_caseroot, clone.get_value('CASEROOT'))
self.assertEqual(os.path.join(new_caseroot, 'run'),
clone.get_value('RUNDIR'))
def test_create_clone(self):
# Setup
old_caseroot = os.path.join(self.tempdir, 'oldcase')
oldcase = CaseFake(old_caseroot)
oldcase.set_value('foo', 'bar')
# Exercise
new_caseroot = os.path.join(self.tempdir, 'newcase')
clone = oldcase.create_clone(new_caseroot)
# Verify
self.assertEqual('bar', clone.get_value('foo'))
self.assertEqual('newcase', clone.get_value('CASE'))
self.assertEqual('newcase', clone.get_value('CASEBASEID'))
self.assertEqual(new_caseroot, clone.get_value('CASEROOT'))
self.assertEqual(os.path.join(new_caseroot, 'run'),
clone.get_value('RUNDIR'))
def test_run_phase_internal_calls_multisubmit_phase2(self):
# Make sure that the correct calls are made to methods stubbed out by
# SystemTestsCompareTwoFake (when runs succeed), when we have a
# multi-submit test, in the second phase
# Setup
run_one_suffix = "base"
run_two_suffix = "run2"
case1root, case2root = self.get_caseroots()
case1 = CaseFake(case1root)
mytest = SystemTestsCompareTwoFake(
case1=case1,
run_one_suffix=run_one_suffix,
run_two_suffix=run_two_suffix,
multisubmit=True,
compare_should_pass=True,
)
# RESUBMIT=0 signals second phase
case1.set_value("RESUBMIT", 0)
# Exercise
mytest.run()
# Verify
expected_calls = [
Call(METHOD_case_two_custom_prerun_action, {}),
Call(METHOD_run_indv, {
"suffix": run_two_suffix,
"CASEROOT": case2root
}),
Call(METHOD_case_two_custom_postrun_action, {}),
Call(METHOD_link_to_case2_output, {}),
]
self.assertEqual(expected_calls, mytest.log)
# Also verify that comparison is called:
compare_phase_name = self.get_compare_phase_name(mytest)
self.assertEqual(
test_status.TEST_PASS_STATUS,
mytest._test_status.get_status(compare_phase_name),
)
def setup_test_and_directories(self, casename1, run2_suffix):
"""
Returns test object
"""
case1root = os.path.join(self.tempdir, casename1)
case1 = CaseFake(case1root)
mytest = SystemTestsCompareTwoFake(case1, run_two_suffix=run2_suffix)
mytest._case1.make_rundir() #pylint: disable=maybe-no-member
mytest._case2.make_rundir() #pylint: disable=maybe-no-member
return mytest
def test_run2_fails(self):
# Make sure that a failure in run2 is reported correctly
# Setup
case1root = os.path.join(self.tempdir, 'case1')
case1 = CaseFake(case1root)
mytest = SystemTestsCompareTwoFake(case1, run_two_should_pass=False)
# Exercise
mytest.run()
# Verify
self.assertEqual(test_status.TEST_FAIL_STATUS,
mytest._test_status.get_status(test_status.RUN_PHASE))
def test_run_phase_passes(self):
# Make sure the run phase behaves properly when all runs succeed.
# Setup
case1root = os.path.join(self.tempdir, 'case1')
case1 = CaseFake(case1root)
mytest = SystemTestsCompareTwoFake(case1)
# Exercise
mytest.run()
# Verify
self.assertEqual(test_status.TEST_PASS_STATUS,
mytest._test_status.get_status(test_status.RUN_PHASE))
def test_compare_fails(self):
# Make sure that a failure in the comparison is reported correctly
# Setup
case1root = os.path.join(self.tempdir, 'case1')
case1 = CaseFake(case1root)
mytest = SystemTestsCompareTwoFake(case1, compare_should_pass=False)
# Exercise
mytest.run()
# Verify
compare_phase_name = self.get_compare_phase_name(mytest)
self.assertEqual(test_status.TEST_FAIL_STATUS,
mytest._test_status.get_status(compare_phase_name))
def test_setup_error(self):
# If there is an error in setup, an exception should be raised and the
# case2 directory should be removed
# Setup
case1root = os.path.join(self.tempdir, 'case1')
case1 = CaseFake(case1root)
# Exercise
with self.assertRaises(Exception):
SystemTestsCompareTwoFake(case1,
run_two_suffix='test',
case2setup_raises_exception=True)
# Verify
self.assertFalse(os.path.exists(os.path.join(case1root, 'case1.test')))
def test_run1_fails(self):
# Make sure that a failure in run1 is reported correctly
# Setup
case1root = os.path.join(self.tempdir, "case1")
case1 = CaseFake(case1root)
mytest = SystemTestsCompareTwoFake(case1, run_one_should_pass=False)
# Exercise
try:
mytest.run()
except Exception:
pass
# Verify
self.assertEqual(
test_status.TEST_FAIL_STATUS,
mytest._test_status.get_status(test_status.RUN_PHASE),
)
def test_run_phase_internal_calls(self):
# Make sure that the correct calls are made to methods stubbed out by
# SystemTestsCompareTwoFake (when runs succeed)
#
# The point of this is: A number of methods called from the run_phase
# method are stubbed out in the Fake test implementation, because their
# actions are awkward in these unit tests. But we still want to make
# sure that those methods actually got called correctly.
# Setup
run_one_suffix = 'base'
run_two_suffix = 'run2'
casename = 'mytest'
case1root = os.path.join(self.tempdir, casename)
case1 = CaseFake(case1root)
mytest = SystemTestsCompareTwoFake(case1,
run_one_suffix=run_one_suffix,
run_two_suffix=run_two_suffix)
# Exercise
mytest.run()
# Verify
expected_calls = [
Call(METHOD_run_indv, {
'suffix': run_one_suffix,
'CASE': casename
}),
Call(
METHOD_run_indv, {
'suffix': run_two_suffix,
'CASE': '{}.{}'.format(casename, run_two_suffix)
}),
Call(METHOD_link_to_case2_output, {}),
Call(METHOD_component_compare_test, {
'suffix1': run_one_suffix,
'suffix2': run_two_suffix
})
]
self.assertEqual(expected_calls, mytest.log)
def test_setup_case2_exists(self):
# If case2 already exists, then setup code should not be called
# Setup
case1root = os.path.join(self.tempdir, 'case1')
case1 = CaseFake(case1root)
os.makedirs(os.path.join(case1root, 'case1.test'))
# Exercise
mytest = SystemTestsCompareTwoFake(case1, run_two_suffix='test')
# Verify:
# Make sure that case2 object is set (i.e., that it doesn't remain None)
self.assertEqual('case1.test', mytest._case2.get_value('CASE'))
# Variables set in various setup methods should not be set
# (In the real world - i.e., outside of this unit testing fakery - these
# values would be set when the Case objects are created.)
self.assertIsNone(mytest._case1.get_value('var_set_in_common_setup'))
self.assertIsNone(mytest._case2.get_value('var_set_in_common_setup'))
self.assertIsNone(mytest._case1.get_value('var_set_in_setup'))
self.assertIsNone(mytest._case2.get_value('var_set_in_setup'))
def _case_from_existing_caseroot(self, caseroot):
"""
Returns a CaseFake object instead of a Case object
"""
return CaseFake(caseroot, create_case_root=False)
def test_internal_calls_multisubmit_failed_state(self):
run_one_suffix = 'base'
run_two_suffix = 'run2'
case1root, _ = self.get_caseroots()
case1 = CaseFake(case1root)
def _set_initial_test_values(x):
x.set_value('RESUBMIT', 1)
case1.set_initial_test_values = functools.partial(
_set_initial_test_values, case1)
# Standard first phase
case1.set_value('IS_FIRST_RUN', True)
case1.set_value('RESUBMIT', 1)
mytest = SystemTestsCompareTwoFake(case1=case1,
run_one_suffix=run_one_suffix,
run_two_suffix=run_two_suffix,
multisubmit=True)
mytest.run()
expected_calls = [
Call(METHOD_case_one_custom_prerun_action, {}),
Call(METHOD_run_indv, {
'CASEROOT': case1root,
'suffix': 'base'
}),
Call(METHOD_case_one_custom_postrun_action, {})
]
self.assertEqual(expected_calls, mytest.log)
# Emulate a rerun ensure phase 1 still runs
case1.set_value('IS_FIRST_RUN', True)
case1.set_value('RESUBMIT', 0)
# Reset the log
mytest.log = []
mytest.run()
expected_calls = [
Call(METHOD_case_one_custom_prerun_action, {}),
Call(METHOD_run_indv, {
'CASEROOT': case1root,
'suffix': 'base'
}),
Call(METHOD_case_one_custom_postrun_action, {})
]
self.assertEqual(expected_calls, mytest.log)