def test_get_queue_specs(self, get):
node = mock.MagicMock()
batch = EnvBatch()
get.side_effect = [
"1",
"1",
None,
None,
"case.run",
"05:00:00",
"12:00:00",
"false",
]
nodemin, nodemax, jobname, walltimemin, walltimemax, jobmin, jobmax, \
strict = batch.get_queue_specs(node)
self.assertTrue(nodemin == 1)
self.assertTrue(nodemax == 1)
self.assertTrue(jobname == "case.run")
self.assertTrue(walltimemin == "05:00:00")
self.assertTrue(walltimemax == "12:00:00")
self.assertTrue(jobmin == None)
self.assertTrue(jobmax == None)
self.assertFalse(strict)
def check_lockedfiles(caseroot=None):
"""
Check that all lockedfiles match what's in case
If caseroot is not specified, it is set to the current working directory
"""
caseroot = os.getcwd() if caseroot is None else caseroot
lockedfiles = glob.glob(os.path.join(caseroot, "LockedFiles", "*.xml"))
for lfile in lockedfiles:
fpart = os.path.basename(lfile)
# ignore files used for tests such as env_mach_pes.ERP1.xml by looking for extra dots in the name
if fpart.count('.') > 1:
continue
cfile = os.path.join(caseroot, fpart)
if os.path.isfile(cfile):
objname = fpart.split('.')[0]
if objname == "env_build":
f1obj = EnvBuild(caseroot, cfile)
f2obj = EnvBuild(caseroot, lfile)
elif objname == "env_mach_pes":
f1obj = EnvMachPes(caseroot, cfile)
f2obj = EnvMachPes(caseroot, lfile)
elif objname == "env_case":
f1obj = EnvCase(caseroot, cfile)
f2obj = EnvCase(caseroot, lfile)
elif objname == "env_batch":
f1obj = EnvBatch(caseroot, cfile)
f2obj = EnvBatch(caseroot, lfile)
else:
logging.warn("Locked XML file '%s' is not current being handled" % fpart)
continue
diffs = f1obj.compare_xml(f2obj)
if diffs:
logging.warn("File %s has been modified"%lfile)
for key in diffs.keys():
print(" found difference in %s : case %s locked %s" %
(key, repr(diffs[key][0]), repr(diffs[key][1])))
if objname == "env_mach_pes":
expect(False, "Invoke case.setup --reset ")
elif objname == "env_case":
expect(False, "Cannot change file env_case.xml, please"
" recover the original copy from LockedFiles")
elif objname == "env_build":
logging.warn("Setting build complete to False")
f1obj.set_value("BUILD_COMPLETE", False)
if "PIO_VERSION" in diffs.keys():
f1obj.set_value("BUILD_STATUS", 2)
f1obj.write()
logging.critical("Changing PIO_VERSION requires running "
"case.build --clean-all and rebuilding")
else:
f1obj.set_value("BUILD_STATUS", 1)
f1obj.write()
elif objname == "env_batch":
expect(False, "Batch configuration has changed, please run case.setup --reset")
else:
expect(False, "'%s' diff was not handled" % objname)
def read_xml(self):
if (len(self._env_files_that_need_rewrite) > 0):
files = ""
for env_file in self._env_files_that_need_rewrite:
files += " " + env_file.filename
expect(
False,
"Object(s) %s seem to have newer data than the corresponding case file"
% files)
self._env_entryid_files = []
self._env_entryid_files.append(EnvCase(self._caseroot,
components=None))
components = self._env_entryid_files[0].get_values("COMP_CLASSES")
self._env_entryid_files.append(
EnvRun(self._caseroot, components=components))
self._env_entryid_files.append(
EnvBuild(self._caseroot, components=components))
self._env_entryid_files.append(
EnvMachPes(self._caseroot, components=components))
if os.path.isfile(os.path.join(self._caseroot, "env_test.xml")):
self._env_entryid_files.append(
EnvTest(self._caseroot, components=components))
self._env_generic_files = []
self._env_generic_files.append(EnvBatch(self._caseroot))
self._env_generic_files.append(EnvMachSpecific(self._caseroot))
self._env_generic_files.append(EnvArchive(self._caseroot))
self._files = self._env_entryid_files + self._env_generic_files
def check_lockedfile(self, filebase):
caseroot = self.get_value("CASEROOT")
cfile = os.path.join(caseroot, filebase)
lfile = os.path.join(caseroot, "LockedFiles", filebase)
components = self.get_values("COMP_CLASSES")
if os.path.isfile(cfile):
objname = filebase.split('.')[0]
if objname == "env_build":
f1obj = self.get_env('build')
f2obj = EnvBuild(caseroot, lfile)
elif objname == "env_mach_pes":
f1obj = self.get_env('mach_pes')
f2obj = EnvMachPes(caseroot, lfile, components=components)
elif objname == "env_case":
f1obj = self.get_env('case')
f2obj = EnvCase(caseroot, lfile)
elif objname == "env_batch":
f1obj = self.get_env('batch')
f2obj = EnvBatch(caseroot, lfile)
else:
logging.warning("Locked XML file '{}' is not current being handled".format(filebase))
return
diffs = f1obj.compare_xml(f2obj)
if diffs:
logging.warning("File {} has been modified".format(lfile))
toggle_build_status = False
for key in diffs.keys():
if key != "BUILD_COMPLETE":
print(" found difference in {} : case {} locked {}"
.format(key, repr(diffs[key][0]), repr(diffs[key][1])))
toggle_build_status = True
if objname == "env_mach_pes":
expect(False, "Invoke case.setup --reset ")
elif objname == "env_case":
expect(False, "Cannot change file env_case.xml, please"
" recover the original copy from LockedFiles")
elif objname == "env_build":
if toggle_build_status:
logging.warning("Setting build complete to False")
self.set_value("BUILD_COMPLETE", False)
if "PIO_VERSION" in diffs:
self.set_value("BUILD_STATUS", 2)
logging.critical("Changing PIO_VERSION requires running "
"case.build --clean-all and rebuilding")
else:
self.set_value("BUILD_STATUS", 1)
f2obj.set_value("BUILD_COMPLETE", False)
elif objname == "env_batch":
expect(False, "Batch configuration has changed, please run case.setup --reset")
else:
expect(False, "'{}' diff was not handled".format(objname))
class TestXMLEnvBatch(unittest.TestCase):
@mock.patch("CIME.XML.env_batch.EnvBatch.get")
def test_get_queue_specs(self, get):
node = mock.MagicMock()
batch = EnvBatch()
get.side_effect = [
"1",
"1",
None,
None,
"case.run",
"08:00:00",
"05:00:00",
"12:00:00",
"false",
]
(
nodemin,
nodemax,
jobname,
walltimedef,
walltimemin,
walltimemax,
jobmin,
jobmax,
strict,
) = batch.get_queue_specs(node)
self.assertTrue(nodemin == 1)
self.assertTrue(nodemax == 1)
self.assertTrue(jobname == "case.run")
self.assertTrue(walltimedef == "08:00:00")
self.assertTrue(walltimemin == "05:00:00")
self.assertTrue(walltimemax == "12:00:00")
self.assertTrue(jobmin == None)
self.assertTrue(jobmax == None)
self.assertFalse(strict)
@mock.patch("CIME.XML.env_batch.EnvBatch.text", return_value="default")
# nodemin, nodemax, jobname, walltimemin, walltimemax, jobmin, jobmax, strict
@mock.patch(
"CIME.XML.env_batch.EnvBatch.get_queue_specs",
return_value=[
1,
1,
"case.run",
"10:00:00",
"08:00:00",
"12:00:00",
1,
1,
False,
],
)
@mock.patch("CIME.XML.env_batch.EnvBatch.select_best_queue")
@mock.patch("CIME.XML.env_batch.EnvBatch.get_default_queue")
def test_set_job_defaults_honor_walltimemax(self, get_default_queue,
select_best_queue,
get_queue_specs, text):
case = mock.MagicMock()
batch_jobs = [(
"case.run",
{
"template": "template.case.run",
"prereq": "$BUILD_COMPLETE and not $TEST",
},
)]
def get_value(*args, **kwargs):
if args[0] == "USER_REQUESTED_WALLTIME":
return "20:00:00"
return mock.MagicMock()
case.get_value = get_value
case.get_env.return_value.get_jobs.return_value = ["case.run"]
batch = EnvBatch()
batch.set_job_defaults(batch_jobs, case)
env_workflow = case.get_env.return_value
env_workflow.set_value.assert_any_call("JOB_QUEUE",
"default",
subgroup="case.run",
ignore_type=False)
env_workflow.set_value.assert_any_call("JOB_WALLCLOCK_TIME",
"20:00:00",
subgroup="case.run")
"template": "template.case.run",
"prereq": "$BUILD_COMPLETE and not $TEST",
},
)]
def get_value(*args, **kwargs):
if args[0] == "USER_REQUESTED_WALLTIME":
return "05:00:00"
return mock.MagicMock()
case.get_value = get_value
case.get_env.return_value.get_jobs.return_value = ["case.run"]
batch = EnvBatch()
batch.set_job_defaults(batch_jobs, case)
env_workflow = case.get_env.return_value
env_workflow.set_value.assert_any_call("JOB_QUEUE",
"default",
subgroup="case.run",
ignore_type=False)
env_workflow.set_value.assert_any_call("JOB_WALLCLOCK_TIME",
"05:00:00",
subgroup="case.run")
@mock.patch("CIME.XML.env_batch.EnvBatch.text", return_value="default")
# nodemin, nodemax, jobname, walltimemax, jobmin, jobmax, strict
def check_lockedfiles(case):
"""
Check that all lockedfiles match what's in case
If caseroot is not specified, it is set to the current working directory
"""
caseroot = case.get_value("CASEROOT")
lockedfiles = glob.glob(os.path.join(caseroot, "LockedFiles", "*.xml"))
for lfile in lockedfiles:
fpart = os.path.basename(lfile)
# ignore files used for tests such as env_mach_pes.ERP1.xml by looking for extra dots in the name
if fpart.count('.') > 1:
continue
cfile = os.path.join(caseroot, fpart)
components = case.get_values("COMP_CLASSES")
if os.path.isfile(cfile):
objname = fpart.split('.')[0]
if objname == "env_build":
f1obj = case.get_env('build')
f2obj = EnvBuild(caseroot, lfile)
elif objname == "env_mach_pes":
f1obj = case.get_env('mach_pes')
f2obj = EnvMachPes(caseroot, lfile, components=components)
elif objname == "env_case":
f1obj = case.get_env('case')
f2obj = EnvCase(caseroot, lfile)
elif objname == "env_batch":
f1obj = case.get_env('batch')
f2obj = EnvBatch(caseroot, lfile)
else:
logging.warning("Locked XML file '{}' is not current being handled".format(fpart))
continue
diffs = f1obj.compare_xml(f2obj)
if diffs:
logging.warning("File {} has been modified".format(lfile))
toggle_build_status = False
for key in diffs.keys():
if key != "BUILD_COMPLETE":
print(" found difference in {} : case {} locked {}"
.format(key, repr(diffs[key][0]), repr(diffs[key][1])))
toggle_build_status = True
if objname == "env_mach_pes":
expect(False, "Invoke case.setup --reset ")
elif objname == "env_case":
expect(False, "Cannot change file env_case.xml, please"
" recover the original copy from LockedFiles")
elif objname == "env_build":
logging.warning("Setting build complete to False")
case.set_value("BUILD_COMPLETE", False)
if "PIO_VERSION" in diffs:
case.set_value("BUILD_STATUS", 2)
logging.critical("Changing PIO_VERSION requires running "
"case.build --clean-all and rebuilding")
elif toggle_build_status:
case.set_value("BUILD_STATUS", 1)
f2obj.set_value("BUILD_COMPLETE", False)
elif objname == "env_batch":
expect(False, "Batch configuration has changed, please run case.setup --reset")
else:
expect(False, "'{}' diff was not handled".format(objname))
lfile = os.path.join(caseroot, "LockedFiles", filebase)
components = case.get_values("COMP_CLASSES")
if os.path.isfile(cfile):
objname = filebase.split('.')[0]
if objname == "env_build":
f1obj = case.get_env('build')
f2obj = EnvBuild(caseroot, lfile)
elif objname == "env_mach_pes":
f1obj = case.get_env('mach_pes')
f2obj = EnvMachPes(caseroot, lfile, components=components)
elif objname == "env_case":
f1obj = case.get_env('case')
f2obj = EnvCase(caseroot, lfile)
elif objname == "env_batch":
f1obj = case.get_env('batch')
f2obj = EnvBatch(caseroot, lfile)
else:
logging.warning("Locked XML file '{}' is not current being handled".format(filebase))
return
diffs = f1obj.compare_xml(f2obj)
if diffs:
logging.warning("File {} has been modified".format(lfile))
toggle_build_status = False
for key in diffs.keys():
if key != "BUILD_COMPLETE":
print(" found difference in {} : case {} locked {}"
.format(key, repr(diffs[key][0]), repr(diffs[key][1])))
toggle_build_status = True
if objname == "env_mach_pes":
