def _check_pelayouts_require_rebuild(case, models):
###############################################################################
"""
Create if we require a rebuild, expects cwd is caseroot
"""
locked_pes = "LockedFiles/env_mach_pes.xml"
if os.path.exists(locked_pes):
# Look to see if $comp_PE_CHANGE_REQUIRES_REBUILD is defined
# for any component
env_mach_pes_locked = EnvMachPes(infile=locked_pes)
for comp in models:
if case.get_value("%s_PE_CHANGE_REQUIRES_REBUILD" % comp):
# Changing these values in env_mach_pes.xml will force
# you to clean the corresponding component
old_tasks = env_mach_pes_locked.get_value("NTASKS_%s" % comp)
old_threads = env_mach_pes_locked.get_value("NTHRDS_%s" % comp)
old_inst = env_mach_pes_locked.get_value("NINST_%s" % comp)
new_tasks = case.get_value("NTASKS_%s" % comp)
new_threads = case.get_value("NTHRDS_%s" % comp)
new_inst = case.get_value("NINST_%s" % comp)
if old_tasks != new_tasks or old_threads != new_threads or old_inst != new_inst:
logger.warn("%s pe change requires clean build" % comp)
cleanflag = comp.lower()
run_cmd_no_fail("./case.build --clean %s" % cleanflag)
os.remove(locked_pes)
def check_pelayouts_require_rebuild(self, models):
"""
Create if we require a rebuild, expects cwd is caseroot
"""
locked_pes = os.path.join(LOCKED_DIR, "env_mach_pes.xml")
if os.path.exists(locked_pes):
# Look to see if $comp_PE_CHANGE_REQUIRES_REBUILD is defined
# for any component
env_mach_pes_locked = EnvMachPes(infile=locked_pes, components=self.get_values("COMP_CLASSES"))
for comp in models:
if self.get_value("{}_PE_CHANGE_REQUIRES_REBUILD".format(comp)):
# Changing these values in env_mach_pes.xml will force
# you to clean the corresponding component
old_tasks = env_mach_pes_locked.get_value("NTASKS_{}".format(comp))
old_threads = env_mach_pes_locked.get_value("NTHRDS_{}".format(comp))
old_inst = env_mach_pes_locked.get_value("NINST_{}".format(comp))
new_tasks = self.get_value("NTASKS_{}".format(comp))
new_threads = self.get_value("NTHRDS_{}".format(comp))
new_inst = self.get_value("NINST_{}".format(comp))
if old_tasks != new_tasks or old_threads != new_threads or old_inst != new_inst:
logging.warning("{} pe change requires clean build {} {}".format(comp, old_tasks, new_tasks))
cleanflag = comp.lower()
run_cmd_no_fail("./case.build --clean {}".format(cleanflag))
unlock_file("env_mach_pes.xml", self.get_value("CASEROOT"))
def check_lockedfiles(caseroot=None):
"""
Check that all lockedfiles match what's in case
If caseroot is not specified, it is set to the current working directory
"""
caseroot = os.getcwd() if caseroot is None else caseroot
lockedfiles = glob.glob(os.path.join(caseroot, "LockedFiles", "*.xml"))
for lfile in lockedfiles:
fpart = os.path.basename(lfile)
# ignore files used for tests such as env_mach_pes.ERP1.xml by looking for extra dots in the name
if fpart.count('.') > 1:
continue
cfile = os.path.join(caseroot, fpart)
if os.path.isfile(cfile):
objname = fpart.split('.')[0]
if objname == "env_build":
f1obj = EnvBuild(caseroot, cfile)
f2obj = EnvBuild(caseroot, lfile)
elif objname == "env_mach_pes":
f1obj = EnvMachPes(caseroot, cfile)
f2obj = EnvMachPes(caseroot, lfile)
elif objname == "env_case":
f1obj = EnvCase(caseroot, cfile)
f2obj = EnvCase(caseroot, lfile)
elif objname == "env_batch":
f1obj = EnvBatch(caseroot, cfile)
f2obj = EnvBatch(caseroot, lfile)
else:
logging.warn("Locked XML file '%s' is not current being handled" % fpart)
continue
diffs = f1obj.compare_xml(f2obj)
if diffs:
logging.warn("File %s has been modified"%lfile)
for key in diffs.keys():
print(" found difference in %s : case %s locked %s" %
(key, repr(diffs[key][0]), repr(diffs[key][1])))
if objname == "env_mach_pes":
expect(False, "Invoke case.setup --reset ")
elif objname == "env_case":
expect(False, "Cannot change file env_case.xml, please"
" recover the original copy from LockedFiles")
elif objname == "env_build":
logging.warn("Setting build complete to False")
f1obj.set_value("BUILD_COMPLETE", False)
if "PIO_VERSION" in diffs.keys():
f1obj.set_value("BUILD_STATUS", 2)
f1obj.write()
logging.critical("Changing PIO_VERSION requires running "
"case.build --clean-all and rebuilding")
else:
f1obj.set_value("BUILD_STATUS", 1)
f1obj.write()
elif objname == "env_batch":
expect(False, "Batch configuration has changed, please run case.setup --reset")
else:
expect(False, "'%s' diff was not handled" % objname)
def read_xml(self):
if (len(self._env_files_that_need_rewrite) > 0):
files = ""
for env_file in self._env_files_that_need_rewrite:
files += " " + env_file.filename
expect(
False,
"Object(s) %s seem to have newer data than the corresponding case file"
% files)
self._env_entryid_files = []
self._env_entryid_files.append(EnvCase(self._caseroot,
components=None))
components = self._env_entryid_files[0].get_values("COMP_CLASSES")
self._env_entryid_files.append(
EnvRun(self._caseroot, components=components))
self._env_entryid_files.append(
EnvBuild(self._caseroot, components=components))
self._env_entryid_files.append(
EnvMachPes(self._caseroot, components=components))
if os.path.isfile(os.path.join(self._caseroot, "env_test.xml")):
self._env_entryid_files.append(
EnvTest(self._caseroot, components=components))
self._env_generic_files = []
self._env_generic_files.append(EnvBatch(self._caseroot))
self._env_generic_files.append(EnvMachSpecific(self._caseroot))
self._env_generic_files.append(EnvArchive(self._caseroot))
self._files = self._env_entryid_files + self._env_generic_files
def check_lockedfile(self, filebase):
caseroot = self.get_value("CASEROOT")
cfile = os.path.join(caseroot, filebase)
lfile = os.path.join(caseroot, "LockedFiles", filebase)
components = self.get_values("COMP_CLASSES")
if os.path.isfile(cfile):
objname = filebase.split('.')[0]
if objname == "env_build":
f1obj = self.get_env('build')
f2obj = EnvBuild(caseroot, lfile)
elif objname == "env_mach_pes":
f1obj = self.get_env('mach_pes')
f2obj = EnvMachPes(caseroot, lfile, components=components)
elif objname == "env_case":
f1obj = self.get_env('case')
f2obj = EnvCase(caseroot, lfile)
elif objname == "env_batch":
f1obj = self.get_env('batch')
f2obj = EnvBatch(caseroot, lfile)
else:
logging.warning("Locked XML file '{}' is not current being handled".format(filebase))
return
diffs = f1obj.compare_xml(f2obj)
if diffs:
logging.warning("File {} has been modified".format(lfile))
toggle_build_status = False
for key in diffs.keys():
if key != "BUILD_COMPLETE":
print(" found difference in {} : case {} locked {}"
.format(key, repr(diffs[key][0]), repr(diffs[key][1])))
toggle_build_status = True
if objname == "env_mach_pes":
expect(False, "Invoke case.setup --reset ")
elif objname == "env_case":
expect(False, "Cannot change file env_case.xml, please"
" recover the original copy from LockedFiles")
elif objname == "env_build":
if toggle_build_status:
logging.warning("Setting build complete to False")
self.set_value("BUILD_COMPLETE", False)
if "PIO_VERSION" in diffs:
self.set_value("BUILD_STATUS", 2)
logging.critical("Changing PIO_VERSION requires running "
"case.build --clean-all and rebuilding")
else:
self.set_value("BUILD_STATUS", 1)
f2obj.set_value("BUILD_COMPLETE", False)
elif objname == "env_batch":
expect(False, "Batch configuration has changed, please run case.setup --reset")
else:
expect(False, "'{}' diff was not handled".format(objname))
shutil.copyfile(os.path.join(caseroot, LOCKED_DIR, filename),
os.path.join(caseroot, newname))
# relock the restored file if names diffs
if newname != filename:
lock_file(newname, caseroot)
def check_pelayouts_require_rebuild(case, models):
"""
Create if we require a rebuild, expects cwd is caseroot
"""
locked_pes = os.path.join(LOCKED_DIR, "env_mach_pes.xml")
if os.path.exists(locked_pes):
# Look to see if $comp_PE_CHANGE_REQUIRES_REBUILD is defined
# for any component
env_mach_pes_locked = EnvMachPes(
infile=locked_pes, components=case.get_values("COMP_CLASSES"))
for comp in models:
if case.get_value("{}_PE_CHANGE_REQUIRES_REBUILD".format(comp)):
# Changing these values in env_mach_pes.xml will force
# you to clean the corresponding component
old_tasks = env_mach_pes_locked.get_value(
"NTASKS_{}".format(comp))
old_threads = env_mach_pes_locked.get_value(
"NTHRDS_{}".format(comp))
old_inst = env_mach_pes_locked.get_value(
"NINST_{}".format(comp))
new_tasks = case.get_value("NTASKS_{}".format(comp))
new_threads = case.get_value("NTHRDS_{}".format(comp))
new_inst = case.get_value("NINST_{}".format(comp))
import shutil, time, glob
logger = logging.getLogger(__name__)
###############################################################################
def _check_pelayouts_require_rebuild(case, models):
###############################################################################
"""
Create if we require a rebuild, expects cwd is caseroot
"""
locked_pes = "LockedFiles/env_mach_pes.xml"
if os.path.exists(locked_pes):
# Look to see if $comp_PE_CHANGE_REQUIRES_REBUILD is defined
# for any component
env_mach_pes_locked = EnvMachPes(infile=locked_pes)
for comp in models:
if case.get_value("%s_PE_CHANGE_REQUIRES_REBUILD" % comp):
# Changing these values in env_mach_pes.xml will force
# you to clean the corresponding component
old_tasks = env_mach_pes_locked.get_value("NTASKS_%s" % comp)
old_threads = env_mach_pes_locked.get_value("NTHRDS_%s" % comp)
old_inst = env_mach_pes_locked.get_value("NINST_%s" % comp)
new_tasks = case.get_value("NTASKS_%s" % comp)
new_threads = case.get_value("NTHRDS_%s" % comp)
new_inst = case.get_value("NINST_%s" % comp)
if old_tasks != new_tasks or old_threads != new_threads or old_inst != new_inst:
logger.warn("%s pe change requires clean build" % comp)
cleanflag = comp.lower()
def check_lockedfiles(case):
"""
Check that all lockedfiles match what's in case
If caseroot is not specified, it is set to the current working directory
"""
caseroot = case.get_value("CASEROOT")
lockedfiles = glob.glob(os.path.join(caseroot, "LockedFiles", "*.xml"))
for lfile in lockedfiles:
fpart = os.path.basename(lfile)
# ignore files used for tests such as env_mach_pes.ERP1.xml by looking for extra dots in the name
if fpart.count('.') > 1:
continue
cfile = os.path.join(caseroot, fpart)
components = case.get_values("COMP_CLASSES")
if os.path.isfile(cfile):
objname = fpart.split('.')[0]
if objname == "env_build":
f1obj = case.get_env('build')
f2obj = EnvBuild(caseroot, lfile)
elif objname == "env_mach_pes":
f1obj = case.get_env('mach_pes')
f2obj = EnvMachPes(caseroot, lfile, components=components)
elif objname == "env_case":
f1obj = case.get_env('case')
f2obj = EnvCase(caseroot, lfile)
elif objname == "env_batch":
f1obj = case.get_env('batch')
f2obj = EnvBatch(caseroot, lfile)
else:
logging.warning("Locked XML file '{}' is not current being handled".format(fpart))
continue
diffs = f1obj.compare_xml(f2obj)
if diffs:
logging.warning("File {} has been modified".format(lfile))
toggle_build_status = False
for key in diffs.keys():
if key != "BUILD_COMPLETE":
print(" found difference in {} : case {} locked {}"
.format(key, repr(diffs[key][0]), repr(diffs[key][1])))
toggle_build_status = True
if objname == "env_mach_pes":
expect(False, "Invoke case.setup --reset ")
elif objname == "env_case":
expect(False, "Cannot change file env_case.xml, please"
" recover the original copy from LockedFiles")
elif objname == "env_build":
logging.warning("Setting build complete to False")
case.set_value("BUILD_COMPLETE", False)
if "PIO_VERSION" in diffs:
case.set_value("BUILD_STATUS", 2)
logging.critical("Changing PIO_VERSION requires running "
"case.build --clean-all and rebuilding")
elif toggle_build_status:
case.set_value("BUILD_STATUS", 1)
f2obj.set_value("BUILD_COMPLETE", False)
elif objname == "env_batch":
expect(False, "Batch configuration has changed, please run case.setup --reset")
else:
expect(False, "'{}' diff was not handled".format(objname))
caseroot = os.getcwd() if caseroot is None else caseroot
newname = filename if newname is None else newname
shutil.copyfile(os.path.join(caseroot, LOCKED_DIR, filename), os.path.join(caseroot, newname))
# relock the restored file if names diffs
if newname != filename:
lock_file(newname, caseroot)
def check_pelayouts_require_rebuild(case, models):
"""
Create if we require a rebuild, expects cwd is caseroot
"""
locked_pes = os.path.join(LOCKED_DIR, "env_mach_pes.xml")
if os.path.exists(locked_pes):
# Look to see if $comp_PE_CHANGE_REQUIRES_REBUILD is defined
# for any component
env_mach_pes_locked = EnvMachPes(infile=locked_pes, components=case.get_values("COMP_CLASSES"))
for comp in models:
if case.get_value("%s_PE_CHANGE_REQUIRES_REBUILD" % comp):
# Changing these values in env_mach_pes.xml will force
# you to clean the corresponding component
old_tasks = env_mach_pes_locked.get_value("NTASKS_%s" % comp)
old_threads = env_mach_pes_locked.get_value("NTHRDS_%s" % comp)
old_inst = env_mach_pes_locked.get_value("NINST_%s" % comp)
new_tasks = case.get_value("NTASKS_%s" % comp)
new_threads = case.get_value("NTHRDS_%s" % comp)
new_inst = case.get_value("NINST_%s" % comp)
if old_tasks != new_tasks or old_threads != new_threads or old_inst != new_inst:
logging.warn("%s pe change requires clean build %s %s" % (comp, old_tasks, new_tasks))
cleanflag = comp.lower()
logging.debug("Unlocking file {}".format(filename))
def is_locked(filename, caseroot=None):
expect("/" not in filename, "Please just provide basename of locked file")
caseroot = os.getcwd() if caseroot is None else caseroot
return os.path.exists(os.path.join(caseroot, LOCKED_DIR, filename))
def check_pelayouts_require_rebuild(case, models):
"""
Create if we require a rebuild, expects cwd is caseroot
"""
locked_pes = os.path.join(LOCKED_DIR, "env_mach_pes.xml")
if os.path.exists(locked_pes):
# Look to see if $comp_PE_CHANGE_REQUIRES_REBUILD is defined
# for any component
env_mach_pes_locked = EnvMachPes(infile=locked_pes, components=case.get_values("COMP_CLASSES"))
for comp in models:
if case.get_value("{}_PE_CHANGE_REQUIRES_REBUILD".format(comp)):
# Changing these values in env_mach_pes.xml will force
# you to clean the corresponding component
old_tasks = env_mach_pes_locked.get_value("NTASKS_{}".format(comp))
old_threads = env_mach_pes_locked.get_value("NTHRDS_{}".format(comp))
old_inst = env_mach_pes_locked.get_value("NINST_{}".format(comp))
new_tasks = case.get_value("NTASKS_{}".format(comp))
new_threads = case.get_value("NTHRDS_{}".format(comp))
new_inst = case.get_value("NINST_{}".format(comp))
if old_tasks != new_tasks or old_threads != new_threads or old_inst != new_inst:
logging.warning("{} pe change requires clean build {} {}".format(comp, old_tasks, new_tasks))
cleanflag = comp.lower()
