def _do_full_nl_comp(case, test, compare_name, baseline_root=None):
test_dir = case.get_value("CASEROOT")
casedoc_dir = os.path.join(test_dir, "CaseDocs")
baseline_root = case.get_value("BASELINE_ROOT") if baseline_root is None else baseline_root
all_match = True
baseline_dir = os.path.join(baseline_root, compare_name, test)
baseline_casedocs = os.path.join(baseline_dir, "CaseDocs")
# Start off by comparing everything in CaseDocs except a few arbitrary files (ugh!)
# TODO: Namelist files should have consistent suffix
all_items_to_compare = [item for item in glob.glob("{}/*".format(casedoc_dir))\
if "README" not in os.path.basename(item)\
and not item.endswith("doc")\
and not item.endswith("prescribed")\
and not os.path.basename(item).startswith(".")]
comments = "NLCOMP\n"
for item in all_items_to_compare:
baseline_counterpart = os.path.join(baseline_casedocs \
if os.path.dirname(item).endswith("CaseDocs") \
else baseline_dir,os.path.basename(item))
if not os.path.exists(baseline_counterpart):
comments += "Missing baseline namelist '{}'\n".format(baseline_counterpart)
all_match = False
else:
if item.endswith("runconfig") or item.endswith("runseq"):
success, current_comments = compare_runconfigfiles(baseline_counterpart, item, test)
elif is_namelist_file(item):
success, current_comments = compare_namelist_files(baseline_counterpart, item, test)
else:
success, current_comments = compare_files(baseline_counterpart, item, test)
all_match &= success
if not success:
comments += "Comparison failed between '{}' with '{}'\n".format(item, baseline_counterpart)
comments += current_comments
logging.info(comments)
return all_match, comments
def create_clone(self,
newcase,
keepexe=False,
mach_dir=None,
project=None,
cime_output_root=None,
exeroot=None,
rundir=None,
user_mods_dir=None):
"""
Create a case clone
If exeroot or rundir are provided (not None), sets these directories
to the given paths; if not provided, uses default values for these
directories. It is an error to provide exeroot if keepexe is True.
"""
if cime_output_root is None:
cime_output_root = self.get_value("CIME_OUTPUT_ROOT")
newcaseroot = os.path.abspath(newcase)
expect(not os.path.isdir(newcaseroot),
"New caseroot directory {} already exists".format(newcaseroot))
newcasename = os.path.basename(newcaseroot)
expect(check_name(newcasename),
"New case name invalid {} ".format(newcasename))
newcase_cimeroot = os.path.abspath(get_cime_root())
# create clone from case to case
clone_cimeroot = self.get_value("CIMEROOT")
if newcase_cimeroot != clone_cimeroot:
logger.warning(" case CIMEROOT is {} ".format(newcase_cimeroot))
logger.warning(" clone CIMEROOT is {} ".format(clone_cimeroot))
logger.warning(
" It is NOT recommended to clone cases from different versions of CIME."
)
# *** create case object as deepcopy of clone object ***
srcroot = os.path.join(newcase_cimeroot, "..")
newcase = self.copy(newcasename, newcaseroot, newsrcroot=srcroot)
newcase.set_value("CIMEROOT", newcase_cimeroot)
# if we are cloning to a different user modify the output directory
olduser = self.get_value("USER")
newuser = os.environ.get("USER")
if olduser != newuser:
cime_output_root = cime_output_root.replace(olduser, newuser)
newcase.set_value("USER", newuser)
newcase.set_value("CIME_OUTPUT_ROOT", cime_output_root)
# try to make the new output directory and raise an exception
# on any error other than directory already exists.
if os.path.isdir(cime_output_root):
expect(
os.access(cime_output_root, os.W_OK),
"Directory {} is not writable "
"by this user. Use the --cime-output-root flag to provide a writable "
"scratch directory".format(cime_output_root))
else:
try:
os.makedirs(cime_output_root)
except:
if not os.path.isdir(cime_output_root):
raise
# determine if will use clone executable or not
if keepexe:
orig_exeroot = self.get_value("EXEROOT")
newcase.set_value("EXEROOT", orig_exeroot)
newcase.set_value("BUILD_COMPLETE", "TRUE")
orig_bld_complete = self.get_value("BUILD_COMPLETE")
if not orig_bld_complete:
logger.warning(
"\nWARNING: Creating a clone with --keepexe before building the original case may cause PIO_TYPENAME to be invalid in the clone"
)
logger.warning(
"Avoid this message by building case one before you clone.\n")
else:
newcase.set_value("BUILD_COMPLETE", "FALSE")
# set machdir
if mach_dir is not None:
newcase.set_value("MACHDIR", mach_dir)
# set exeroot and rundir if requested
if exeroot is not None:
expect(
not keepexe, "create_case_clone: if keepexe is True, "
"then exeroot cannot be set")
newcase.set_value("EXEROOT", exeroot)
if rundir is not None:
newcase.set_value("RUNDIR", rundir)
# Set project id
# Note: we do not just copy this from the clone because it seems likely that
# users will want to change this sometimes, especially when cloning another
# user's case. However, note that, if a project is not given, the fallback will
# be to copy it from the clone, just like other xml variables are copied.
if project is None:
project = self.get_value("PROJECT", subgroup=self.get_primary_job())
if project is not None:
newcase.set_value("PROJECT", project)
# create caseroot
newcase.create_caseroot(clone=True)
newcase.flush(flushall=True)
# copy user_ files
cloneroot = self.get_case_root()
files = glob.glob(cloneroot + '/user_*')
for item in files:
safe_copy(item, newcaseroot)
# copy SourceMod and Buildconf files
# if symlinks exist, copy rather than follow links
for casesub in ("SourceMods", "Buildconf"):
shutil.copytree(os.path.join(cloneroot, casesub),
os.path.join(newcaseroot, casesub),
symlinks=True)
# lock env_case.xml in new case
lock_file("env_case.xml", newcaseroot)
# apply user_mods if appropriate
newcase_root = newcase.get_value("CASEROOT")
if user_mods_dir is not None:
if keepexe:
# If keepexe CANNOT change any env_build.xml variables - so make a temporary copy of
# env_build.xml and verify that it has not been modified
safe_copy(
os.path.join(newcaseroot, "env_build.xml"),
os.path.join(newcaseroot, "LockedFiles", "env_build.xml"))
# Now apply contents of user_mods directory
apply_user_mods(newcase_root, user_mods_dir, keepexe=keepexe)
# Determine if env_build.xml has changed
if keepexe:
success, comment = compare_files(
os.path.join(newcaseroot, "env_build.xml"),
os.path.join(newcaseroot, "LockedFiles", "env_build.xml"))
if not success:
logger.warning(comment)
shutil.rmtree(newcase_root)
expect(
False,
"env_build.xml cannot be changed via usermods if keepexe is an option: \n "
"Failed to clone case, removed {}\n".format(newcase_root))
# if keep executable, then remove the new case SourceMods directory and link SourceMods to
# the clone directory
if keepexe:
shutil.rmtree(os.path.join(newcase_root, "SourceMods"))
os.symlink(os.path.join(cloneroot, "SourceMods"),
os.path.join(newcase_root, "SourceMods"))
# Update README.case
fclone = open(cloneroot + "/README.case", "r")
fnewcase = open(newcaseroot + "/README.case", "a")
fnewcase.write("\n *** original clone README follows ****")
fnewcase.write("\n " + fclone.read())
clonename = self.get_value("CASE")
logger.info(" Successfully created new case {} from clone case {} ".format(
newcasename, clonename))
newcase.case_setup()
return newcase
newcase_root = newcase.get_value("CASEROOT")
if user_mods_dir is not None:
if keepexe:
# If keepexe CANNOT change any env_build.xml variables - so make a temporary copy of
# env_build.xml and verify that it has not been modified
shutil.copy(
os.path.join(newcaseroot, "env_build.xml"),
os.path.join(newcaseroot, "LockedFiles", "env_build.xml"))
# Now apply contents of user_mods directory
apply_user_mods(newcase_root, user_mods_dir, keepexe=keepexe)
# Determine if env_build.xml has changed
if keepexe:
success, comment = compare_files(
os.path.join(newcaseroot, "env_build.xml"),
os.path.join(newcaseroot, "LockedFiles", "env_build.xml"))
if not success:
logger.warning(comment)
shutil.rmtree(newcase_root)
expect(
False,
"env_build.xml cannot be changed via usermods if keepexe is an option: \n "
"Failed to clone case, removed {}\n".format(newcase_root))
# if keep executable, then remove the new case SourceMods directory and link SourceMods to
# the clone directory
if keepexe:
shutil.rmtree(os.path.join(newcase_root, "SourceMods"))
os.symlink(os.path.join(cloneroot, "SourceMods"),
os.path.join(newcase_root, "SourceMods"))
and not os.path.basename(item).startswith(".")] + \
glob.glob("%s/*user_nl*" % test_dir)
comments = ""
for item in all_items_to_compare:
baseline_counterpart = os.path.join(baseline_casedocs \
if os.path.dirname(item).endswith("CaseDocs") \
else baseline_dir,os.path.basename(item))
if not os.path.exists(baseline_counterpart):
comments += "Missing baseline namelist '%s'\n" % baseline_counterpart
all_match = False
else:
if is_namelist_file(item):
success, current_comments = compare_namelist_files(baseline_counterpart, item, test)
else:
success, current_comments = compare_files(baseline_counterpart, item, test)
all_match &= success
comments += current_comments
logging.info(comments)
return all_match, comments
def _do_full_nl_gen(case, test, generate_name):
test_dir = case.get_value("CASEROOT")
casedoc_dir = os.path.join(test_dir, "CaseDocs")
baseline_root = case.get_value("BASELINE_ROOT")
baseline_dir = os.path.join(baseline_root, generate_name, test)
baseline_casedocs = os.path.join(baseline_dir, "CaseDocs")
def create_clone(self, newcase, keepexe=False, mach_dir=None, project=None,
cime_output_root=None, exeroot=None, rundir=None,
user_mods_dir=None):
"""
Create a case clone
If exeroot or rundir are provided (not None), sets these directories
to the given paths; if not provided, uses default values for these
directories. It is an error to provide exeroot if keepexe is True.
"""
if cime_output_root is None:
cime_output_root = self.get_value("CIME_OUTPUT_ROOT")
newcaseroot = os.path.abspath(newcase)
expect(not os.path.isdir(newcaseroot),
"New caseroot directory {} already exists".format(newcaseroot))
newcasename = os.path.basename(newcaseroot)
expect(check_name(newcasename),
"New case name invalid {} ".format(newcasename))
newcase_cimeroot = os.path.abspath(get_cime_root())
# create clone from case to case
clone_cimeroot = self.get_value("CIMEROOT")
if newcase_cimeroot != clone_cimeroot:
logger.warning(" case CIMEROOT is {} ".format(newcase_cimeroot))
logger.warning(" clone CIMEROOT is {} ".format(clone_cimeroot))
logger.warning(" It is NOT recommended to clone cases from different versions of CIME.")
# *** create case object as deepcopy of clone object ***
srcroot = os.path.join(newcase_cimeroot,"..")
newcase = self.copy(newcasename, newcaseroot, newsrcroot=srcroot)
newcase.set_value("CIMEROOT", newcase_cimeroot)
# if we are cloning to a different user modify the output directory
olduser = self.get_value("USER")
newuser = os.environ.get("USER")
if olduser != newuser:
cime_output_root = cime_output_root.replace(olduser, newuser)
newcase.set_value("USER", newuser)
newcase.set_value("CIME_OUTPUT_ROOT", cime_output_root)
# try to make the new output directory and raise an exception
# on any error other than directory already exists.
if os.path.isdir(cime_output_root):
expect(os.access(cime_output_root, os.W_OK), "Directory {} is not writable "
"by this user. Use the --cime-output-root flag to provide a writable "
"scratch directory".format(cime_output_root))
else:
try:
os.makedirs(cime_output_root)
except:
if not os.path.isdir(cime_output_root):
raise
# determine if will use clone executable or not
if keepexe:
orig_exeroot = self.get_value("EXEROOT")
newcase.set_value("EXEROOT", orig_exeroot)
newcase.set_value("BUILD_COMPLETE","TRUE")
orig_bld_complete = self.get_value("BUILD_COMPLETE")
if not orig_bld_complete:
logger.warning("\nWARNING: Creating a clone with --keepexe before building the original case may cause PIO_TYPENAME to be invalid in the clone")
logger.warning("Avoid this message by building case one before you clone.\n")
else:
newcase.set_value("BUILD_COMPLETE","FALSE")
# set machdir
if mach_dir is not None:
newcase.set_value("MACHDIR", mach_dir)
# set exeroot and rundir if requested
if exeroot is not None:
expect(not keepexe, "create_case_clone: if keepexe is True, "
"then exeroot cannot be set")
newcase.set_value("EXEROOT", exeroot)
if rundir is not None:
newcase.set_value("RUNDIR", rundir)
# Set project id
# Note: we do not just copy this from the clone because it seems likely that
# users will want to change this sometimes, especially when cloning another
# user's case. However, note that, if a project is not given, the fallback will
# be to copy it from the clone, just like other xml variables are copied.
if project is None:
project = self.get_value("PROJECT", subgroup=self.get_primary_job())
if project is not None:
newcase.set_value("PROJECT", project)
# create caseroot
newcase.create_caseroot(clone=True)
newcase.flush(flushall=True)
# copy user_ files
cloneroot = self.get_case_root()
files = glob.glob(cloneroot + '/user_*')
for item in files:
safe_copy(item, newcaseroot)
# copy SourceMod and Buildconf files
# if symlinks exist, copy rather than follow links
for casesub in ("SourceMods", "Buildconf"):
shutil.copytree(os.path.join(cloneroot, casesub),
os.path.join(newcaseroot, casesub),
symlinks=True)
# lock env_case.xml in new case
lock_file("env_case.xml", newcaseroot)
# apply user_mods if appropriate
newcase_root = newcase.get_value("CASEROOT")
if user_mods_dir is not None:
if keepexe:
# If keepexe CANNOT change any env_build.xml variables - so make a temporary copy of
# env_build.xml and verify that it has not been modified
safe_copy(os.path.join(newcaseroot, "env_build.xml"),
os.path.join(newcaseroot, "LockedFiles", "env_build.xml"))
# Now apply contents of user_mods directory
apply_user_mods(newcase_root, user_mods_dir, keepexe=keepexe)
# Determine if env_build.xml has changed
if keepexe:
success, comment = compare_files(os.path.join(newcaseroot, "env_build.xml"),
os.path.join(newcaseroot, "LockedFiles", "env_build.xml"))
if not success:
logger.warning(comment)
shutil.rmtree(newcase_root)
expect(False, "env_build.xml cannot be changed via usermods if keepexe is an option: \n "
"Failed to clone case, removed {}\n".format(newcase_root))
# if keep executable, then remove the new case SourceMods directory and link SourceMods to
# the clone directory
if keepexe:
shutil.rmtree(os.path.join(newcase_root, "SourceMods"))
os.symlink(os.path.join(cloneroot, "SourceMods"),
os.path.join(newcase_root, "SourceMods"))
# Update README.case
fclone = open(cloneroot + "/README.case", "r")
fnewcase = open(newcaseroot + "/README.case", "a")
fnewcase.write("\n *** original clone README follows ****")
fnewcase.write("\n " + fclone.read())
clonename = self.get_value("CASE")
logger.info(" Successfully created new case {} from clone case {} ".format(newcasename, clonename))
newcase.case_setup()
return newcase
