def test_keepexe(self):
     self.createUserMod("foo")
     with six.assertRaisesRegex(self, CIMEError,
                                "cannot have any source mods"):
         apply_user_mods(self._caseroot,
                         os.path.join(self._user_mods_parent_dir, "foo"),
                         keepexe=True)
    def test_duplicate_includes(self):
        """Test multiple includes, where both include the same base mod.

        The base mod should only be included once.
        """

        self.createUserMod("base")
        self.createUserMod("derived1", include_dirs=["base"])
        self.createUserMod("derived2", include_dirs=["base"])
        self.createUserMod("derived_combo",
                           include_dirs = ["derived1", "derived2"])

        apply_user_mods(self._caseroot,
                        os.path.join(self._user_mods_parent_dir, "derived_combo"))

        # NOTE(wjs, 2017-04-15) The ordering of derived1 vs. derived2 is not
        # critical here: If this aspect of the behavior changes, the
        # expected_contents can be changed to match the new behavior in this
        # respect.
        expected_contents = """base
derived2
derived1
derived_combo
"""
        self.assertResults(expected_user_nl_cpl = expected_contents,
                           expected_shell_commands_result = expected_contents,
                           expected_sourcemod = "derived_combo\n",
                           msg = "test_duplicate_includes")
Exemple #3
0
    def test_duplicate_includes(self):
        """Test multiple includes, where both include the same base mod.

        The base mod should only be included once.
        """

        self.createUserMod("base")
        self.createUserMod("derived1", include_dirs=["base"])
        self.createUserMod("derived2", include_dirs=["base"])
        self.createUserMod("derived_combo", include_dirs=["derived1", "derived2"])

        apply_user_mods(
            self._caseroot, os.path.join(self._user_mods_parent_dir, "derived_combo")
        )

        # NOTE(wjs, 2017-04-15) The ordering of derived1 vs. derived2 is not
        # critical here: If this aspect of the behavior changes, the
        # expected_contents can be changed to match the new behavior in this
        # respect.
        expected_contents = """base
derived2
derived1
derived_combo
"""
        self.assertResults(
            expected_user_nl_cpl=expected_contents,
            expected_shell_commands_result=expected_contents,
            expected_sourcemod="derived_combo\n",
            msg="test_duplicate_includes",
        )
 def test_basic(self):
     self.createUserMod("foo")
     apply_user_mods(self._caseroot,
                     os.path.join(self._user_mods_parent_dir, "foo"))
     self.assertResults(expected_user_nl_cpl="foo\n",
                        expected_shell_commands_result="foo\n",
                        expected_sourcemod="foo\n",
                        msg="test_basic")
 def test_basic(self):
     self.createUserMod("foo")
     apply_user_mods(self._caseroot,
                     os.path.join(self._user_mods_parent_dir, "foo"))
     self.assertResults(expected_user_nl_cpl = "foo\n",
                        expected_shell_commands_result = "foo\n",
                        expected_sourcemod = "foo\n",
                        msg = "test_basic")
Exemple #6
0
 def apply_user_mods(self, user_mods_dir=None):
     if user_mods_dir is not None:
         if os.path.isabs(user_mods_dir):
             user_mods_path = user_mods_dir
         else:
             user_mods_path = self.get_value('USER_MODS_DIR')
             user_mods_path = os.path.join(user_mods_path, user_mods_dir)
         self.set_value("USER_MODS_FULLPATH",user_mods_path)
         apply_user_mods(self._caseroot, user_mods_path)
Exemple #7
0
 def apply_user_mods(self, user_mods_dir=None):
     if user_mods_dir is not None:
         if os.path.isabs(user_mods_dir):
             user_mods_path = user_mods_dir
         else:
             user_mods_path = self.get_value('USER_MODS_DIR')
             user_mods_path = os.path.join(user_mods_path, user_mods_dir)
         self.set_value("USER_MODS_FULLPATH", user_mods_path)
         apply_user_mods(self._caseroot, user_mods_path)
    def test_include(self):
        """If there is an included mod, the main one should appear after the included one so that it takes precedence."""

        self.createUserMod("base")
        self.createUserMod("derived", include_dirs=["base"])

        apply_user_mods(self._caseroot,
                        os.path.join(self._user_mods_parent_dir, "derived"))

        self.assertResults(expected_user_nl_cpl = "base\nderived\n",
                           expected_shell_commands_result = "base\nderived\n",
                           expected_sourcemod = "derived\n",
                           msg = "test_include")
    def test_two_applications(self):
        """If apply_user_mods is called twice, the second should appear after the first so that it takes precedence."""

        self.createUserMod("foo1")
        self.createUserMod("foo2")
        apply_user_mods(self._caseroot,
                        os.path.join(self._user_mods_parent_dir, "foo1"))
        apply_user_mods(self._caseroot,
                        os.path.join(self._user_mods_parent_dir, "foo2"))
        self.assertResults(expected_user_nl_cpl = "foo1\nfoo2\n",
                           expected_shell_commands_result = "foo1\nfoo2\n",
                           expected_sourcemod = "foo2\n",
                           msg = "test_two_applications")
    def test_two_applications(self):
        """If apply_user_mods is called twice, the second should appear after the first so that it takes precedence."""

        self.createUserMod("foo1")
        self.createUserMod("foo2")
        apply_user_mods(self._caseroot,
                        os.path.join(self._user_mods_parent_dir, "foo1"))
        apply_user_mods(self._caseroot,
                        os.path.join(self._user_mods_parent_dir, "foo2"))
        self.assertResults(expected_user_nl_cpl="foo1\nfoo2\n",
                           expected_shell_commands_result="foo1\nfoo2\n",
                           expected_sourcemod="foo2\n",
                           msg="test_two_applications")
    def test_include(self):
        """If there is an included mod, the main one should appear after the included one so that it takes precedence."""

        self.createUserMod("base")
        self.createUserMod("derived", include_dirs=["base"])

        apply_user_mods(self._caseroot,
                        os.path.join(self._user_mods_parent_dir, "derived"))

        self.assertResults(expected_user_nl_cpl="base\nderived\n",
                           expected_shell_commands_result="base\nderived\n",
                           expected_sourcemod="derived\n",
                           msg="test_include")
Exemple #12
0
 def apply_user_mods(self, user_mods_dir=None):
     if user_mods_dir is not None:
         if os.path.isabs(user_mods_dir):
             user_mods_path = user_mods_dir
         else:
             user_mods_path = self.get_value('USER_MODS_DIR')
             user_mods_path = os.path.join(user_mods_path, user_mods_dir)
         ninst_vals = {}
         for i in xrange(1,len(self._component_classes)):
             comp_class = self._component_classes[i]
             comp_name  = self._components[i-1]
             if comp_class == "DRV":
                 continue
             ninst_comp = self.get_value("NINST_%s"%comp_class)
             if ninst_comp > 1:
                 ninst_vals[comp_name] = ninst_comp
         apply_user_mods(self.get_value("CASEROOT"), user_mods_path, ninst_vals)
Exemple #13
0
 def apply_user_mods(self, user_mods_dir=None):
     if user_mods_dir is not None:
         if os.path.isabs(user_mods_dir):
             user_mods_path = user_mods_dir
         else:
             user_mods_path = self.get_value('USER_MODS_DIR')
             user_mods_path = os.path.join(user_mods_path, user_mods_dir)
         self.set_value("USER_MODS_FULLPATH",user_mods_path)
         ninst_vals = {}
         for i in xrange(1,len(self._component_classes)):
             comp_class = self._component_classes[i]
             comp_name  = self._components[i-1]
             if comp_class == "DRV":
                 continue
             ninst_comp = self.get_value("NINST_%s"%comp_class)
             if ninst_comp > 1:
                 ninst_vals[comp_name] = ninst_comp
         apply_user_mods(self._caseroot, user_mods_path, ninst_vals)
Exemple #14
0
def create_clone(self, newcase, keepexe=False, mach_dir=None, project=None,
                      cime_output_root=None, exeroot=None, rundir=None,
                      user_mods_dir=None):
    """
    Create a case clone

    If exeroot or rundir are provided (not None), sets these directories
    to the given paths; if not provided, uses default values for these
    directories. It is an error to provide exeroot if keepexe is True.
    """
    if cime_output_root is None:
        cime_output_root = self.get_value("CIME_OUTPUT_ROOT")

    newcaseroot = os.path.abspath(newcase)
    expect(not os.path.isdir(newcaseroot),
           "New caseroot directory {} already exists".format(newcaseroot))
    newcasename = os.path.basename(newcaseroot)
    expect(check_name(newcasename),
           "New case name invalid {} ".format(newcasename))
    newcase_cimeroot = os.path.abspath(get_cime_root())

    # create clone from case to case
    clone_cimeroot = self.get_value("CIMEROOT")
    if newcase_cimeroot != clone_cimeroot:
        logger.warning(" case  CIMEROOT is {} ".format(newcase_cimeroot))
        logger.warning(" clone CIMEROOT is {} ".format(clone_cimeroot))
        logger.warning(" It is NOT recommended to clone cases from different versions of CIME.")

    # *** create case object as deepcopy of clone object ***
    srcroot = os.path.join(newcase_cimeroot,"..")
    newcase = self.copy(newcasename, newcaseroot, newsrcroot=srcroot)
    newcase.set_value("CIMEROOT", newcase_cimeroot)

    # if we are cloning to a different user modify the output directory
    olduser = self.get_value("USER")
    newuser = os.environ.get("USER")
    if olduser != newuser:
        cime_output_root = cime_output_root.replace(olduser, newuser)
        newcase.set_value("USER", newuser)
    newcase.set_value("CIME_OUTPUT_ROOT", cime_output_root)

    # try to make the new output directory and raise an exception
    # on any error other than directory already exists.
    if os.path.isdir(cime_output_root):
        expect(os.access(cime_output_root, os.W_OK), "Directory {} is not writable "
               "by this user.  Use the --cime-output-root flag to provide a writable "
               "scratch directory".format(cime_output_root))
    else:
        try:
            os.makedirs(cime_output_root)
        except:
            if not os.path.isdir(cime_output_root):
                raise

    # determine if will use clone executable or not
    if keepexe:
        orig_exeroot = self.get_value("EXEROOT")
        newcase.set_value("EXEROOT", orig_exeroot)
        newcase.set_value("BUILD_COMPLETE","TRUE")
        orig_bld_complete = self.get_value("BUILD_COMPLETE")
        if not orig_bld_complete:
            logger.warning("\nWARNING: Creating a clone with --keepexe before building the original case may cause PIO_TYPENAME to be invalid in the clone")
            logger.warning("Avoid this message by building case one before you clone.\n")
    else:
        newcase.set_value("BUILD_COMPLETE","FALSE")

    # set machdir
    if mach_dir is not None:
        newcase.set_value("MACHDIR", mach_dir)

    # set exeroot and rundir if requested
    if exeroot is not None:
        expect(not keepexe, "create_case_clone: if keepexe is True, "
               "then exeroot cannot be set")
        newcase.set_value("EXEROOT", exeroot)
    if rundir is not None:
        newcase.set_value("RUNDIR", rundir)

    # Set project id
    # Note: we do not just copy this from the clone because it seems likely that
    # users will want to change this sometimes, especially when cloning another
    # user's case. However, note that, if a project is not given, the fallback will
    # be to copy it from the clone, just like other xml variables are copied.
    if project is None:
        project = self.get_value("PROJECT", subgroup=self.get_primary_job())
    if project is not None:
        newcase.set_value("PROJECT", project)

    # create caseroot
    newcase.create_caseroot(clone=True)
    newcase.flush(flushall=True)

    # copy user_ files
    cloneroot = self.get_case_root()
    files = glob.glob(cloneroot + '/user_*')

    for item in files:
        safe_copy(item, newcaseroot)

    # copy SourceMod and Buildconf files
    # if symlinks exist, copy rather than follow links
    for casesub in ("SourceMods", "Buildconf"):
        shutil.copytree(os.path.join(cloneroot, casesub),
                        os.path.join(newcaseroot, casesub),
                        symlinks=True)

    # lock env_case.xml in new case
    lock_file("env_case.xml", newcaseroot)

    # apply user_mods if appropriate
    newcase_root = newcase.get_value("CASEROOT")
    if user_mods_dir is not None:
        if keepexe:
            # If keepexe CANNOT change any env_build.xml variables - so make a temporary copy of
            # env_build.xml and verify that it has not been modified
            safe_copy(os.path.join(newcaseroot, "env_build.xml"),
                      os.path.join(newcaseroot, "LockedFiles", "env_build.xml"))

        # Now apply contents of user_mods directory
        apply_user_mods(newcase_root, user_mods_dir, keepexe=keepexe)

        # Determine if env_build.xml has changed
        if keepexe:
            success, comment = compare_files(os.path.join(newcaseroot, "env_build.xml"),
                                             os.path.join(newcaseroot, "LockedFiles", "env_build.xml"))
            if not success:
                logger.warning(comment)
                shutil.rmtree(newcase_root)
                expect(False, "env_build.xml cannot be changed via usermods if keepexe is an option: \n "
                           "Failed to clone case, removed {}\n".format(newcase_root))

    # if keep executable, then remove the new case SourceMods directory and link SourceMods to
    # the clone directory
    if keepexe:
        shutil.rmtree(os.path.join(newcase_root, "SourceMods"))
        os.symlink(os.path.join(cloneroot, "SourceMods"),
                   os.path.join(newcase_root, "SourceMods"))

    # Update README.case
    fclone   = open(cloneroot + "/README.case", "r")
    fnewcase = open(newcaseroot  + "/README.case", "a")
    fnewcase.write("\n    *** original clone README follows ****")
    fnewcase.write("\n " +  fclone.read())

    clonename = self.get_value("CASE")
    logger.info(" Successfully created new case {} from clone case {} ".format(newcasename, clonename))

    newcase.case_setup()

    return newcase
Exemple #15
0
def create_clone(self,
                 newcase,
                 keepexe=False,
                 mach_dir=None,
                 project=None,
                 cime_output_root=None,
                 exeroot=None,
                 rundir=None,
                 user_mods_dir=None):
    """
    Create a case clone

    If exeroot or rundir are provided (not None), sets these directories
    to the given paths; if not provided, uses default values for these
    directories. It is an error to provide exeroot if keepexe is True.
    """
    if cime_output_root is None:
        cime_output_root = self.get_value("CIME_OUTPUT_ROOT")

    newcaseroot = os.path.abspath(newcase)
    expect(not os.path.isdir(newcaseroot),
           "New caseroot directory {} already exists".format(newcaseroot))
    newcasename = os.path.basename(newcaseroot)
    expect(check_name(newcasename),
           "New case name invalid {} ".format(newcasename))
    newcase_cimeroot = os.path.abspath(get_cime_root())

    # create clone from case to case
    clone_cimeroot = self.get_value("CIMEROOT")
    if newcase_cimeroot != clone_cimeroot:
        logger.warning(" case  CIMEROOT is {} ".format(newcase_cimeroot))
        logger.warning(" clone CIMEROOT is {} ".format(clone_cimeroot))
        logger.warning(
            " It is NOT recommended to clone cases from different versions of CIME."
        )

    # *** create case object as deepcopy of clone object ***
    srcroot = os.path.join(newcase_cimeroot, "..")
    newcase = self.copy(newcasename, newcaseroot, newsrcroot=srcroot)
    newcase.set_value("CIMEROOT", newcase_cimeroot)

    # if we are cloning to a different user modify the output directory
    olduser = self.get_value("USER")
    newuser = os.environ.get("USER")
    if olduser != newuser:
        cime_output_root = cime_output_root.replace(olduser, newuser)
        newcase.set_value("USER", newuser)
    newcase.set_value("CIME_OUTPUT_ROOT", cime_output_root)

    # try to make the new output directory and raise an exception
    # on any error other than directory already exists.
    if os.path.isdir(cime_output_root):
        expect(
            os.access(cime_output_root, os.W_OK),
            "Directory {} is not writable "
            "by this user.  Use the --cime-output-root flag to provide a writable "
            "scratch directory".format(cime_output_root))
    else:
        try:
            os.makedirs(cime_output_root)
        except:
            if not os.path.isdir(cime_output_root):
                raise

    # determine if will use clone executable or not
    if keepexe:
        orig_exeroot = self.get_value("EXEROOT")
        newcase.set_value("EXEROOT", orig_exeroot)
        newcase.set_value("BUILD_COMPLETE", "TRUE")
        orig_bld_complete = self.get_value("BUILD_COMPLETE")
        if not orig_bld_complete:
            logger.warning(
                "\nWARNING: Creating a clone with --keepexe before building the original case may cause PIO_TYPENAME to be invalid in the clone"
            )
            logger.warning(
                "Avoid this message by building case one before you clone.\n")
    else:
        newcase.set_value("BUILD_COMPLETE", "FALSE")

    # set machdir
    if mach_dir is not None:
        newcase.set_value("MACHDIR", mach_dir)

    # set exeroot and rundir if requested
    if exeroot is not None:
        expect(
            not keepexe, "create_case_clone: if keepexe is True, "
            "then exeroot cannot be set")
        newcase.set_value("EXEROOT", exeroot)
    if rundir is not None:
        newcase.set_value("RUNDIR", rundir)

    # Set project id
    # Note: we do not just copy this from the clone because it seems likely that
    # users will want to change this sometimes, especially when cloning another
    # user's case. However, note that, if a project is not given, the fallback will
    # be to copy it from the clone, just like other xml variables are copied.
    if project is None:
        project = self.get_value("PROJECT", subgroup=self.get_primary_job())
    if project is not None:
        newcase.set_value("PROJECT", project)

    # create caseroot
    newcase.create_caseroot(clone=True)
    newcase.flush(flushall=True)

    # copy user_ files
    cloneroot = self.get_case_root()
    files = glob.glob(cloneroot + '/user_*')

    for item in files:
        safe_copy(item, newcaseroot)

    # copy SourceMod and Buildconf files
    # if symlinks exist, copy rather than follow links
    for casesub in ("SourceMods", "Buildconf"):
        shutil.copytree(os.path.join(cloneroot, casesub),
                        os.path.join(newcaseroot, casesub),
                        symlinks=True)

    # lock env_case.xml in new case
    lock_file("env_case.xml", newcaseroot)

    # apply user_mods if appropriate
    newcase_root = newcase.get_value("CASEROOT")
    if user_mods_dir is not None:
        if keepexe:
            # If keepexe CANNOT change any env_build.xml variables - so make a temporary copy of
            # env_build.xml and verify that it has not been modified
            safe_copy(
                os.path.join(newcaseroot, "env_build.xml"),
                os.path.join(newcaseroot, "LockedFiles", "env_build.xml"))

        # Now apply contents of user_mods directory
        apply_user_mods(newcase_root, user_mods_dir, keepexe=keepexe)

        # Determine if env_build.xml has changed
        if keepexe:
            success, comment = compare_files(
                os.path.join(newcaseroot, "env_build.xml"),
                os.path.join(newcaseroot, "LockedFiles", "env_build.xml"))
            if not success:
                logger.warning(comment)
                shutil.rmtree(newcase_root)
                expect(
                    False,
                    "env_build.xml cannot be changed via usermods if keepexe is an option: \n "
                    "Failed to clone case, removed {}\n".format(newcase_root))

    # if keep executable, then remove the new case SourceMods directory and link SourceMods to
    # the clone directory
    if keepexe:
        shutil.rmtree(os.path.join(newcase_root, "SourceMods"))
        os.symlink(os.path.join(cloneroot, "SourceMods"),
                   os.path.join(newcase_root, "SourceMods"))

    # Update README.case
    fclone = open(cloneroot + "/README.case", "r")
    fnewcase = open(newcaseroot + "/README.case", "a")
    fnewcase.write("\n    *** original clone README follows ****")
    fnewcase.write("\n " + fclone.read())

    clonename = self.get_value("CASE")
    logger.info(" Successfully created new case {} from clone case {} ".format(
        newcasename, clonename))

    newcase.case_setup()

    return newcase
Exemple #16
0
        # loop over models
        for model in models:
            comp = case.get_value("COMP_%s" % model)
            logger.info("Building %s usernl files" % model)
            _build_usernl_files(case, model, comp)
            if comp == "cism":
                run_cmd_no_fail(
                    "%s/../components/cism/cime_config/cism.template %s" %
                    (cimeroot, caseroot))

        _build_usernl_files(case, "drv", "cpl")

        user_mods_path = case.get_value("USER_MODS_FULLPATH")
        if user_mods_path is not None:
            apply_user_mods(caseroot,
                            user_mods_path=user_mods_path,
                            ninst=ninst)
        elif case.get_value("TEST"):
            test_mods = parse_test_name(casebaseid)[6]
            if test_mods is not None:
                user_mods_path = os.path.join(case.get_value("TESTS_MODS_DIR"),
                                              test_mods)
                apply_user_mods(caseroot,
                                user_mods_path=user_mods_path,
                                ninst=ninst)

        # Run preview namelists for scripts
        logger.info("preview_namelists")
        preview_namelists(case)

        logger.info("See ./CaseDoc for component namelists")
def _case_setup_impl(case, caseroot, casebaseid, clean=False, test_mode=False, reset=False):
###############################################################################
    os.chdir(caseroot)
    msg = "case.setup starting"
    append_status(msg, caseroot=caseroot, sfile="CaseStatus")

    cimeroot = get_cime_root(case)

    # Check that $DIN_LOC_ROOT exists - and abort if not a namelist compare tests
    din_loc_root = case.get_value("DIN_LOC_ROOT")
    testcase     = case.get_value("TESTCASE")
    expect(not (not os.path.isdir(din_loc_root) and testcase != "SBN"),
           "inputdata root is not a directory: \"$din_loc_root\" ")

    # Check that userdefine settings are specified before expanding variable
    for vid, value in case:
        expect(not (type(value) is str and "USERDEFINED_required_build" in value),
               "Parameter '%s' must be defined" % vid)

    # Create batch script
    if reset or clean:
        # Clean batch script

        backup_dir = "PESetupHist/b.%s" % time.strftime("%y%m%d-%H%M%S")
        if not os.path.isdir(backup_dir):
            os.makedirs(backup_dir)

        # back up relevant files
        for fileglob in ["case.run", "env_build.xml", "env_mach_pes.xml", "Macros*"]:
            for filename in glob.glob(fileglob):
                shutil.copy(filename, backup_dir)
        if os.path.exists("case.run"):
            os.remove("case.run")

        # only do the following if are NOT in testmode
        if not test_mode:
            # rebuild the models (even on restart)
            case.set_value("BUILD_COMPLETE", False)

            # backup and then clean test script
            if os.path.exists("case.test"):
                shutil.copy("case.test", backup_dir)
                os.remove("case.test")
                logger.info("Successfully cleaned test script case.test")

            if os.path.exists("case.testdriver"):
                shutil.copy("case.testdriver", backup_dir)
                os.remove("case.testdriver")
                logger.info("Successfully cleaned test script case.testdriver")

        logger.info("Successfully cleaned batch script case.run")

        logger.info("Successfully cleaned batch script case.run")
        logger.info("Some files have been saved to %s" % backup_dir)

        msg = "case.setup clean complete"
        append_status(msg, caseroot=caseroot, sfile="CaseStatus")

    if not clean:
        models = case.get_values("COMP_CLASSES")

        mach, compiler, debug, mpilib = \
            case.get_value("MACH"), case.get_value("COMPILER"), case.get_value("DEBUG"), case.get_value("MPILIB")
        expect(mach is not None, "xml variable MACH is not set")

        # Create Macros file only if it does not exist
        if not os.path.exists("Macros"):
            logger.debug("Creating Macros file for %s" % mach)
            compilers = Compilers(compiler=compiler, machine=mach, os_=case.get_value("OS"), mpilib=mpilib)
            compilers.write_macros_file()
        else:
            logger.debug("Macros script already created ...skipping")

        # Set tasks to 1 if mpi-serial library
        if mpilib == "mpi-serial":
            for vid, value in case:
                if vid.startswith("NTASKS_") and value != 1:
                    case.set_value(vid, 1)

        # Check ninst.
        # In CIME there can be multiple instances of each component model (an ensemble) NINST is the instance of that component.
        # Save ninst in a dict to use later in apply_user_mods
        ninst = dict()
        for comp in models:
            if comp == "DRV":
                continue
            comp_model = case.get_value("COMP_%s" % comp)
            ninst[comp_model]  = case.get_value("NINST_%s" % comp)
            ntasks = case.get_value("NTASKS_%s" % comp)
            if ninst[comp_model] > ntasks:
                if ntasks == 1:
                    case.set_value("NTASKS_%s" % comp, ninst[comp_model])
                else:
                    expect(False, "NINST_%s value %d greater than NTASKS_%s %d" % (comp, ninst[comp_model], comp, ntasks))

        if os.path.exists("case.run"):
            logger.info("Machine/Decomp/Pes configuration has already been done ...skipping")
        else:
            _check_pelayouts_require_rebuild(case, models)

            if os.path.exists("LockedFiles/env_build.xml"):
                os.remove("LockedFiles/env_build.xml")

            case.flush()
            check_lockedfiles()
            env_mach_pes = case.get_env("mach_pes")
            pestot = env_mach_pes.get_total_tasks(models)
            logger.debug("at update TOTALPES = %s"%pestot)
            case.set_value("TOTALPES", pestot)
            thread_count = env_mach_pes.get_max_thread_count(models)
            if thread_count > 1:
                case.set_value("BUILD_THREADED", True)

            expect(not (case.get_value("BUILD_THREADED")  and compiler == "nag"),
                   "it is not possible to run with OpenMP if using the NAG Fortran compiler")


            cost_pes = env_mach_pes.get_cost_pes(pestot, thread_count, machine=case.get_value("MACH"))
            case.set_value("COST_PES", cost_pes)

            # create batch file
            logger.info("Creating batch script case.run")

            # Use BatchFactory to get the appropriate instance of a BatchMaker,
            # use it to create our batch scripts
            env_batch = case.get_env("batch")
            num_nodes = env_mach_pes.get_total_nodes(pestot, thread_count)
            tasks_per_node = env_mach_pes.get_tasks_per_node(pestot, thread_count)
            for job in env_batch.get_jobs():
                input_batch_script  = os.path.join(case.get_value("MACHDIR"), env_batch.get_value('template', subgroup=job))
                if job == "case.test" and testcase is not None and not test_mode:
                    logger.info("Writing %s script" % job)
                    testscript = os.path.join(cimeroot, "scripts", "Testing", "Testcases", "%s_script" % testcase)
                    # Short term fix to be removed when csh tests are removed
                    if not os.path.exists(testscript):
                        env_batch.make_batch_script(input_batch_script, job, case, pestot, tasks_per_node, num_nodes, thread_count)
                elif job != "case.test":
                    logger.info("Writing %s script from input template %s" % (job, input_batch_script))
                    env_batch.make_batch_script(input_batch_script, job, case, pestot, tasks_per_node, num_nodes, thread_count)

            # Make a copy of env_mach_pes.xml in order to be able
            # to check that it does not change once case.setup is invoked
            logger.info("Locking file env_mach_pes.xml")
            case.flush()
            logger.debug("at copy TOTALPES = %s"%case.get_value("TOTALPES"))
            shutil.copy("env_mach_pes.xml", "LockedFiles")

        # Create user_nl files for the required number of instances
        if not os.path.exists("user_nl_cpl"):
            logger.info("Creating user_nl_xxx files for components and cpl")
        # loop over models
        for model in models:
            comp = case.get_value("COMP_%s" % model)
            logger.info("Building %s usernl files"%model)
            _build_usernl_files(case, model, comp)
            if comp == "cism":
                run_cmd_no_fail("%s/../components/cism/cime_config/cism.template %s" % (cimeroot, caseroot))

        _build_usernl_files(case, "drv", "cpl")

        user_mods_path = case.get_value("USER_MODS_FULLPATH")
        if user_mods_path is not None:
            apply_user_mods(caseroot, user_mods_path=user_mods_path, ninst=ninst)
        elif case.get_value("TEST"):
            test_mods = parse_test_name(casebaseid)[6]
            if test_mods is not None:
                user_mods_path = os.path.join(case.get_value("TESTS_MODS_DIR"), test_mods)
                apply_user_mods(caseroot, user_mods_path=user_mods_path, ninst=ninst)


        # Run preview namelists for scripts
        logger.info("preview_namelists")
        preview_namelists(case)

        logger.info("See ./CaseDoc for component namelists")
        logger.info("If an old case build already exists, might want to run \'case.build --clean\' before building")

        # Create test script if appropriate
        # Short term fix to be removed when csh tests are removed
        if os.path.exists("env_test.xml"):
            if not os.path.exists("case.test"):
                logger.info("Starting testcase.setup")
                run_cmd_no_fail("./testcase.setup -caseroot %s" % caseroot)
                logger.info("Finished testcase.setup")

        msg = "case.setup complete"
        append_status(msg, caseroot=caseroot, sfile="CaseStatus")

        # Record env information
        env_module = case.get_env("mach_specific")
        env_module.make_env_mach_specific_file(compiler, debug, mpilib, "sh")
        env_module.make_env_mach_specific_file(compiler, debug, mpilib, "csh")
        with open("software_environment.txt", "w") as f:
            f.write(env_module.list_modules())
        run_cmd_no_fail("echo -e '\n' >> software_environment.txt && \
                         env >> software_environment.txt")
 def test_keepexe(self):
     self.createUserMod("foo")
     with six.assertRaisesRegex(self, CIMEError, "cannot have any source mods"):
         apply_user_mods(self._caseroot,
                         os.path.join(self._user_mods_parent_dir, "foo"), keepexe=True)
Exemple #19
0
def _case_setup_impl(case, caseroot, casebaseid, clean=False, test_mode=False, reset=False):
###############################################################################
    os.chdir(caseroot)
    msg = "case.setup starting"
    append_status(msg, caseroot=caseroot, sfile="CaseStatus")

    cimeroot = os.environ["CIMEROOT"]

    # Check that $DIN_LOC_ROOT exists - and abort if not a namelist compare tests
    din_loc_root = case.get_value("DIN_LOC_ROOT")
    testcase     = case.get_value("TESTCASE")
    expect(not (not os.path.isdir(din_loc_root) and testcase != "SBN"),
           "inputdata root is not a directory: \"$din_loc_root\" ")

    # Check that userdefine settings are specified before expanding variable
    for vid, value in case:
        expect(not (type(value) is str and "USERDEFINED_required_build" in value),
               "Parameter '%s' must be defined" % vid)

    # Create batch script
    if reset or clean:
        # Clean batch script

        backup_dir = "PESetupHist/b.%s" % time.strftime("%y%m%d-%H%M%S")
        if not os.path.isdir(backup_dir):
            os.makedirs(backup_dir)

        # back up relevant files
        for fileglob in ["case.run", "env_build.xml", "env_mach_pes.xml", "Macros*"]:
            for filename in glob.glob(fileglob):
                shutil.copy(filename, backup_dir)
        if os.path.exists("case.run"):
            os.remove("case.run")

        # only do the following if are NOT in testmode
        if not test_mode:
            # rebuild the models (even on restart)
            case.set_value("BUILD_COMPLETE", False)

            # backup and then clean test script
            if os.path.exists("case.test"):
                shutil.copy("case.test", backup_dir)
                os.remove("case.test")
                logger.info("Successfully cleaned test script case.test")

            if os.path.exists("case.testdriver"):
                shutil.copy("case.testdriver", backup_dir)
                os.remove("case.testdriver")
                logger.info("Successfully cleaned test script case.testdriver")

        logger.info("Successfully cleaned batch script case.run")

        logger.info("Successfully cleaned batch script case.run")
        logger.info("Some files have been saved to %s" % backup_dir)

        msg = "case.setup clean complete"
        append_status(msg, caseroot=caseroot, sfile="CaseStatus")

    if not clean:
        drv_comp = Component()
        models = drv_comp.get_valid_model_components()
        models.remove("DRV")

        mach, compiler, debug, mpilib = \
            case.get_value("MACH"), case.get_value("COMPILER"), case.get_value("DEBUG"), case.get_value("MPILIB")
        expect(mach is not None, "xml variable MACH is not set")

        # Create Macros file only if it does not exist
        if not os.path.exists("Macros"):
            logger.debug("Creating Macros file for %s" % mach)
            compilers = Compilers(compiler=compiler, machine=mach, os_=case.get_value("OS"), mpilib=mpilib)
            compilers.write_macros_file()
        else:
            logger.debug("Macros script already created ...skipping")

        # Set tasks to 1 if mpi-serial library
        if mpilib == "mpi-serial":
            for vid, value in case:
                if vid.startswith("NTASKS_") and value != 1:
                    case.set_value(vid, 1)

        # Check ninst.
        # In CIME there can be multiple instances of each component model (an ensemble) NINST is the instance of that component.
        # Save ninst in a dict to use later in apply_user_mods
        ninst = dict()
        for comp in models:
            comp_model = case.get_value("COMP_%s" % comp)
            ninst[comp_model]  = case.get_value("NINST_%s" % comp)
            ntasks = case.get_value("NTASKS_%s" % comp)
            if ninst[comp_model] > ntasks:
                if ntasks == 1:
                    case.set_value("NTASKS_%s" % comp, ninst[comp_model])
                else:
                    expect(False, "NINST_%s value %d greater than NTASKS_%s %d" % (comp, ninst[comp_model], comp, ntasks))

        expect(not (case.get_value("BUILD_THREADED") and compiler == "nag"),
               "it is not possible to run with OpenMP if using the NAG Fortran compiler")

        if os.path.exists("case.run"):
            logger.info("Machine/Decomp/Pes configuration has already been done ...skipping")
        else:
            _check_pelayouts_require_rebuild(case, models)

            if os.path.exists("LockedFiles/env_build.xml"):
                os.remove("LockedFiles/env_build.xml")

            case.flush()
            check_lockedfiles()

            tm = TaskMaker(case)
            mtpn = case.get_value("MAX_TASKS_PER_NODE")
            pespn = case.get_value("PES_PER_NODE")
            # This is hardcoded because on yellowstone by default we
            # run with 15 pes per node
            # but pay for 16 pes per node.  See github issue #518
            if case.get_value("MACH") == "yellowstone":
                pespn = 16
            pestot = tm.totaltasks
            if mtpn > pespn:
                pestot = pestot * (mtpn // pespn)
                case.set_value("COST_PES", tm.num_nodes*pespn)
            else:
                # reset cost_pes to totalpes
                case.set_value("COST_PES", 0)

            case.set_value("TOTALPES", pestot)

            # Compute cost based on PE count
            pval = 1
            pcnt = 0
            while pval < pestot:
                pval *= 2
                pcnt += 6 # (scaling like sqrt(6/10))
            pcost = 3 - pcnt / 10 # (3 is 64 with 6)

            # Compute cost based on DEBUG
            dcost = 3 if debug else 0

            # Compute cost based on run length
            # For simplicity, we use a heuristic just based on STOP_OPTION (not considering
            # STOP_N), and only deal with options longer than ndays
            lcost = 0
            if "nmonth" in case.get_value("STOP_OPTION"):
                # N months costs 30x as much as N days; since cost is based on log-base-2, add 5
                lcost = 5
            elif "nyear" in case.get_value("STOP_OPTION"):
                # N years costs 365x as much as N days; since cost is based on log-base-2, add 9
                lcost = 9

            estcost = pcost + dcost + lcost
            for cost in ["CCSM_CCOST", "CCSM_GCOST", "CCSM_TCOST", "CCSM_CCOST"]:
                estcost += case.get_value(cost)

            case.set_value("CCSM_PCOST", pcost)
            case.set_value("CCSM_ESTCOST", estcost)

            # create batch file
            logger.info("Creating batch script case.run")

            # Use BatchFactory to get the appropriate instance of a BatchMaker,
            # use it to create our batch scripts
            env_batch = case.get_env("batch")
            for job in env_batch.get_jobs():
                input_batch_script  = os.path.join(case.get_value("MACHDIR"), env_batch.get_value('template', subgroup=job))
                if job == "case.test" and testcase is not None and not test_mode:
                    logger.info("Writing %s script" % job)
                    testscript = os.path.join(cimeroot, "scripts", "Testing", "Testcases", "%s_script" % testcase)
                    # Short term fix to be removed when csh tests are removed
                    if not os.path.exists(testscript):
                        env_batch.make_batch_script(input_batch_script, job, case)
                elif job != "case.test":
                    logger.info("Writing %s script" % job)
                    env_batch.make_batch_script(input_batch_script, job, case)

            # Make a copy of env_mach_pes.xml in order to be able
            # to check that it does not change once case.setup is invoked
            logger.info("Locking file env_mach_pes.xml")
            case.flush()
            shutil.copy("env_mach_pes.xml", "LockedFiles")

        # Create user_nl files for the required number of instances
        if not os.path.exists("user_nl_cpl"):
            logger.info("Creating user_nl_xxx files for components and cpl")
        # loop over models
        for model in models:
            comp = case.get_value("COMP_%s" % model)
            logger.info("Building %s usernl files"%model)
            _build_usernl_files(case, model, comp)
            if comp == "cism":
                run_cmd_no_fail("%s/../components/cism/cime_config/cism.template %s" % (cimeroot, caseroot))

        _build_usernl_files(case, "drv", "cpl")

        user_mods_path = case.get_value("USER_MODS_FULLPATH")
        if user_mods_path is not None:
            apply_user_mods(caseroot, user_mods_path=user_mods_path, ninst=ninst)
        elif case.get_value("TEST"):
            test_mods = parse_test_name(casebaseid)[6]
            if test_mods is not None:
                user_mods_path = os.path.join(case.get_value("TESTS_MODS_DIR"), test_mods)
                apply_user_mods(caseroot, user_mods_path=user_mods_path, ninst=ninst)


        # Run preview namelists for scripts
        logger.info("preview_namelists")
        preview_namelists(case)

        logger.info("See ./CaseDoc for component namelists")
        logger.info("If an old case build already exists, might want to run \'case.build --clean\' before building")

        # Create test script if appropriate
        # Short term fix to be removed when csh tests are removed
        if os.path.exists("env_test.xml"):
            if not os.path.exists("case.test"):
                logger.info("Starting testcase.setup")
                run_cmd_no_fail("./testcase.setup -caseroot %s" % caseroot)
                logger.info("Finished testcase.setup")

        msg = "case.setup complete"
        append_status(msg, caseroot=caseroot, sfile="CaseStatus")

        # Record env information
        env_module = case.get_env("mach_specific")
        env_module.make_env_mach_specific_file(compiler, debug, mpilib, "sh")
        env_module.make_env_mach_specific_file(compiler, debug, mpilib, "csh")
        with open("software_environment.txt", "w") as f:
            f.write(env_module.list_modules())
        run_cmd_no_fail("echo -e '\n' >> software_environment.txt && \
                         env >> software_environment.txt")
Exemple #20
0
    # lock env_case.xml in new case
    lock_file("env_case.xml", newcaseroot)

    # apply user_mods if appropriate
    newcase_root = newcase.get_value("CASEROOT")
    if user_mods_dir is not None:
        if keepexe:
            # If keepexe CANNOT change any env_build.xml variables - so make a temporary copy of
            # env_build.xml and verify that it has not been modified
            shutil.copy(
                os.path.join(newcaseroot, "env_build.xml"),
                os.path.join(newcaseroot, "LockedFiles", "env_build.xml"))

        # Now apply contents of user_mods directory
        apply_user_mods(newcase_root, user_mods_dir, keepexe=keepexe)

        # Determine if env_build.xml has changed
        if keepexe:
            success, comment = compare_files(
                os.path.join(newcaseroot, "env_build.xml"),
                os.path.join(newcaseroot, "LockedFiles", "env_build.xml"))
            if not success:
                logger.warning(comment)
                shutil.rmtree(newcase_root)
                expect(
                    False,
                    "env_build.xml cannot be changed via usermods if keepexe is an option: \n "
                    "Failed to clone case, removed {}\n".format(newcase_root))

    # if keep executable, then remove the new case SourceMods directory and link SourceMods to