Example #1
0
    def run_case(self, base_case_root, run_type, run_length, overwrite=False):
        user_mods_dirs = [
            os.path.join(self.cesmroot, "cime_config", "usermods_dirs", "NEON",
                         self.name)
        ]
        expect(os.path.isdir(base_case_root),
               "Error base case does not exist in {}".format(base_case_root))
        case_root = os.path.abspath(
            os.path.join(base_case_root, "..", self.name + "." + run_type))
        if os.path.isdir(case_root) and overwrite:
            logger.info("---- removing the existing case -------")
            shutil.rmtree(case_root)
        if not os.path.isdir(case_root):
            # read_only = False should not be required here
            with Case(base_case_root, read_only=False) as basecase:
                logger.info(
                    "---- cloning the base case in {}".format(case_root))
                basecase.create_clone(case_root,
                                      keepexe=True,
                                      user_mods_dirs=user_mods_dirs)

        with Case(case_root, read_only=False) as case:
            case.set_value("STOP_OPTION", "nyears")
            case.set_value("STOP_N", run_length)
            case.set_value("REST_N", 100)
            case.set_value("CONTINUE_RUN", False)

            if run_type == "ad":
                case.set_value("CLM_FORCE_COLDSTART", "on")
                case.set_value("CLM_ACCELERATED_SPINUP", "on")
                case.set_value("RUN_REFDATE", "0018-01-01")
                case.set_value("RUN_STARTDATE", "0018-01-01")
Example #2
0
def build_xcpl_nml(argv, compclass):
###############################################################################

    caseroot = parse_input(argv)

    compname = "x" + compclass

    caseroot = parse_input(argv)

    case = Case(caseroot)

    rundir = case.get_value("RUNDIR")
    ninst  = case.get_value("NINST_%s" % compclass.upper())
    nx     = case.get_value("%s_NX" % compclass.upper())
    ny     = case.get_value("%s_NY" % compclass.upper())

    extras = []
    dtype = 1
    npes = 0
    length = 0

    if compname == "xatm":
        if ny == 1:
            dtype = 2
        extras = [["24",
                   "ncpl  number of communications w/coupler per dat"],
                  ["0.0",
                   "simul time proxy (secs): time between cpl comms"]]
    elif compname == "xglc" or compname == "xice":
        dtype = 2
    elif compname == "xlnd":
        dtype = 11
    elif compname == "xocn":
        dtype = 4
    elif compname == "xrof":
        dtype = 11
        flood_mode = Case('XROF_FLOOD_MODE')
        if flood_mode == "ACTIVE":
            extras = [[".true.", "flood flag"]]
        else:
            extras = [[".false.", "flood flag"]]

    for i in range(1, ninst + 1):
        # If only 1 file, name is 'compclass_in'
        # otherwise files are 'compclass_in0001', 'compclass_in0002', etc
        if ninst == 1:
            filename = os.path.join(rundir, "%s_in" % compname)
        else:
            filename = os.path.join(rundir, "%s_in%4.4d" % (compname, i))

        with open(filename, 'w') as infile:
            infile.write("%-20d ! i-direction global dimension\n" % nx)
            infile.write("%-20d ! j-direction global dimension\n" % ny)
            infile.write("%-20d ! decomp_type  1=1d-by-lat, 2=1d-by-lon,"
                         " 3=2d, 4=2d evensquare, 11=segmented\n" % dtype)
            infile.write("%-20d ! num of pes for i (type 3 only)\n" % npes)
            infile.write("%-20d ! length of segments (type 4 only)\n"
                         % length)
            for extra in extras:
                infile.write("%-20s ! %s\n" % (extra[0], extra[1]))
Example #3
0
def build_data_nml(argv, compclass):
    ###############################################################################
    # This function is just a wrapper for the one below, to avoid having to
    # indent everything for the "with" block.
    caseroot = parse_input(argv)
    with Case(caseroot) as case:
        _build_data_nml(case, caseroot, compclass)
def build_base_case(baseroot, basecasename, res, compset, overwrite,
                    user_mods_dir):
    print(">>>>> BUILDING BASE CASE...")
    caseroot = os.path.join(baseroot,basecasename+'.'+basecase_startval)
    if overwrite and os.path.isdir(caseroot):
        shutil.rmtree(caseroot)
    with Case(caseroot, read_only=False) as case:
        if not os.path.isdir(caseroot):
            case.create(os.path.basename(caseroot), cesmroot, compset, res,
                        machine_name="cheyenne", driver="mct",
                        run_unsupported=True, answer="r",walltime="01:00:00",
                        user_mods_dir=user_mods_dir, project=project)
            # make sure that changing the casename will not affect these variables                                           
            case.set_value("EXEROOT",case.get_value("EXEROOT", resolved=True))
            case.set_value("RUNDIR",case.get_value("RUNDIR",resolved=True)+".00")

            case.set_value("RUN_TYPE","startup")
            case.set_value("GET_REFCASE",False)


        rundir = case.get_value("RUNDIR")
        caseroot = case.get_value("CASEROOT")
        print(">> base case_setup...")
        case.case_setup()
        print(">> base case_build...")
        build.case_build(caseroot, case=case)

        return caseroot
Example #5
0
    def test_submit(self, read_xml, get_value, init, _submit):  # pylint: disable=unused-argument
        with tempfile.TemporaryDirectory() as tempdir:
            get_value.side_effect = [
                tempdir,
                tempdir,
                True,
                "baseid",
                None,
                True,
            ]

            make_valid_case(tempdir)
            with Case(tempdir) as case:
                case.submit(chksum=True)

            _submit.assert_called_with(
                case,
                job=None,
                no_batch=False,
                prereq=None,
                allow_fail=False,
                resubmit=False,
                resubmit_immediate=False,
                skip_pnl=False,
                mail_user=None,
                mail_type=None,
                batch_args=None,
                workflow=True,
                chksum=True,
            )
Example #6
0
def bless_history(test_name, testcase_dir_for_test, baseline_name,
                  baseline_root, compiler, report_only, force):
    ###############################################################################
    with Case(testcase_dir_for_test) as case:
        if get_model() == "acme":
            baseline_full_dir = os.path.join(baseline_root, compiler,
                                             baseline_name,
                                             case.get_value("CASEBASEID"))
        else:
            baseline_full_dir = os.path.join(baseline_root, baseline_name,
                                             case.get_value("CASEBASEID"))

        result, comments = compare_baseline(case,
                                            baseline_dir=baseline_full_dir,
                                            outfile_suffix=None)
        if result:
            logging.info("Diff appears to have been already resolved.")
            return True, None
        else:
            print comments
            if (not report_only and
                (force or raw_input("Update this diff (y/n)? ").upper()
                 in ["Y", "YES"])):
                result, comments = generate_baseline(
                    case, baseline_dir=baseline_full_dir)
                if not result:
                    logging.warning("Hist file bless FAILED for test %s" %
                                    test_name)
                    return False, "Generate baseline failed: %s" % comments
                else:
                    print comments
                    return True, None
            else:
                return True, None
Example #7
0
    def test_cmd_arg(self, get_value, flush, init):  # pylint: disable=unused-argument
        Case._force_read_only = False  # pylint: disable=protected-access

        mocked_open = mock.mock_open()

        with self.tempdir as tempdir, mock.patch("CIME.case.case.open",
                                                 mocked_open):
            with Case(tempdir) as case:
                get_value.side_effect = [
                    tempdir,
                    "/src",
                ]

                # We didn't need to make tempdir look like a valid case for the Case
                # constructor because we mock that constructor, but we *do* need to make
                # it look like a valid case for record_cmd.
                make_valid_case(tempdir)
                case.record_cmd(["/some/custom/command", "arg1"])

        expected = [
            "/some/custom/command arg1\n\n",
        ]

        handle = mocked_open()
        handle.writelines.assert_called_with(expected)
Example #8
0
def bless_history(test_name, test_dir, baseline_name, baseline_root, report_only, force):
###############################################################################
    with Case(test_dir) as case:
        real_user = case.get_value("REALUSER")
        with EnvironmentContext(USER=real_user):
            if baseline_name is None:
                baseline_name = case.get_value("BASELINE_NAME_CMP")
                if not baseline_name:
                    baseline_name = CIME.utils.get_current_branch(repo=CIME.utils.get_cime_root())

            if baseline_root is None:
                baseline_root = case.get_value("BASELINE_ROOT")

            baseline_full_dir = os.path.join(baseline_root, baseline_name, case.get_value("CASEBASEID"))

            cmp_result, cmp_comments = compare_baseline(case, baseline_dir=baseline_full_dir, outfile_suffix=None)
            if cmp_result:
                logger.info("Diff appears to have been already resolved.")
                return True, None
            else:
                logger.info(cmp_comments)
                if (not report_only and
                    (force or six.moves.input("Update this diff (y/n)? ").upper() in ["Y", "YES"])):
                    gen_result, gen_comments = generate_baseline(case, baseline_dir=baseline_full_dir)
                    if not gen_result:
                        logger.warning("Hist file bless FAILED for test {}".format(test_name))
                        return False, "Generate baseline failed: {}".format(gen_comments)
                    else:
                        logger.info(gen_comments)
                        return True, None
                else:
                    return True, None
Example #9
0
def parse_input(argv):
###############################################################################

    parser = argparse.ArgumentParser()

    setup_standard_logging_options(parser)

    parser.add_argument("caseroot", default=os.getcwd(),
                        help="Case directory")

    parser.add_argument("libroot",
                        help="root for creating the library")

    parser.add_argument("bldroot",
                        help="root for building library")

    args = parse_args_and_handle_standard_logging_options(argv, parser)

    # Some compilers have trouble with long include paths, setting
    # EXEROOT to the relative path from bldroot solves the problem
    # doing it in the environment means we don't need to change all of
    # the component buildlib scripts
    with Case(args.caseroot) as case:
        os.environ["EXEROOT"] = os.path.relpath(case.get_value("EXEROOT"), args.bldroot)


    return args.caseroot, args.libroot, args.bldroot
Example #10
0
    def test_create(self, get_user, getfqdn, configure, create_caseroot, # pylint: disable=unused-argument
                    apply_user_mods, set_lookup_value, lock_file, strftime, # pylint: disable=unused-argument
                    read_xml): # pylint: disable=unused-argument
        with self.tempdir as tempdir:
            caseroot = os.path.join(tempdir, "test1")
            with Case(caseroot, read_only=False) as case:
                os.environ["CIME_MODEL"] = "cesm"

                srcroot = os.path.abspath(os.path.join(
                    os.path.dirname(__file__), "../../../../../"))
                case.create("test1", srcroot, "A", "f19_g16_rx1",
                            machine_name="ubuntu-latest")

                # Check that they're all called
                configure.assert_called_with("A", "f19_g16_rx1",
                                             machine_name="ubuntu-latest",
                                             project=None, pecount=None,
                                             compiler=None, mpilib=None,
                                             pesfile=None, gridfile=None,
                                             multi_driver=False, ninst=1,
                                             test=False, walltime=None,
                                             queue=None, output_root=None,
                                             run_unsupported=False, answer=None,
                                             input_dir=None, driver=None,
                                             workflowid="default",
                                             non_local=False,
                                             extra_machines_dir=None,
                                             case_group=None)
                create_caseroot.assert_called()
                apply_user_mods.assert_called()
                lock_file.assert_called()

                set_lookup_value.assert_called_with("CASE_HASH",
                                                    "134a939f62115fb44bf08a46bfb2bd13426833b5c8848cf7c4884af7af05b91a")
Example #11
0
    def test_sub_relative(self, strftime, get_value, flush, init): # pylint: disable=unused-argument
        Case._force_read_only = False # pylint: disable=protected-access

        mocked_open = mock.mock_open()

        with self.tempdir as tempdir, mock.patch("CIME.case.case.open",
                                                 mocked_open):
            with Case(tempdir) as case:
                get_value.side_effect = [
                    tempdir,
                    "/src"
                ]

                case.record_cmd(init=True)

        expected = [
            "#!/bin/bash\n\n",
            "set -e\n\n",
            "# Created 00:00:00\n\n",
            "CASEDIR=\"{}\"\n\n".format(tempdir),
            "/src/scripts/create_newcase\n\n",
            "cd \"${CASEDIR}\"\n\n",
        ]

        handle = mocked_open()
        handle.writelines.assert_called_with(expected)
Example #12
0
 def set_ref_case(self, case):
     rundir = case.get_value("RUNDIR")
     case_root = case.get_value("CASEROOT")
     if case_root.endswith(".postad"):
         ref_case_root = case_root.replace(".postad", ".ad")
         root = ".ad"
     else:
         ref_case_root = case_root.replace(".transient", ".postad")
         root = ".postad"
     expect(
         os.path.isdir(ref_case_root),
         "ERROR: spinup must be completed first, could not find directory {}"
         .format(ref_case_root))
     with Case(ref_case_root) as refcase:
         refrundir = refcase.get_value("RUNDIR")
     case.set_value("RUN_REFDIR", refrundir)
     case.set_value("RUN_REFCASE", os.path.basename(ref_case_root))
     for reffile in glob.iglob(refrundir +
                               "/{}{}.*.nc".format(self.name, root)):
         m = re.search("(\d\d\d\d-\d\d-\d\d)-\d\d\d\d\d.nc", reffile)
         if m:
             refdate = m.group(1)
         symlink_force(reffile,
                       os.path.join(rundir, os.path.basename(reffile)))
     for rpfile in glob.iglob(refrundir + "/rpointer*"):
         safe_copy(rpfile, rundir)
     if not os.path.isdir(os.path.join(
             rundir, "inputdata")) and os.path.isdir(
                 os.path.join(refrundir, "inputdata")):
         symlink_force(os.path.join(refrundir, "inputdata"),
                       os.path.join(rundir, "inputdata"))
     case.set_value("RUN_REFDATE", refdate)
     if case_root.endswith(".postad"):
         case.set_value("RUN_STARTDATE", refdate)
Example #13
0
def bless_history(test_name, testcase_dir_for_test, baseline_name,
                  baseline_root, report_only, force):
    ###############################################################################
    with Case(testcase_dir_for_test) as case:
        baseline_full_dir = os.path.join(baseline_root, baseline_name,
                                         case.get_value("CASEBASEID"))
        result, comments = compare_baseline(case,
                                            baseline_dir=baseline_full_dir)
        if result:
            return True, None
        else:
            print comments
            if (not report_only and
                (force or raw_input("Update this diff (y/n)? ").upper()
                 in ["Y", "YES"])):
                result, comments = generate_baseline(
                    case, baseline_dir=baseline_full_dir)
                if not result:
                    logging.warning("Hist file bless FAILED for test %s" %
                                    test_name)
                    return False, "Generate baseline failed: %s" % comments
                else:
                    print comments
                    return True, None
            else:
                return True, None
    def _case_from_existing_caseroot(self, caseroot):
        """
        Returns a Case object from an existing caseroot directory

        Args:
            caseroot (str): path to existing caseroot
        """
        return Case(case_root=caseroot, read_only=False)
Example #15
0
def build_data_lib(argv, compclass):
###############################################################################

    caseroot, libroot, bldroot = parse_input(argv)

    case = Case(caseroot)

    cimeroot  = case.get_value("CIMEROOT")

    # Write directory list (Filepath)
    compname = "d" + compclass
    with open('Filepath', 'w') as out:
        out.write(os.path.join(caseroot, "SourceMods", "src.%s" %compname) + "\n")
        out.write(os.path.join(cimeroot, "components", "data_comps", compname) + "\n")

    # Build the component
    run_gmake(case, compclass, libroot)
def clone_base_case(caseroot, ensemble, user_mods_dir, overwrite, paramdict):
    print(">>>>> CLONING BASE CASE...")
    startval = ensemble_startval
    nint = len(ensemble_startval)
    cloneroot = caseroot
    for i in range(int(startval), int(startval)+ensemble):
        member_string = '{{0:0{0:d}d}}'.format(nint).format(i)
        print("member_string="+member_string)
        if ensemble > 1:
            caseroot = caseroot[:-nint] + member_string
        if overwrite and os.path.isdir(caseroot):
            shutil.rmtree(caseroot)
        if not os.path.isdir(caseroot):
            with Case(cloneroot, read_only=False) as clone:
                clone.create_clone(caseroot, keepexe=True,
                                   user_mods_dir=user_mods_dir)
        with Case(caseroot, read_only=False) as case:
            per_run_case_updates(case, user_mods_dir, member_string, nint, paramdict)
Example #17
0
def build_xcpl_nml(argv, compclass):
    ###############################################################################

    caseroot = parse_input(argv)

    compname = "x" + compclass

    caseroot = parse_input(argv)

    with Case(caseroot) as case:
        rundir = case.get_value("RUNDIR")
        ninst = case.get_value("NINST_%s" % compclass.upper())
        nx = case.get_value("%s_NX" % compclass.upper())
        ny = case.get_value("%s_NY" % compclass.upper())
        if compname == "xrof":
            flood_mode = case.get_value('XROF_FLOOD_MODE')

    extras = []
    dtype = 1
    npes = 0
    length = 0

    if compname == "xatm":
        if ny == 1:
            dtype = 2
        extras = [["24", "ncpl  number of communications w/coupler per dat"],
                  ["0.0", "simul time proxy (secs): time between cpl comms"]]
    elif compname == "xglc" or compname == "xice":
        dtype = 2
    elif compname == "xlnd":
        dtype = 11
    elif compname == "xocn":
        dtype = 4
    elif compname == "xrof":
        dtype = 11
        if flood_mode == "ACTIVE":
            extras = [[".true.", "flood flag"]]
        else:
            extras = [[".false.", "flood flag"]]

    for i in range(1, ninst + 1):
        # If only 1 file, name is 'compclass_in'
        # otherwise files are 'compclass_in0001', 'compclass_in0002', etc
        if ninst == 1:
            filename = os.path.join(rundir, "%s_in" % compname)
        else:
            filename = os.path.join(rundir, "%s_in_%4.4d" % (compname, i))

        with open(filename, 'w') as infile:
            infile.write("%-20d ! i-direction global dimension\n" % nx)
            infile.write("%-20d ! j-direction global dimension\n" % ny)
            infile.write("%-20d ! decomp_type  1=1d-by-lat, 2=1d-by-lon,"
                         " 3=2d, 4=2d evensquare, 11=segmented\n" % dtype)
            infile.write("%-20d ! num of pes for i (type 3 only)\n" % npes)
            infile.write("%-20d ! length of segments (type 4 only)\n" % length)
            for extra in extras:
                infile.write("%-20s ! %s\n" % (extra[0], extra[1]))
Example #18
0
 def __init__(self, casename):
     super(CaseWindow, self).__init__()
     self.setupUi(self)
     self.setMouseTracking(1)
     with Case(casename) as case:
         self.populate_case_tab(case)
         self.populate_setup_tab(case)
         self.populate_build_tab(case)
         self.populate_run_tab(case)
Example #19
0
def compare_history(testcase_dir_for_test, baseline_name, baseline_root):
    ###############################################################################
    with Case(testcase_dir_for_test) as case:
        baseline_full_dir = os.path.join(baseline_root, baseline_name,
                                         case.get_value("CASEBASEID"))
        result, comments = compare_baseline(case,
                                            baseline_dir=baseline_full_dir)
        if result:
            return True, None
        else:
            logging.info(comments)
            return False, "Diff'd"
Example #20
0
 def create_new_case(self):
     srcroot = os.path.dirname(os.path.abspath(get_cime_root()))
     casename = self.CreateNewcaseArgs["case"]
     with Case(os.path.join(self._casedir, casename),
               read_only=False) as case:
         case.create(casename,
                     srcroot,
                     self.CreateNewcaseArgs["compset"],
                     self.CreateNewcaseArgs["res"],
                     machine_name=self.CreateNewcaseArgs["machine"],
                     compiler=self.CreateNewcaseArgs["compiler"],
                     mpilib=self.CreateNewcaseArgs["mpilib"])
Example #21
0
def compare_history(testcase_dir_for_test, baseline_name, baseline_root,
                    log_id):
    ###############################################################################
    with Case(testcase_dir_for_test) as case:
        baseline_full_dir = os.path.join(baseline_root, baseline_name,
                                         case.get_value("CASEBASEID"))
        outfile_suffix = "%s.%s" % (baseline_name, log_id)
        result, comments = compare_baseline(case,
                                            baseline_dir=baseline_full_dir,
                                            outfile_suffix=outfile_suffix)

        return result, comments
def download_input_data(rundir):
    """Implementation of the download_input_data command

    Args:
    rundir: str - path to directory containing input_data_list files
    """
    _create_lilac_input_data_list(rundir)
    case = Case(os.path.realpath(os.path.join(rundir, os.pardir, 'case')))
    case.check_all_input_data(data_list_dir=rundir,
                              download=True,
                              chksum=False)
    os.remove(os.path.join(rundir, 'lilac.input_data_list'))
Example #23
0
def clone_base_case(date, caseroot, ensemble, sdrestdir, user_mods_dir,
                    overwrite):

    startval = "01"
    nint = len(startval)
    cloneroot = caseroot
    for i in range(int(startval), int(startval) + ensemble):
        member_string = '{{0:0{0:d}d}}'.format(nint).format(i)
        if ensemble > 1:
            caseroot = caseroot[:-nint] + member_string
        if overwrite and os.path.isdir(caseroot):
            shutil.rmtree(caseroot)
        if not os.path.isdir(caseroot):
            with Case(cloneroot, read_only=False) as clone:
                clone.create_clone(caseroot,
                                   keepexe=True,
                                   user_mods_dir=user_mods_dir)
        with Case(caseroot, read_only=True) as case:
            # rundir is initially 00 reset to current member
            rundir = case.get_value("RUNDIR")
            rundir = rundir[:-nint] + member_string
            case.set_value("RUNDIR", rundir)
            per_run_case_updates(case, date, sdrestdir, user_mods_dir, rundir)
Example #24
0
def cleanup_queue(test_root, test_id):
###############################################################################
    """
    Delete all jobs left in the queue
    """
    for teststatus_file in glob.iglob("{}/*{}*/TestStatus".format(test_root, test_id)):
        case_dir = os.path.dirname(teststatus_file)
        with Case(case_dir, read_only=True) as case:
            jobmap = case.get_job_info()
            jobkills = []
            for jobname, jobid in jobmap.items():
                logging.warning("Found leftover batch job {} ({}) that need to be deleted".format(jobid, jobname))
                jobkills.append(jobid)

            case.cancel_batch_jobs(jobkills)
Example #25
0
    def test_error(self, strftime, get_value, flush, init):  # pylint: disable=unused-argument
        Case._force_read_only = False  # pylint: disable=protected-access

        with self.tempdir as tempdir, mock.patch("CIME.case.case.open",
                                                 mock.mock_open()) as m:
            m.side_effect = PermissionError()

            with Case(tempdir) as case:
                get_value.side_effect = [tempdir, "/src"]

                # We didn't need to make tempdir look like a valid case for the Case
                # constructor because we mock that constructor, but we *do* need to make
                # it look like a valid case for record_cmd.
                make_valid_case(tempdir)
                case.record_cmd()
Example #26
0
def build_data_lib(argv, compclass):
###############################################################################

    caseroot, libroot, _ = parse_input(argv)

    with Case(caseroot) as case:

        cimeroot  = case.get_value("CIMEROOT")

        # Write directory list (Filepath)
        compname = "d" + compclass
        with open('Filepath', 'w') as out:
            out.write(os.path.join(caseroot, "SourceMods", "src.{}\n".format(compname)) + "\n")
            out.write(os.path.join(cimeroot, "src", "components", "data_comps", compname) + "\n")

        # Build the component
        run_gmake(case, compclass, libroot)
Example #27
0
    def test_copy(self, getuser, getfqdn, configure, create_caseroot, # pylint: disable=unused-argument
                  apply_user_mods, set_lookup_value, lock_file, strftime, # pylint: disable=unused-argument
                  read_xml): # pylint: disable=unused-argument
        expected_first_hash = "134a939f62115fb44bf08a46bfb2bd13426833b5c8848cf7c4884af7af05b91a"
        expected_second_hash = "3561339a49daab999e3c4ea2f03a9c6acc33296a5bc35f1bfb82e7b5e10bdf38"

        with self.tempdir as tempdir:
            caseroot = os.path.join(tempdir, "test1")
            with Case(caseroot, read_only=False) as case:
                os.environ["CIME_MODEL"] = "cesm"

                srcroot = os.path.abspath(os.path.join(
                    os.path.dirname(__file__), "../../../../../"))
                case.create("test1", srcroot, "A", "f19_g16_rx1",
                            machine_name="ubuntu-latest")

                # Check that they're all called
                configure.assert_called_with("A", "f19_g16_rx1",
                                             machine_name="ubuntu-latest",
                                             project=None, pecount=None,
                                             compiler=None, mpilib=None,
                                             pesfile=None, gridfile=None,
                                             multi_driver=False, ninst=1,
                                             test=False, walltime=None,
                                             queue=None, output_root=None,
                                             run_unsupported=False, answer=None,
                                             input_dir=None, driver=None,
                                             workflowid="default",
                                             non_local=False,
                                             extra_machines_dir=None,
                                             case_group=None)
                create_caseroot.assert_called()
                apply_user_mods.assert_called()
                lock_file.assert_called()

                set_lookup_value.assert_called_with("CASE_HASH",
                                                    expected_first_hash)

                strftime.return_value = "10:00:00"
                with mock.patch("CIME.case.case.Case.set_value") as set_value, \
                        mock.patch("sys.argv", ["/src/create_clone"]):
                    case.copy("test2", "{}_2".format(tempdir))

                    set_value.assert_called_with("CASE_HASH",
                                                expected_second_hash)
Example #28
0
def build_stub_lib(argv, compclass):
###############################################################################

    caseroot, libroot, _ = parse_input(argv)

    with Case(caseroot) as case:

        cimeroot = case.get_value("CIMEROOT")

        # Write directory list
        compname = "s" + compclass
        with open('Filepath', 'w') as out:
            out.write(os.path.join(caseroot, "SourceMods", "src.{}", compname) + "\n")
            out.write(os.path.join(cimeroot, "src", "components", "stub_comps", "xshare") + "\n")
            out.write(os.path.join(cimeroot, "src", "components", "stub_comps",compname, "cpl") + "\n")

        # Build the component
        run_gmake(case, compclass, libroot)
Example #29
0
    def test_new_hash(self, getuser, getfqdn, strftime, read_xml): # pylint: disable=unused-argument
        with self.tempdir as tempdir:
            with Case(tempdir) as case:
                expected = "134a939f62115fb44bf08a46bfb2bd13426833b5c8848cf7c4884af7af05b91a"

                # Check idempotency
                for _ in range(2):
                    value = case.new_hash()

                    self.assertTrue(value == expected,
                                    "{} != {}".format(value, expected))

                expected = "bb59f1c473ac07e9dd30bfab153c0530a777f89280b716cf42e6fe2f49811a6e"

                value = case.new_hash()

                self.assertTrue(value == expected,
                                "{} != {}".format(value, expected))
Example #30
0
def _main_func(description):
    casename, _ = parse_command_line(sys.argv, description)
    caseroot = os.path.abspath(casename)

    #    forcing_dir = "/glade/collections/nmme/ncgd0022/jcaron/CFSv2/forcing_files/"
    forcing_dir = "/glade/scratch/ssfcst/forcing_files/"
    forcing_files = {
        "Precip": glob.glob(forcing_dir + "*Prec*.nc"),
        "Solar": glob.glob(forcing_dir + "*Solar*.nc"),
        "TPQW": glob.glob(forcing_dir + "*TPQW*.nc")
    }

    with Case(caseroot, read_only=False) as case:
        rundir = case.get_value("RUNDIR")
        for ftype in ["Solar", "Precip", "TPQW"]:
            forcing_files[ftype] = sorted(forcing_files[ftype], reverse=True)
            streamfile = os.path.join(rundir,
                                      "datm.streams.txt.CLMCRUNCEP." + ftype)
            newfile = os.path.join(caseroot,
                                   "user_" + os.path.basename(streamfile))
            #print("New forcing_files from {} to {}".format(forcing_files["Solar"][0],forcing_files["Solar"][-1]))
            with open(streamfile) as fin, open(newfile, "w") as fout:
                input_lines = fin.readlines()
                skip_line = False
                for line in input_lines:
                    if "</filePath>" in line:
                        skip_line = False
                    if skip_line:
                        continue
                    if "domain.lnd" in line:
                        line = "domain.ncepCFSv2.c2019.0.2d.nc\n"
                    if "clmforc.cruncep" in line or "clmforc.NCEPCFSv2" in line:
                        while forcing_files[ftype]:
                            newline = os.path.basename(
                                forcing_files[ftype].pop())
                            fout.write(newline + "\n")
                        continue
                    fout.write(line)
                    if "<filePath>" in line:
                        skip_line = True
                        fout.write("/glade/scratch/ssfcst/forcing_files/\n")
Example #31
0
def build_base_case(date,
                    baseroot,
                    basecasename,
                    basemonth,
                    res,
                    compset,
                    overwrite,
                    sdrestdir,
                    user_mods_dir,
                    pecount=None):
    caseroot = os.path.join(baseroot,
                            basecasename + ".{:02d}".format(basemonth) + ".00")
    if overwrite and os.path.isdir(caseroot):
        shutil.rmtree(caseroot)

    with Case(caseroot, read_only=False) as case:
        if not os.path.isdir(caseroot):
            case.create(os.path.basename(caseroot),
                        cesmroot,
                        compset,
                        res,
                        run_unsupported=True,
                        answer="r",
                        walltime="04:00:00",
                        user_mods_dir=user_mods_dir,
                        pecount=pecount,
                        project="NCGD0042")
            # make sure that changing the casename will not affect these variables
            case.set_value("EXEROOT", case.get_value("EXEROOT", resolved=True))
            case.set_value("RUNDIR",
                           case.get_value("RUNDIR", resolved=True) + ".00")

            case.set_value("RUN_TYPE", "hybrid")
            case.set_value("GET_REFCASE", False)
            case.set_value("RUN_REFDIR", sdrestdir)
            if basecasename == "70Lwaccm6":
                case.set_value("RUN_REFCASE",
                               "b.e21.BWHIST.SD.{}.002.nudgedOcn".format(res))
                case.set_value("OCN_TRACER_MODULES", "")
                case.set_value("NTHRDS", 1)
            else:
Example #32
0
    def test_cmd_arg(self, get_value, flush, init): # pylint: disable=unused-argument
        Case._force_read_only = False # pylint: disable=protected-access

        mocked_open = mock.mock_open()

        with self.tempdir as tempdir, mock.patch("CIME.case.case.open",
                                                 mocked_open):
            with Case(tempdir) as case:
                get_value.side_effect = [
                    tempdir,
                    "/src",
                ]

                case.record_cmd(["/some/custom/command", "arg1"])

        expected = [
            "/some/custom/command arg1\n\n",
        ]

        handle = mocked_open()
        handle.writelines.assert_called_with(expected)
Example #33
0
def preview_namelists(dryrun=False, case=None, casedir=None):
    if (case is None):
        if (casedir is None):
            case = Case()
        else:
            case = Case(case_root=casedir)

    # Get data from XML
    exeroot = case.get_value("EXEROOT")
    libroot = case.get_value("LIBROOT")
    incroot = case.get_value("INCROOT")
    rundir = case.get_value("RUNDIR")
    caseroot = case.get_value("CASEROOT")
    cimeroot = case.get_value("CIMEROOT")
    casebuild = case.get_value("CASEBUILD")
    testcase = case.get_value("TESTCASE")
    compiler = case.get_value("COMPILER")
    mach = case.get_value("MACH")
    debug = case.get_value("DEBUG")
    mpilib = case.get_value("MPILIB")

    logging.debug("LID is: '%s'" % os.getenv("LID", ""))
    logging.debug("caseroot is: '%s'" % caseroot)

    dryrun = True if (testcase == "SBN") else dryrun

    models = ["atm", "lnd", "ice", "ocn", "glc", "wav", "rof", "cpl"]
    docdir = os.path.join(caseroot, "CaseDocs")

    if (dryrun):
        # Only create rundir
        try:
            os.makedirs(rundir)
        except OSError:
            logging.warning("Not able to create $RUNDIR, trying a subdirectory of $CASEROOT")
            rundir = os.path.join(caseroot, rundir)
            try:
                os.makedirs(rundir)
                logging.info("Success! Setting RUNDIR=%s" % rundir)
                case.set_value("RUNDIR", rundir)
            except OSError:
                expect(False, "Could not create rundir")

    else:

        # Load modules
        env_module = EnvModule(mach, compiler, cimeroot, caseroot, mpilib, debug)
        env_module.load_env_for_case()

        # Make necessary directories
        dirs_to_make = [os.path.join(exeroot, model, "obj") for model in models]
        dirs_to_make.extend([exeroot, libroot, incroot, rundir, docdir])

        for dir_to_make in dirs_to_make:
            if (not os.path.isdir(dir_to_make)):
                try:
                    logging.debug("Making dir '%s'" % dir_to_make)
                    os.makedirs(dir_to_make)
                except OSError as e:
                    expect(False, "Could not make directory '%s', error: %s" % (dir_to_make, e))

    # Create namelists
    for model in models:
        model_str = "drv" if model == "cpl" else model
        config_file = case.get_value("CONFIG_%s_FILE" % model_str.upper())
        config_dir = os.path.dirname(config_file)
        cmd = os.path.join(config_dir, "buildnml")

        if (logging.getLogger().level == logging.DEBUG):
            run_cmd("PREVIEW_NML=1 %s %s" % (cmd, caseroot))
        else:
            run_cmd("%s %s" % (cmd, caseroot))

    # Save namelists to docdir
    if (not os.path.isdir(docdir)):
        os.makedirs(docdir)
        try:
            with open(os.path.join(docdir, "README"), "w") as fd:
                fd.write(" CESM Resolved Namelist Files\n   For documentation only DO NOT MODIFY\n")
        except (OSError, IOError) as e:
            expect(False, "Failed to write %s/README: %s" % (docdir, e))


    for cpglob in ["*_in_[0-9]*", "*modelio*", "*_in",
                   "*streams*txt*", "*stxt", "*maps.rc", "*cism.config*"]:
        for file_to_copy in glob.glob(os.path.join(rundir, cpglob)):
            logging.debug("Copy file from '%s' to '%s'" % (file_to_copy, docdir))
            shutil.copy2(file_to_copy, docdir)

    # Copy over chemistry mechanism docs if they exist
    if (os.path.isdir(os.path.join(casebuild, "camconf"))):
        for file_to_copy in glob.glob(os.path.join(casebuild, "camconf", "*chem_mech*")):
            shutil.copy2(file_to_copy, docdir)
Example #34
0
def case_build(caseroot, case=None, testmode=False, sharedlib_only=False, model_only=False):
###############################################################################
    t1 = time.time()

    expect(not (sharedlib_only and model_only),
           "Contradiction: both sharedlib_only and model_only")

    logger.info("sharedlib_only is %s" % sharedlib_only)
    logger.info("model_only is %s" % model_only)

    expect(os.path.isdir(caseroot), "'%s' is not a valid directory" % caseroot)
    os.chdir(caseroot)

    expect(os.path.exists("case.run"),
           "ERROR: must invoke case.setup script before calling build script ")

    case = Case() if case is None else case
    testcase = case.get_value("TESTCASE")
    cimeroot = case.get_value("CIMEROOT")
    expect(not (testcase is not None and
                os.path.exists("%s/scripts/Testing/Testcases/%s_build.csh" %
                               (cimeroot, testcase)) and not testmode),
           "%s build must be invoked via case.testbuild script" % testcase)

    if not sharedlib_only:
        check_all_input_data(case)

    run_cmd("./Tools/check_lockedfiles --caseroot %s" % caseroot)

    # Retrieve relevant case data
    build_threaded      = case.get_value("BUILD_THREADED")
    casetools           = case.get_value("CASETOOLS")
    exeroot             = case.get_value("EXEROOT")
    incroot             = case.get_value("INCROOT")
    libroot             = case.get_value("LIBROOT")
    sharedlibroot       = case.get_value("SHAREDLIBROOT")
    comp_atm            = case.get_value("COMP_ATM")
    comp_lnd            = case.get_value("COMP_LND")
    comp_ice            = case.get_value("COMP_ICE")
    comp_ocn            = case.get_value("COMP_OCN")
    comp_glc            = case.get_value("COMP_GLC")
    comp_wav            = case.get_value("COMP_WAV")
    comp_rof            = case.get_value("COMP_ROF")
    compiler            = case.get_value("COMPILER")
    comp_interface      = case.get_value("COMP_INTERFACE")
    mpilib              = case.get_value("MPILIB")
    use_esmf_lib        = case.get_value("USE_ESMF_LIB")
    debug               = case.get_value("DEBUG")
    ninst_build         = case.get_value("NINST_BUILD")
    smp_value           = case.get_value("SMP_VALUE")
    clm_use_petsc       = case.get_value("CLM_USE_PETSC")
    cism_use_trilinos   = case.get_value("CISM_USE_TRILINOS")
    mpasli_use_albany   = case.get_value("MPASLI_USE_ALBANY")
    clm_config_opts     = case.get_value("CLM_CONFIG_OPTS")
    cam_config_opts     = case.get_value("CAM_CONFIG_OPTS")
    pio_config_opts     = case.get_value("PIO_CONFIG_OPTS")
    ninst_value         = case.get_value("NINST_VALUE")
    mach                = case.get_value("MACH")
    os_                 = case.get_value("OS")
    comp_cpl            = case.get_value("COMP_CPL")
    machines_file       = case.get_value("MACHINES_SPEC_FILE")
    ocn_submodel        = case.get_value("OCN_SUBMODEL")
    profile_papi_enable = case.get_value("PROFILE_PAPI_ENABLE")
    nthrds_cpl          = int(case.get_value("NTHRDS_CPL"))
    nthrds_atm          = int(case.get_value("NTHRDS_ATM"))
    nthrds_lnd          = int(case.get_value("NTHRDS_LND"))
    nthrds_ice          = int(case.get_value("NTHRDS_ICE"))
    nthrds_ocn          = int(case.get_value("NTHRDS_OCN"))
    nthrds_glc          = int(case.get_value("NTHRDS_GLC"))
    nthrds_wav          = int(case.get_value("NTHRDS_WAV"))
    nthrds_rof          = int(case.get_value("NTHRDS_ROF"))

    # Load some params into env
    os.environ["CIMEROOT"]             = cimeroot
    os.environ["CASETOOLS"]            = casetools
    os.environ["EXEROOT"]              = exeroot
    os.environ["INCROOT"]              = incroot
    os.environ["LIBROOT"]              = libroot
    os.environ["SHAREDLIBROOT"]        = sharedlibroot
    os.environ["CASEROOT"]             = caseroot
    os.environ["COMPILER"]             = compiler
    os.environ["COMP_INTERFACE"]       = comp_interface
    os.environ["NINST_VALUE"]          = ninst_value
    os.environ["BUILD_THREADED"]       = build_threaded
    os.environ["MACH"]                 = mach
    os.environ["USE_ESMF_LIB"]         = use_esmf_lib
    os.environ["MPILIB"]               = mpilib
    os.environ["DEBUG"]                = debug
    os.environ["OS"]                   = os_
    os.environ["COMP_CPL"]             = comp_cpl
    os.environ["COMP_ATM"]             = comp_atm
    os.environ["COMP_LND"]             = comp_lnd
    os.environ["COMP_ICE"]             = comp_ice
    os.environ["COMP_OCN"]             = comp_ocn
    os.environ["COMP_GLC"]             = comp_glc
    os.environ["COMP_WAV"]             = comp_wav
    os.environ["COMP_ROF"]             = comp_rof
    os.environ["CLM_CONFIG_OPTS"]      = clm_config_opts     if clm_config_opts     is not None else ""
    os.environ["CAM_CONFIG_OPTS"]      = cam_config_opts     if cam_config_opts     is not None else ""
    os.environ["PIO_CONFIG_OPTS"]      = pio_config_opts     if pio_config_opts     is not None else ""
    os.environ["OCN_SUBMODEL"]         = ocn_submodel        if ocn_submodel        is not None else ""
    os.environ["PROFILE_PAPI_ENABLE"]  = profile_papi_enable if profile_papi_enable is not None else ""
    os.environ["CLM_USE_PETSC"]        = clm_use_petsc       if clm_use_petsc       is not None else ""
    os.environ["CISM_USE_TRILINOS"]    = cism_use_trilinos   if cism_use_trilinos   is not None else ""
    os.environ["MPASLI_USE_ALBANY"]    = mpasli_use_albany   if mpasli_use_albany   is not None else ""

    # This is a timestamp for the build , not the same as the testid,
    # and this case may not be a test anyway. For a production
    # experiment there may be many builds of the same case.
    lid               = run_cmd("date +%y%m%d-%H%M%S")
    os.environ["LID"] = lid

    # Set the overall USE_PETSC variable to TRUE if any of the
    # XXX_USE_PETSC variables are TRUE.
    # For now, there is just the one CLM_USE_PETSC variable, but in
    # the future there may be others -- so USE_PETSC will be true if
    # ANY of those are true.

    use_petsc = "TRUE" if clm_use_petsc == "TRUE" else "FALSE"
    case.set_value("USE_PETSC", use_petsc)
    os.environ["USE_PETSC"] = use_petsc

    # Set the overall USE_TRILINOS variable to TRUE if any of the
    # XXX_USE_TRILINOS variables are TRUE.
    # For now, there is just the one CISM_USE_TRILINOS variable, but in
    # the future there may be others -- so USE_TRILINOS will be true if
    # ANY of those are true.

    use_trilinos = "TRUE" if cism_use_trilinos == "TRUE" else "FALSE"
    case.set_value("USE_TRILINOS", use_trilinos)
    os.environ["USE_TRILINOS"] = use_trilinos

    # Set the overall USE_ALBANY variable to TRUE if any of the
    # XXX_USE_ALBANY variables are TRUE.
    # For now, there is just the one MPASLI_USE_ALBANY variable, but in
    # the future there may be others -- so USE_ALBANY will be true if
    # ANY of those are true.

    use_albany = "TRUE" if mpasli_use_albany == "TRUE" else "FALSE"
    case.set_value("USE_ALBANY", use_albany)
    os.environ["USE_ALBANY"] = use_albany

    # Load modules
    env_module = EnvModule(mach, compiler, cimeroot, caseroot, mpilib, debug)
    env_module.load_env_for_case()

    # Need to flush case xml to disk before calling preview_namelists
    case.flush()

    if not sharedlib_only:
        preview_namelists(case=case)

    build_checks(case, build_threaded, comp_interface, use_esmf_lib, debug, compiler, mpilib, 
                 sharedlibroot, nthrds_cpl, nthrds_atm, nthrds_lnd, nthrds_ice, nthrds_ocn, 
                 nthrds_glc, nthrds_wav, nthrds_rof, ninst_build, smp_value)

    t2 = time.time()
    logs = []

    if not model_only:
        logs = build_libraries(exeroot, caseroot, cimeroot, libroot, mpilib, lid, machines_file)
        if sharedlib_only and comp_lnd == "clm" and not "clm4_0" in clm_config_opts:
            logging.info("         - Building clm4_5/clm5_0 Library ")
            esmfdir = "esmf" if use_esmf_lib == "TRUE" else "noesmf"
            sharedpath = os.environ["SHAREDPATH"]
            bldroot = os.path.join(sharedpath, comp_interface, esmfdir)
            objdir = os.path.join(bldroot, "lnd", "obj")
            libdir = os.path.join(bldroot, "lib")
            file_build = os.path.join(exeroot, "lnd.bldlog.%s" %  lid)
            config_lnd_dir = os.path.dirname(case.get_value("CONFIG_LND_FILE"))
            results = []
            for ndir in [bldroot, objdir, libdir]:
                if(not os.path.isdir(ndir)):
                    os.makedirs(ndir)

            _build_model_thread(config_lnd_dir, caseroot, bldroot, "clm", file_build,
                                exeroot, "lnd", "clm", objdir, incroot, results)


    if not sharedlib_only:
        logs.extend(build_model(case, build_threaded, exeroot, clm_config_opts, incroot,
                                comp_atm,   comp_lnd,   comp_ice,   comp_ocn,   comp_glc,   comp_wav,   comp_rof,
                                nthrds_atm, nthrds_lnd, nthrds_ice, nthrds_ocn, nthrds_glc, nthrds_wav, nthrds_rof,
                                lid, caseroot, cimeroot, use_esmf_lib, comp_interface))

    if not sharedlib_only:
        post_build(case, logs)

    t3 = time.time()

    logger.info("Time spent not building: %f sec" % (t2 - t1))
    logger.info("Time spent building: %f sec" % (t3 - t2))
Example #35
0
def case_build(caseroot, case=None, sharedlib_only=False, model_only=False):
###############################################################################

    t1 = time.time()

    expect(not (sharedlib_only and model_only),
           "Contradiction: both sharedlib_only and model_only")

    logger.info("sharedlib_only is %s" % sharedlib_only)
    logger.info("model_only is %s" % model_only)

    expect(os.path.isdir(caseroot), "'%s' is not a valid directory" % caseroot)
    os.chdir(caseroot)

    expect(os.path.exists("case.run"),
           "ERROR: must invoke case.setup script before calling build script ")

    case = Case() if case is None else case
    cimeroot = case.get_value("CIMEROOT")

    comp_classes = case.get_value("COMP_CLASSES").split(',')

    if not sharedlib_only:
        check_all_input_data(case)

    run_cmd("./Tools/check_lockedfiles --caseroot %s" % caseroot)

    # Retrieve relevant case data
    # This environment variable gets set for cesm Make and
    # needs to be unset before building again.
    if "MODEL" in os.environ.keys():
        del os.environ["MODEL"]
    build_threaded      = case.get_value("BUILD_THREADED")
    casetools           = case.get_value("CASETOOLS")
    exeroot             = case.get_value("EXEROOT")
    incroot             = case.get_value("INCROOT")
    libroot             = case.get_value("LIBROOT")
    sharedlibroot       = case.get_value("SHAREDLIBROOT")

    complist = []
    for comp_class in comp_classes:
        if comp_class == "DRV":
            comp_class = "CPL"
            ninst = 1
            config_dir = None
        else:
            ninst = case.get_value("NINST_%s"%comp_class)
            config_dir = os.path.dirname(case.get_value("CONFIG_%s_FILE"%comp_class))

        comp = case.get_value("COMP_%s"%comp_class)
        thrds =  case.get_value("NTHRDS_%s"%comp_class)
        complist.append((comp_class.lower(), comp, thrds, ninst, config_dir ))
        os.environ["COMP_%s"%comp_class] = comp
    machines_file       = case.get_value("MACHINES_SPEC_FILE")
    ocn_submodel        = case.get_value("OCN_SUBMODEL")
    profile_papi_enable = case.get_value("PROFILE_PAPI_ENABLE")
    compiler            = case.get_value("COMPILER")
    comp_interface      = case.get_value("COMP_INTERFACE")
    mpilib              = case.get_value("MPILIB")
    use_esmf_lib        = case.get_value("USE_ESMF_LIB")
    debug               = case.get_value("DEBUG")
    ninst_build         = case.get_value("NINST_BUILD")
    smp_value           = case.get_value("SMP_VALUE")
    clm_use_petsc       = case.get_value("CLM_USE_PETSC")
    cism_use_trilinos   = case.get_value("CISM_USE_TRILINOS")
    mpasli_use_albany   = case.get_value("MPASLI_USE_ALBANY")
    clm_config_opts     = case.get_value("CLM_CONFIG_OPTS")
    cam_config_opts     = case.get_value("CAM_CONFIG_OPTS")
    pio_config_opts     = case.get_value("PIO_CONFIG_OPTS")
    ninst_value         = case.get_value("NINST_VALUE")
    mach                = case.get_value("MACH")
    os_                 = case.get_value("OS")
    # Load some params into env
    os.environ["CIMEROOT"]             = cimeroot
    os.environ["CASETOOLS"]            = casetools
    os.environ["EXEROOT"]              = exeroot
    os.environ["INCROOT"]              = incroot
    os.environ["LIBROOT"]              = libroot
    os.environ["SHAREDLIBROOT"]        = sharedlibroot
    os.environ["CASEROOT"]             = caseroot
    os.environ["COMPILER"]             = compiler
    os.environ["COMP_INTERFACE"]       = comp_interface
    os.environ["NINST_VALUE"]          = str(ninst_value)
    os.environ["BUILD_THREADED"]       = stringify_bool(build_threaded)
    os.environ["MACH"]                 = mach
    os.environ["USE_ESMF_LIB"]         = stringify_bool(use_esmf_lib)
    os.environ["MPILIB"]               = mpilib
    os.environ["DEBUG"]                = stringify_bool(debug)
    os.environ["OS"]                   = os_
    os.environ["CLM_CONFIG_OPTS"]      = clm_config_opts     if clm_config_opts     is not None else ""
    os.environ["CAM_CONFIG_OPTS"]      = cam_config_opts     if cam_config_opts     is not None else ""
    os.environ["PIO_CONFIG_OPTS"]      = pio_config_opts     if pio_config_opts     is not None else ""
    os.environ["OCN_SUBMODEL"]         = ocn_submodel        if ocn_submodel        is not None else ""
    os.environ["PROFILE_PAPI_ENABLE"]  = stringify_bool(profile_papi_enable) if profile_papi_enable is not None else ""
    os.environ["CLM_USE_PETSC"]        = stringify_bool(clm_use_petsc)       if clm_use_petsc       is not None else ""
    os.environ["CISM_USE_TRILINOS"]    = stringify_bool(cism_use_trilinos)   if cism_use_trilinos   is not None else ""
    os.environ["MPASLI_USE_ALBANY"]    = stringify_bool(mpasli_use_albany)   if mpasli_use_albany   is not None else ""

    # This is a timestamp for the build , not the same as the testid,
    # and this case may not be a test anyway. For a production
    # experiment there may be many builds of the same case.
    lid               = run_cmd("date +%y%m%d-%H%M%S")
    os.environ["LID"] = lid

    # Set the overall USE_PETSC variable to TRUE if any of the
    # XXX_USE_PETSC variables are TRUE.
    # For now, there is just the one CLM_USE_PETSC variable, but in
    # the future there may be others -- so USE_PETSC will be true if
    # ANY of those are true.

    use_petsc = clm_use_petsc
    case.set_value("USE_PETSC", use_petsc)
    os.environ["USE_PETSC"] = stringify_bool(use_petsc)

    # Set the overall USE_TRILINOS variable to TRUE if any of the
    # XXX_USE_TRILINOS variables are TRUE.
    # For now, there is just the one CISM_USE_TRILINOS variable, but in
    # the future there may be others -- so USE_TRILINOS will be true if
    # ANY of those are true.

    use_trilinos = False if cism_use_trilinos is None else cism_use_trilinos
    case.set_value("USE_TRILINOS", use_trilinos)
    os.environ["USE_TRILINOS"] = stringify_bool(use_trilinos)

    # Set the overall USE_ALBANY variable to TRUE if any of the
    # XXX_USE_ALBANY variables are TRUE.
    # For now, there is just the one MPASLI_USE_ALBANY variable, but in
    # the future there may be others -- so USE_ALBANY will be true if
    # ANY of those are true.

    use_albany = mpasli_use_albany
    case.set_value("USE_ALBANY", use_albany)
    os.environ["USE_ALBANY"] = stringify_bool(use_albany)

    # Load modules
    env_module = case._get_env("mach_specific")
    env_module.load_env_for_case(compiler=case.get_value("COMPILER"),
                                 debug=case.get_value("DEBUG"),
                                 mpilib=case.get_value("MPILIB"))


    # Need to flush case xml to disk before calling preview_namelists
    case.flush()

    if not sharedlib_only:
        preview_namelists(case)

    build_checks(case, build_threaded, comp_interface, use_esmf_lib, debug, compiler, mpilib,
                 sharedlibroot, complist, ninst_build, smp_value)

    t2 = time.time()
    logs = []

    if not model_only:
        logs = build_libraries(case, exeroot, caseroot, cimeroot, libroot, mpilib, lid,
                               machines_file)

    if not sharedlib_only:
        logs.extend(build_model(case, build_threaded, exeroot, clm_config_opts, incroot, complist,
                                lid, caseroot, cimeroot, use_esmf_lib, comp_interface))

    if not sharedlib_only:
        post_build(case, logs)

    t3 = time.time()

    logger.info("Time spent not building: %f sec" % (t2 - t1))
    logger.info("Time spent building: %f sec" % (t3 - t2))
Example #36
0
def build_data_nml(argv, compclass):
###############################################################################

    caseroot = parse_input(argv)

    case = Case(caseroot)

    cimeroot = case.get_value("CIMEROOT")
    rundir   = case.get_value("RUNDIR")
    ninst    = case.get_value("NINST_%s" % compclass.upper())

    compname = "d" + compclass

    confdir = os.path.join(caseroot,"Buildconf",compname + "conf")
    if not os.path.isdir(confdir):
        os.makedirs(confdir)

    inst_string = ""
    inst_counter = 1
    while (inst_counter <= ninst):

        # determine instance string
        inst_string = ""
        if ninst > 1:
            inst_string = '_' + '%04d' % inst_counter

            # If multi-instance case does not have restart file, use
            # single-case restart for each instance
            rpointer = "rpointer." + compname
            if (os.path.isfile(os.path.join(rundir,rpointer)) and
                (not os.path.isfile(os.path.join(rundir,rpointer+inst_string)))):
                     shutil.copy(os.path.isfile(os.path.join(rundir,rpointer),
                                                os.path.join(rundir,rpointer+inst_string)))

        # create namelist output infile using user_nl_file as input
        user_nl_file = os.path.join(caseroot, "user_nl_" + compname + inst_string)
        expect(os.path.isfile(user_nl_file),
               "Missing required user_nl_file %s " %(user_nl_file))
        namelist_infile = os.path.join(confdir, "cesm_namelist")
        create_namelist_infile(case, user_nl_file, namelist_infile)

        # call build-namelist
        user_xml_dir = os.path.join(caseroot, "SourceMods", "src." + compname)
        inst_string_label = inst_string
        if not inst_string_label:
            inst_string_label = "\"\""
        bldnamelist = os.path.join(cimeroot, "components", "data_comps", compname, "bld", "build-namelist")

        cmd = "%s -caseroot %s -cimeroot %s -inst_string %s -infile %s -user_xml_dir %s" \
            % (bldnamelist, caseroot, cimeroot, inst_string_label, namelist_infile, user_xml_dir)

        rc, out, err = run_cmd(cmd, from_dir=confdir, ok_to_fail=True)
        expect(rc==0,"Command %s failed rc=%d\nout=%s\nerr=%s"%(cmd,rc,out,err))

        # copy namelist files and stream text files, to rundir
        if os.path.isdir(rundir):
            filename = compname + "_in"
            file_src  = os.path.join(confdir, filename)
            file_dest = os.path.join(rundir, filename)
            if inst_string:
                file_dest += inst_string
            shutil.copy(file_src,file_dest)

            filename = compname + "_" + compclass + "_in"
            file_src  = os.path.join(confdir, filename)
            file_dest = os.path.join(rundir, filename)
            if inst_string:
                file_dest += inst_string
            shutil.copy(file_src,file_dest)

            for txtfile in glob.glob(os.path.join(confdir, "*txt*")):
                shutil.copy(txtfile, rundir)

        # increment instance counter
        inst_counter = inst_counter + 1
Example #37
0
def check_input_data(case=None, svn_loc=None, input_data_root=None, data_list_dir="Buildconf", download=False):
    """
    Return True if no files missing
    """
    # Fill in defaults as needed
    case = Case() if case is None else case
    svn_loc = SVN_LOCS[get_model()] if svn_loc is None else svn_loc
    input_data_root = case.get_value("DIN_LOC_ROOT") if input_data_root is None else input_data_root

    expect(os.path.isdir(input_data_root), "Invalid input_data_root directory: '%s'" % input_data_root)
    expect(os.path.isdir(data_list_dir), "Invalid data_list_dir directory: '%s'" % data_list_dir)

    data_list_files = find_files(data_list_dir, "*.input_data_list")
    expect(data_list_files, "No .input_data_list files found in dir '%s'" % data_list_dir)

    no_files_missing = True

    for data_list_file in data_list_files:
        logging.info("Loading input file: '%s'" % data_list_file)
        with open(data_list_file, "r") as fd:
            lines = fd.readlines()

        for line in lines:
            line = line.strip()
            if (line and not line.startswith("#")):
                tokens = line.split('=')
                description, full_path = tokens[0].strip(), tokens[1].strip()
                if(full_path):
                    # expand xml variables
                    full_path = case.get_resolved_value(full_path)
                    rel_path  = full_path.replace(input_data_root, "")

                    # There are some special values of rel_path that
                    # we need to ignore - some of the component models
                    # set things like 'NULL' or 'same_as_TS' -
                    # basically if rel_path does not contain '/' (a
                    # directory tree) you can assume it's a special
                    # value and ignore it (perhaps with a warning)
                    if ("/" in rel_path and not os.path.exists(full_path)):
                        model = os.path.basename(data_list_file).split('.')[0]
                        logging.warning("Model %s missing file %s = '%s'" % (model,description,full_path))

                        if (download):
                            success = download_if_in_repo(svn_loc, input_data_root, rel_path)
                            if (not success):
                                # If ACME, try CESM repo as backup
                                if (get_model() == "acme" and svn_loc != SVN_LOCS["cesm"]):
                                    success = download_if_in_repo(SVN_LOCS["cesm"], input_data_root, rel_path)
                                    if (not success):
                                        no_files_missing = False
                                else:
                                    no_files_missing = False
                            else:
                                no_files_missing = False
                        else:
                            logging.info("Already had input file: '%s'" % full_path)
                else:
                    model = os.path.basename(data_list_file).split('.')[0]
                    logging.warning("Model %s no file specified for %s"%(model,description))

    return no_files_missing
Example #38
0
    def _xml_phase(self, test):
    ###########################################################################
        test_case = CIME.utils.parse_test_name(test)[0]

        # Create, fill and write an envtest object
        test_dir = self._get_test_dir(test)
        envtest = EnvTest(test_dir)

        # Determine list of component classes that this coupler/driver knows how
        # to deal with. This list follows the same order as compset longnames follow.
        files = Files()
        drv_config_file = files.get_value("CONFIG_DRV_FILE")
        drv_comp = Component(drv_config_file)
        component_classes = drv_comp.get_valid_model_components()
        envtest.add_elements_by_group(drv_comp, {}, "env_test.xml")
        envtest.set_value("TESTCASE", test_case)
        envtest.set_value("TEST_TESTID", self._test_id)
        envtest.set_value("CASEBASEID", test)

        test_argv = "-testname %s -testroot %s" % (test, self._test_root)
        if self._generate:
            test_argv += " -generate %s" % self._baseline_gen_name
            envtest.set_value("BASELINE_NAME_GEN", self._baseline_gen_name)
            envtest.set_value("BASEGEN_CASE", os.path.join(self._baseline_gen_name, test))
        if self._compare:
            test_argv += " -compare %s" % self._baseline_cmp_name
            envtest.set_value("BASELINE_NAME_CMP", self._baseline_cmp_name)
            envtest.set_value("BASECMP_CASE", os.path.join(self._baseline_cmp_name, test))

        envtest.set_value("TEST_ARGV", test_argv)
        envtest.set_value("CLEANUP", self._clean)

        if self._generate or self._compare:
            envtest.set_value("BASELINE_ROOT", self._baseline_root)
        envtest.set_value("GENERATE_BASELINE", self._generate)
        envtest.set_value("COMPARE_BASELINE", self._compare)
        envtest.set_value("CCSM_CPRNC", self._machobj.get_value("CCSM_CPRNC", resolved=False))

        """
        Add the test instructions from config_test to env_test in the case
        """
        config_test = Tests()
        testnode = config_test.get_test_node(test_case)
        envtest.add_test(testnode)

        # Determine the test_case from the test name
        test_case, case_opts = CIME.utils.parse_test_name(test)[:2]

        # Determine case_opts from the test_case
        if case_opts is not None:
            logger.debug("case_opts are %s " %case_opts)
            for opt in case_opts:

                logger.debug("case_opt is %s" %opt)
                if opt == 'D':
                    envtest.set_test_parameter("DEBUG", "TRUE")
                    logger.debug (" DEBUG set to TRUE")

                elif opt == 'E':
                    envtest.set_test_parameter("USE_ESMF_LIB", "TRUE")
                    envtest.set_test_parameter("COMP_INTERFACE", "ESMF")
                    logger.debug (" USE_ESMF_LIB set to TRUE")
                    logger.debug (" COMP_INTERFACE set to ESMF")

                elif opt == 'CG':
                    envtest.set_test_parameter("CALENDAR", "GREGORIAN")
                    logger.debug (" CALENDAR set to %s" %opt)

                elif opt.startswith('L'):
                    match =  re.match('L([A-Za-z])([0-9]*)', opt)
                    stop_option = {"y":"nyears", "m":"nmonths", "d":"ndays", "h":"nhours",
                                   "s":"nseconds", "n":"nsteps"}
                    opt = match.group(1)
                    envtest.set_test_parameter("STOP_OPTION",stop_option[opt])
                    opti = match.group(2)
                    envtest.set_test_parameter("STOP_N", opti)
                    logger.debug (" STOP_OPTION set to %s" %stop_option[opt])
                    logger.debug (" STOP_N      set to %s" %opti)

                elif opt.startswith('M'):
                    # M option handled by create newcase
                    continue

                elif opt.startswith('P'):
                    match1 =  re.match('P([0-9]+)', opt)
                    match2 =  re.match('P([0-9]+)x([0-9]+)', opt)
                    match3 =  re.match('P[SMLX][12]?', opt)

                    opti_tasks = None
                    if match1:
                        opti_tasks = match1.group(1)
                        for component_class in component_classes:
                            if component_class == "DRV":
                                component_class = "CPL"
                            string = "NTASKS_" + component_class
                            envtest.set_test_parameter(string, opti_tasks)
                            string = "NTHRDS_" + component_class
                            envtest.set_test_parameter(string, str(1))
                            string = "ROOTPE_" + component_class
                            envtest.set_test_parameter(string, str(0))
                        opti_thrds = 1
                    elif match2:
                        opti_tasks = match2.group(1)
                        opti_thrds = match2.group(2)
                        for component_class in component_classes:
                            if component_class == "DRV":
                                component_class = "CPL"
                            string = "NTASKS_" + component_class
                            envtest.set_test_parameter(string, opti_tasks)
                            string = "NTHRDS_" + component_class
                            envtest.set_test_parameter(string, opti_thrds)
                            string = "ROOTPE_" + component_class
                            envtest.set_test_parameter(string, str(0))
                    elif match3:

                        # handled by create_newcase
                        continue
                    if not match3:
                        expect(opti_tasks is not None, "No match found for PE option %s"%opt)
                        logger.debug (" NTASKS_xxx set to %s" %opti_tasks)
                        logger.debug (" NTHRDS_xxx set to %s" %opti_thrds)
                        logger.debug (" ROOTPE_xxx set to %s 0")

                elif opt.startswith('N'):
                    # handled in create_newcase
                    continue
                elif opt.startswith('IOP'):
                    logger.warn("IOP test option not yet implemented")
                else:
                    expect(False, "Could not parse option '%s' " %opt)

        envtest.write()
        lockedfiles = os.path.join(test_dir, "LockedFiles")
        try:
            os.stat(lockedfiles)
        except:
            os.mkdir(lockedfiles)
        shutil.copy(os.path.join(test_dir,"env_run.xml"),
                    os.path.join(lockedfiles, "env_run.orig.xml"))

        case = Case(test_dir)
        case.set_value("SHAREDLIBROOT",
                       os.path.join(self._test_root,
                                    "sharedlibroot.%s"%self._test_id))

        envtest.set_initial_values(case)
        return True