def preview_namelists(dryrun=False, case=None, casedir=None): if (case is None): if (casedir is None): case = Case() else: case = Case(case_root=casedir) # Get data from XML exeroot = case.get_value("EXEROOT") libroot = case.get_value("LIBROOT") incroot = case.get_value("INCROOT") rundir = case.get_value("RUNDIR") caseroot = case.get_value("CASEROOT") cimeroot = case.get_value("CIMEROOT") casebuild = case.get_value("CASEBUILD") testcase = case.get_value("TESTCASE") compiler = case.get_value("COMPILER") mach = case.get_value("MACH") debug = case.get_value("DEBUG") mpilib = case.get_value("MPILIB") logging.debug("LID is: '%s'" % os.getenv("LID", "")) logging.debug("caseroot is: '%s'" % caseroot) dryrun = True if (testcase == "SBN") else dryrun models = ["atm", "lnd", "ice", "ocn", "glc", "wav", "rof", "cpl"] docdir = os.path.join(caseroot, "CaseDocs") if (dryrun): # Only create rundir try: os.makedirs(rundir) except OSError: logging.warning("Not able to create $RUNDIR, trying a subdirectory of $CASEROOT") rundir = os.path.join(caseroot, rundir) try: os.makedirs(rundir) logging.info("Success! Setting RUNDIR=%s" % rundir) case.set_value("RUNDIR", rundir) except OSError: expect(False, "Could not create rundir") else: # Load modules env_module = EnvModule(mach, compiler, cimeroot, caseroot, mpilib, debug) env_module.load_env_for_case() # Make necessary directories dirs_to_make = [os.path.join(exeroot, model, "obj") for model in models] dirs_to_make.extend([exeroot, libroot, incroot, rundir, docdir]) for dir_to_make in dirs_to_make: if (not os.path.isdir(dir_to_make)): try: logging.debug("Making dir '%s'" % dir_to_make) os.makedirs(dir_to_make) except OSError as e: expect(False, "Could not make directory '%s', error: %s" % (dir_to_make, e)) # Create namelists for model in models: model_str = "drv" if model == "cpl" else model config_file = case.get_value("CONFIG_%s_FILE" % model_str.upper()) config_dir = os.path.dirname(config_file) cmd = os.path.join(config_dir, "buildnml") if (logging.getLogger().level == logging.DEBUG): run_cmd("PREVIEW_NML=1 %s %s" % (cmd, caseroot)) else: run_cmd("%s %s" % (cmd, caseroot)) # Save namelists to docdir if (not os.path.isdir(docdir)): os.makedirs(docdir) try: with open(os.path.join(docdir, "README"), "w") as fd: fd.write(" CESM Resolved Namelist Files\n For documentation only DO NOT MODIFY\n") except (OSError, IOError) as e: expect(False, "Failed to write %s/README: %s" % (docdir, e)) for cpglob in ["*_in_[0-9]*", "*modelio*", "*_in", "*streams*txt*", "*stxt", "*maps.rc", "*cism.config*"]: for file_to_copy in glob.glob(os.path.join(rundir, cpglob)): logging.debug("Copy file from '%s' to '%s'" % (file_to_copy, docdir)) shutil.copy2(file_to_copy, docdir) # Copy over chemistry mechanism docs if they exist if (os.path.isdir(os.path.join(casebuild, "camconf"))): for file_to_copy in glob.glob(os.path.join(casebuild, "camconf", "*chem_mech*")): shutil.copy2(file_to_copy, docdir)
def case_build(caseroot, case=None, testmode=False, sharedlib_only=False, model_only=False): ############################################################################### t1 = time.time() expect(not (sharedlib_only and model_only), "Contradiction: both sharedlib_only and model_only") logger.info("sharedlib_only is %s" % sharedlib_only) logger.info("model_only is %s" % model_only) expect(os.path.isdir(caseroot), "'%s' is not a valid directory" % caseroot) os.chdir(caseroot) expect(os.path.exists("case.run"), "ERROR: must invoke case.setup script before calling build script ") case = Case() if case is None else case testcase = case.get_value("TESTCASE") cimeroot = case.get_value("CIMEROOT") expect(not (testcase is not None and os.path.exists("%s/scripts/Testing/Testcases/%s_build.csh" % (cimeroot, testcase)) and not testmode), "%s build must be invoked via case.testbuild script" % testcase) if not sharedlib_only: check_all_input_data(case) run_cmd("./Tools/check_lockedfiles --caseroot %s" % caseroot) # Retrieve relevant case data build_threaded = case.get_value("BUILD_THREADED") casetools = case.get_value("CASETOOLS") exeroot = case.get_value("EXEROOT") incroot = case.get_value("INCROOT") libroot = case.get_value("LIBROOT") sharedlibroot = case.get_value("SHAREDLIBROOT") comp_atm = case.get_value("COMP_ATM") comp_lnd = case.get_value("COMP_LND") comp_ice = case.get_value("COMP_ICE") comp_ocn = case.get_value("COMP_OCN") comp_glc = case.get_value("COMP_GLC") comp_wav = case.get_value("COMP_WAV") comp_rof = case.get_value("COMP_ROF") compiler = case.get_value("COMPILER") comp_interface = case.get_value("COMP_INTERFACE") mpilib = case.get_value("MPILIB") use_esmf_lib = case.get_value("USE_ESMF_LIB") debug = case.get_value("DEBUG") ninst_build = case.get_value("NINST_BUILD") smp_value = case.get_value("SMP_VALUE") clm_use_petsc = case.get_value("CLM_USE_PETSC") cism_use_trilinos = case.get_value("CISM_USE_TRILINOS") mpasli_use_albany = case.get_value("MPASLI_USE_ALBANY") clm_config_opts = case.get_value("CLM_CONFIG_OPTS") cam_config_opts = case.get_value("CAM_CONFIG_OPTS") pio_config_opts = case.get_value("PIO_CONFIG_OPTS") ninst_value = case.get_value("NINST_VALUE") mach = case.get_value("MACH") os_ = case.get_value("OS") comp_cpl = case.get_value("COMP_CPL") machines_file = case.get_value("MACHINES_SPEC_FILE") ocn_submodel = case.get_value("OCN_SUBMODEL") profile_papi_enable = case.get_value("PROFILE_PAPI_ENABLE") nthrds_cpl = int(case.get_value("NTHRDS_CPL")) nthrds_atm = int(case.get_value("NTHRDS_ATM")) nthrds_lnd = int(case.get_value("NTHRDS_LND")) nthrds_ice = int(case.get_value("NTHRDS_ICE")) nthrds_ocn = int(case.get_value("NTHRDS_OCN")) nthrds_glc = int(case.get_value("NTHRDS_GLC")) nthrds_wav = int(case.get_value("NTHRDS_WAV")) nthrds_rof = int(case.get_value("NTHRDS_ROF")) # Load some params into env os.environ["CIMEROOT"] = cimeroot os.environ["CASETOOLS"] = casetools os.environ["EXEROOT"] = exeroot os.environ["INCROOT"] = incroot os.environ["LIBROOT"] = libroot os.environ["SHAREDLIBROOT"] = sharedlibroot os.environ["CASEROOT"] = caseroot os.environ["COMPILER"] = compiler os.environ["COMP_INTERFACE"] = comp_interface os.environ["NINST_VALUE"] = ninst_value os.environ["BUILD_THREADED"] = build_threaded os.environ["MACH"] = mach os.environ["USE_ESMF_LIB"] = use_esmf_lib os.environ["MPILIB"] = mpilib os.environ["DEBUG"] = debug os.environ["OS"] = os_ os.environ["COMP_CPL"] = comp_cpl os.environ["COMP_ATM"] = comp_atm os.environ["COMP_LND"] = comp_lnd os.environ["COMP_ICE"] = comp_ice os.environ["COMP_OCN"] = comp_ocn os.environ["COMP_GLC"] = comp_glc os.environ["COMP_WAV"] = comp_wav os.environ["COMP_ROF"] = comp_rof os.environ["CLM_CONFIG_OPTS"] = clm_config_opts if clm_config_opts is not None else "" os.environ["CAM_CONFIG_OPTS"] = cam_config_opts if cam_config_opts is not None else "" os.environ["PIO_CONFIG_OPTS"] = pio_config_opts if pio_config_opts is not None else "" os.environ["OCN_SUBMODEL"] = ocn_submodel if ocn_submodel is not None else "" os.environ["PROFILE_PAPI_ENABLE"] = profile_papi_enable if profile_papi_enable is not None else "" os.environ["CLM_USE_PETSC"] = clm_use_petsc if clm_use_petsc is not None else "" os.environ["CISM_USE_TRILINOS"] = cism_use_trilinos if cism_use_trilinos is not None else "" os.environ["MPASLI_USE_ALBANY"] = mpasli_use_albany if mpasli_use_albany is not None else "" # This is a timestamp for the build , not the same as the testid, # and this case may not be a test anyway. For a production # experiment there may be many builds of the same case. lid = run_cmd("date +%y%m%d-%H%M%S") os.environ["LID"] = lid # Set the overall USE_PETSC variable to TRUE if any of the # XXX_USE_PETSC variables are TRUE. # For now, there is just the one CLM_USE_PETSC variable, but in # the future there may be others -- so USE_PETSC will be true if # ANY of those are true. use_petsc = "TRUE" if clm_use_petsc == "TRUE" else "FALSE" case.set_value("USE_PETSC", use_petsc) os.environ["USE_PETSC"] = use_petsc # Set the overall USE_TRILINOS variable to TRUE if any of the # XXX_USE_TRILINOS variables are TRUE. # For now, there is just the one CISM_USE_TRILINOS variable, but in # the future there may be others -- so USE_TRILINOS will be true if # ANY of those are true. use_trilinos = "TRUE" if cism_use_trilinos == "TRUE" else "FALSE" case.set_value("USE_TRILINOS", use_trilinos) os.environ["USE_TRILINOS"] = use_trilinos # Set the overall USE_ALBANY variable to TRUE if any of the # XXX_USE_ALBANY variables are TRUE. # For now, there is just the one MPASLI_USE_ALBANY variable, but in # the future there may be others -- so USE_ALBANY will be true if # ANY of those are true. use_albany = "TRUE" if mpasli_use_albany == "TRUE" else "FALSE" case.set_value("USE_ALBANY", use_albany) os.environ["USE_ALBANY"] = use_albany # Load modules env_module = EnvModule(mach, compiler, cimeroot, caseroot, mpilib, debug) env_module.load_env_for_case() # Need to flush case xml to disk before calling preview_namelists case.flush() if not sharedlib_only: preview_namelists(case=case) build_checks(case, build_threaded, comp_interface, use_esmf_lib, debug, compiler, mpilib, sharedlibroot, nthrds_cpl, nthrds_atm, nthrds_lnd, nthrds_ice, nthrds_ocn, nthrds_glc, nthrds_wav, nthrds_rof, ninst_build, smp_value) t2 = time.time() logs = [] if not model_only: logs = build_libraries(exeroot, caseroot, cimeroot, libroot, mpilib, lid, machines_file) if sharedlib_only and comp_lnd == "clm" and not "clm4_0" in clm_config_opts: logging.info(" - Building clm4_5/clm5_0 Library ") esmfdir = "esmf" if use_esmf_lib == "TRUE" else "noesmf" sharedpath = os.environ["SHAREDPATH"] bldroot = os.path.join(sharedpath, comp_interface, esmfdir) objdir = os.path.join(bldroot, "lnd", "obj") libdir = os.path.join(bldroot, "lib") file_build = os.path.join(exeroot, "lnd.bldlog.%s" % lid) config_lnd_dir = os.path.dirname(case.get_value("CONFIG_LND_FILE")) results = [] for ndir in [bldroot, objdir, libdir]: if(not os.path.isdir(ndir)): os.makedirs(ndir) _build_model_thread(config_lnd_dir, caseroot, bldroot, "clm", file_build, exeroot, "lnd", "clm", objdir, incroot, results) if not sharedlib_only: logs.extend(build_model(case, build_threaded, exeroot, clm_config_opts, incroot, comp_atm, comp_lnd, comp_ice, comp_ocn, comp_glc, comp_wav, comp_rof, nthrds_atm, nthrds_lnd, nthrds_ice, nthrds_ocn, nthrds_glc, nthrds_wav, nthrds_rof, lid, caseroot, cimeroot, use_esmf_lib, comp_interface)) if not sharedlib_only: post_build(case, logs) t3 = time.time() logger.info("Time spent not building: %f sec" % (t2 - t1)) logger.info("Time spent building: %f sec" % (t3 - t2))
def case_build(caseroot, case=None, sharedlib_only=False, model_only=False): ############################################################################### t1 = time.time() expect(not (sharedlib_only and model_only), "Contradiction: both sharedlib_only and model_only") logger.info("sharedlib_only is %s" % sharedlib_only) logger.info("model_only is %s" % model_only) expect(os.path.isdir(caseroot), "'%s' is not a valid directory" % caseroot) os.chdir(caseroot) expect(os.path.exists("case.run"), "ERROR: must invoke case.setup script before calling build script ") case = Case() if case is None else case cimeroot = case.get_value("CIMEROOT") comp_classes = case.get_value("COMP_CLASSES").split(',') if not sharedlib_only: check_all_input_data(case) run_cmd("./Tools/check_lockedfiles --caseroot %s" % caseroot) # Retrieve relevant case data # This environment variable gets set for cesm Make and # needs to be unset before building again. if "MODEL" in os.environ.keys(): del os.environ["MODEL"] build_threaded = case.get_value("BUILD_THREADED") casetools = case.get_value("CASETOOLS") exeroot = case.get_value("EXEROOT") incroot = case.get_value("INCROOT") libroot = case.get_value("LIBROOT") sharedlibroot = case.get_value("SHAREDLIBROOT") complist = [] for comp_class in comp_classes: if comp_class == "DRV": comp_class = "CPL" ninst = 1 config_dir = None else: ninst = case.get_value("NINST_%s"%comp_class) config_dir = os.path.dirname(case.get_value("CONFIG_%s_FILE"%comp_class)) comp = case.get_value("COMP_%s"%comp_class) thrds = case.get_value("NTHRDS_%s"%comp_class) complist.append((comp_class.lower(), comp, thrds, ninst, config_dir )) os.environ["COMP_%s"%comp_class] = comp machines_file = case.get_value("MACHINES_SPEC_FILE") ocn_submodel = case.get_value("OCN_SUBMODEL") profile_papi_enable = case.get_value("PROFILE_PAPI_ENABLE") compiler = case.get_value("COMPILER") comp_interface = case.get_value("COMP_INTERFACE") mpilib = case.get_value("MPILIB") use_esmf_lib = case.get_value("USE_ESMF_LIB") debug = case.get_value("DEBUG") ninst_build = case.get_value("NINST_BUILD") smp_value = case.get_value("SMP_VALUE") clm_use_petsc = case.get_value("CLM_USE_PETSC") cism_use_trilinos = case.get_value("CISM_USE_TRILINOS") mpasli_use_albany = case.get_value("MPASLI_USE_ALBANY") clm_config_opts = case.get_value("CLM_CONFIG_OPTS") cam_config_opts = case.get_value("CAM_CONFIG_OPTS") pio_config_opts = case.get_value("PIO_CONFIG_OPTS") ninst_value = case.get_value("NINST_VALUE") mach = case.get_value("MACH") os_ = case.get_value("OS") # Load some params into env os.environ["CIMEROOT"] = cimeroot os.environ["CASETOOLS"] = casetools os.environ["EXEROOT"] = exeroot os.environ["INCROOT"] = incroot os.environ["LIBROOT"] = libroot os.environ["SHAREDLIBROOT"] = sharedlibroot os.environ["CASEROOT"] = caseroot os.environ["COMPILER"] = compiler os.environ["COMP_INTERFACE"] = comp_interface os.environ["NINST_VALUE"] = str(ninst_value) os.environ["BUILD_THREADED"] = stringify_bool(build_threaded) os.environ["MACH"] = mach os.environ["USE_ESMF_LIB"] = stringify_bool(use_esmf_lib) os.environ["MPILIB"] = mpilib os.environ["DEBUG"] = stringify_bool(debug) os.environ["OS"] = os_ os.environ["CLM_CONFIG_OPTS"] = clm_config_opts if clm_config_opts is not None else "" os.environ["CAM_CONFIG_OPTS"] = cam_config_opts if cam_config_opts is not None else "" os.environ["PIO_CONFIG_OPTS"] = pio_config_opts if pio_config_opts is not None else "" os.environ["OCN_SUBMODEL"] = ocn_submodel if ocn_submodel is not None else "" os.environ["PROFILE_PAPI_ENABLE"] = stringify_bool(profile_papi_enable) if profile_papi_enable is not None else "" os.environ["CLM_USE_PETSC"] = stringify_bool(clm_use_petsc) if clm_use_petsc is not None else "" os.environ["CISM_USE_TRILINOS"] = stringify_bool(cism_use_trilinos) if cism_use_trilinos is not None else "" os.environ["MPASLI_USE_ALBANY"] = stringify_bool(mpasli_use_albany) if mpasli_use_albany is not None else "" # This is a timestamp for the build , not the same as the testid, # and this case may not be a test anyway. For a production # experiment there may be many builds of the same case. lid = run_cmd("date +%y%m%d-%H%M%S") os.environ["LID"] = lid # Set the overall USE_PETSC variable to TRUE if any of the # XXX_USE_PETSC variables are TRUE. # For now, there is just the one CLM_USE_PETSC variable, but in # the future there may be others -- so USE_PETSC will be true if # ANY of those are true. use_petsc = clm_use_petsc case.set_value("USE_PETSC", use_petsc) os.environ["USE_PETSC"] = stringify_bool(use_petsc) # Set the overall USE_TRILINOS variable to TRUE if any of the # XXX_USE_TRILINOS variables are TRUE. # For now, there is just the one CISM_USE_TRILINOS variable, but in # the future there may be others -- so USE_TRILINOS will be true if # ANY of those are true. use_trilinos = False if cism_use_trilinos is None else cism_use_trilinos case.set_value("USE_TRILINOS", use_trilinos) os.environ["USE_TRILINOS"] = stringify_bool(use_trilinos) # Set the overall USE_ALBANY variable to TRUE if any of the # XXX_USE_ALBANY variables are TRUE. # For now, there is just the one MPASLI_USE_ALBANY variable, but in # the future there may be others -- so USE_ALBANY will be true if # ANY of those are true. use_albany = mpasli_use_albany case.set_value("USE_ALBANY", use_albany) os.environ["USE_ALBANY"] = stringify_bool(use_albany) # Load modules env_module = case._get_env("mach_specific") env_module.load_env_for_case(compiler=case.get_value("COMPILER"), debug=case.get_value("DEBUG"), mpilib=case.get_value("MPILIB")) # Need to flush case xml to disk before calling preview_namelists case.flush() if not sharedlib_only: preview_namelists(case) build_checks(case, build_threaded, comp_interface, use_esmf_lib, debug, compiler, mpilib, sharedlibroot, complist, ninst_build, smp_value) t2 = time.time() logs = [] if not model_only: logs = build_libraries(case, exeroot, caseroot, cimeroot, libroot, mpilib, lid, machines_file) if not sharedlib_only: logs.extend(build_model(case, build_threaded, exeroot, clm_config_opts, incroot, complist, lid, caseroot, cimeroot, use_esmf_lib, comp_interface)) if not sharedlib_only: post_build(case, logs) t3 = time.time() logger.info("Time spent not building: %f sec" % (t2 - t1)) logger.info("Time spent building: %f sec" % (t3 - t2))
def _xml_phase(self, test): ########################################################################### test_case = CIME.utils.parse_test_name(test)[0] # Create, fill and write an envtest object test_dir = self._get_test_dir(test) envtest = EnvTest(test_dir) # Determine list of component classes that this coupler/driver knows how # to deal with. This list follows the same order as compset longnames follow. files = Files() drv_config_file = files.get_value("CONFIG_DRV_FILE") drv_comp = Component(drv_config_file) component_classes = drv_comp.get_valid_model_components() envtest.add_elements_by_group(drv_comp, {}, "env_test.xml") envtest.set_value("TESTCASE", test_case) envtest.set_value("TEST_TESTID", self._test_id) envtest.set_value("CASEBASEID", test) test_argv = "-testname %s -testroot %s" % (test, self._test_root) if self._generate: test_argv += " -generate %s" % self._baseline_gen_name envtest.set_value("BASELINE_NAME_GEN", self._baseline_gen_name) envtest.set_value("BASEGEN_CASE", os.path.join(self._baseline_gen_name, test)) if self._compare: test_argv += " -compare %s" % self._baseline_cmp_name envtest.set_value("BASELINE_NAME_CMP", self._baseline_cmp_name) envtest.set_value("BASECMP_CASE", os.path.join(self._baseline_cmp_name, test)) envtest.set_value("TEST_ARGV", test_argv) envtest.set_value("CLEANUP", self._clean) if self._generate or self._compare: envtest.set_value("BASELINE_ROOT", self._baseline_root) envtest.set_value("GENERATE_BASELINE", self._generate) envtest.set_value("COMPARE_BASELINE", self._compare) envtest.set_value("CCSM_CPRNC", self._machobj.get_value("CCSM_CPRNC", resolved=False)) """ Add the test instructions from config_test to env_test in the case """ config_test = Tests() testnode = config_test.get_test_node(test_case) envtest.add_test(testnode) # Determine the test_case from the test name test_case, case_opts = CIME.utils.parse_test_name(test)[:2] # Determine case_opts from the test_case if case_opts is not None: logger.debug("case_opts are %s " %case_opts) for opt in case_opts: logger.debug("case_opt is %s" %opt) if opt == 'D': envtest.set_test_parameter("DEBUG", "TRUE") logger.debug (" DEBUG set to TRUE") elif opt == 'E': envtest.set_test_parameter("USE_ESMF_LIB", "TRUE") envtest.set_test_parameter("COMP_INTERFACE", "ESMF") logger.debug (" USE_ESMF_LIB set to TRUE") logger.debug (" COMP_INTERFACE set to ESMF") elif opt == 'CG': envtest.set_test_parameter("CALENDAR", "GREGORIAN") logger.debug (" CALENDAR set to %s" %opt) elif opt.startswith('L'): match = re.match('L([A-Za-z])([0-9]*)', opt) stop_option = {"y":"nyears", "m":"nmonths", "d":"ndays", "h":"nhours", "s":"nseconds", "n":"nsteps"} opt = match.group(1) envtest.set_test_parameter("STOP_OPTION",stop_option[opt]) opti = match.group(2) envtest.set_test_parameter("STOP_N", opti) logger.debug (" STOP_OPTION set to %s" %stop_option[opt]) logger.debug (" STOP_N set to %s" %opti) elif opt.startswith('M'): # M option handled by create newcase continue elif opt.startswith('P'): match1 = re.match('P([0-9]+)', opt) match2 = re.match('P([0-9]+)x([0-9]+)', opt) match3 = re.match('P[SMLX][12]?', opt) opti_tasks = None if match1: opti_tasks = match1.group(1) for component_class in component_classes: if component_class == "DRV": component_class = "CPL" string = "NTASKS_" + component_class envtest.set_test_parameter(string, opti_tasks) string = "NTHRDS_" + component_class envtest.set_test_parameter(string, str(1)) string = "ROOTPE_" + component_class envtest.set_test_parameter(string, str(0)) opti_thrds = 1 elif match2: opti_tasks = match2.group(1) opti_thrds = match2.group(2) for component_class in component_classes: if component_class == "DRV": component_class = "CPL" string = "NTASKS_" + component_class envtest.set_test_parameter(string, opti_tasks) string = "NTHRDS_" + component_class envtest.set_test_parameter(string, opti_thrds) string = "ROOTPE_" + component_class envtest.set_test_parameter(string, str(0)) elif match3: # handled by create_newcase continue if not match3: expect(opti_tasks is not None, "No match found for PE option %s"%opt) logger.debug (" NTASKS_xxx set to %s" %opti_tasks) logger.debug (" NTHRDS_xxx set to %s" %opti_thrds) logger.debug (" ROOTPE_xxx set to %s 0") elif opt.startswith('N'): # handled in create_newcase continue elif opt.startswith('IOP'): logger.warn("IOP test option not yet implemented") else: expect(False, "Could not parse option '%s' " %opt) envtest.write() lockedfiles = os.path.join(test_dir, "LockedFiles") try: os.stat(lockedfiles) except: os.mkdir(lockedfiles) shutil.copy(os.path.join(test_dir,"env_run.xml"), os.path.join(lockedfiles, "env_run.orig.xml")) case = Case(test_dir) case.set_value("SHAREDLIBROOT", os.path.join(self._test_root, "sharedlibroot.%s"%self._test_id)) envtest.set_initial_values(case) return True