def pre_run_check(case): ############################################################################### # Pre run initialization code.. caseroot = case.get_value("CASEROOT") din_loc_root = case.get_value("DIN_LOC_ROOT") batchsubmit = case.get_value("BATCHSUBMIT") mpilib = case.get_value("MPILIB") rundir = case.get_value("RUNDIR") build_complete = case.get_value("BUILD_COMPLETE") # check for locked files. check_lockedfiles(case.get_value("CASEROOT")) logger.debug("check_lockedfiles OK") # check that build is done expect(build_complete, "BUILD_COMPLETE is not true\nPlease rebuild the model interactively") logger.debug("build complete is %s " %build_complete) # load the module environment... case.load_env() # set environment variables # This is a requirement for yellowstone only if mpilib == "mpi-serial" and "MP_MPILIB" in os.environ: del os.environ["MP_MPILIB"] else: os.environ["MPILIB"] = mpilib if batchsubmit is None or len(batchsubmit) == 0: os.environ["LBQUERY"] = "FALSE" os.environ["BATCHQUERY"] = "undefined" elif batchsubmit == 'UNSET': os.environ["LBQUERY"] = "FALSE" os.environ["BATCHQUERY"] = "undefined" else: os.environ["LBQUERY"] = "TRUE" # create the timing directories, optionally cleaning them if needed. if not os.path.isdir(rundir): os.mkdir(rundir) if os.path.isdir(os.path.join(rundir, "timing")): shutil.rmtree(os.path.join(rundir, "timing")) os.makedirs(os.path.join(rundir, "timing", "checkpoints")) # This needs to be done everytime the LID changes in order for log files to be set up correctly create_namelists(case) # document process append_status("Run started ", caseroot=caseroot, sfile="CaseStatus") logger.info("-------------------------------------------------------------------------") logger.info(" - To prestage required restarts, untar a restart.tar file into %s" %(rundir)) logger.info(" - Case input data directory (DIN_LOC_ROOT) is %s " %(din_loc_root)) logger.info(" - Checking for required input datasets in DIN_LOC_ROOT") logger.info("-------------------------------------------------------------------------")
def case_cmpgen_namelists(case, compare=False, generate=False, compare_name=None, generate_name=None, baseline_root=None, logfile_name="TestStatus.log"): expect(case.get_value("TEST"), "Only makes sense to run this for a test case") caseroot, casebaseid = case.get_value("CASEROOT"), case.get_value("CASEBASEID") if not compare: compare = case.get_value("COMPARE_BASELINE") if not generate: generate = case.get_value("GENERATE_BASELINE") if not compare and not generate: logging.debug("No namelists compares requested") return True # create namelists for case if they haven't been already casedocs = os.path.join(caseroot, "CaseDocs") if not os.path.exists(os.path.join(casedocs, "drv_in")): create_namelists(case) test_name = casebaseid if casebaseid is not None else case.get_value("CASE") with TestStatus(test_dir=caseroot, test_name=test_name) as ts: try: # Inside this try are where we catch non-fatal errors, IE errors involving # baseline operations which may not directly impact the functioning of the viability of this case if compare and not compare_name: compare_name = case.get_value("BASELINE_NAME_CMP") expect(compare_name, "Was asked to do baseline compare but unable to determine baseline name") logging.info("Comparing namelists with baselines '{}'".format(compare_name)) if generate and not generate_name: generate_name = case.get_value("BASELINE_NAME_GEN") expect(generate_name, "Was asked to do baseline generation but unable to determine baseline name") logging.info("Generating namelists to baselines '{}'".format(generate_name)) success = True output = "" if compare: success, output = _do_full_nl_comp(case, test_name, compare_name, baseline_root) if not success and ts.get_status(RUN_PHASE) is not None: run_warn = \ """NOTE: It is not necessarily safe to compare namelists after RUN phase has completed. Running a case can pollute namelists. The namelists kept in the baselines are pre-RUN namelists.""" output += run_warn logging.info(run_warn) if generate: _do_full_nl_gen(case, test_name, generate_name, baseline_root) except: ts.set_status(NAMELIST_PHASE, TEST_FAIL_STATUS) success = False warn = "Exception during namelist operations:\n{}\n{}".format(sys.exc_info()[1], traceback.format_exc()) output += warn logging.warning(warn) finally: ts.set_status(NAMELIST_PHASE, TEST_PASS_STATUS if success else TEST_FAIL_STATUS) try: append_status(output, logfile_name, caseroot=caseroot) except IOError: pass return success
def check_case(case): check_lockedfiles(case) create_namelists(case) # Must be called before check_all_input_data logger.info("Checking that inputdata is available as part of case submission") check_all_input_data(case) expect(case.get_value("BUILD_COMPLETE"), "Build complete is " "not True please rebuild the model by calling case.build") logger.info("Check case OK")
def check_case(case, caseroot): check_lockedfiles(caseroot) create_namelists(case) # Must be called before check_all_input_data check_all_input_data(case) # Now that we have baselines, do baseline operations if case.get_value("TEST"): case_cmpgen_namelists(case) expect(case.get_value("BUILD_COMPLETE"), "Build complete is " "not True please rebuild the model by calling case.build") logger.info("Check case OK")
def case_cmpgen_namelists(case, compare=False, generate=False, compare_name=None, generate_name=None, baseline_root=None, logfile_name="TestStatus.log"): expect(case.get_value("TEST"), "Only makes sense to run this for a test case") caseroot, casebaseid = case.get_value("CASEROOT"), case.get_value("CASEBASEID") if not compare: compare = case.get_value("COMPARE_BASELINE") if not generate: generate = case.get_value("GENERATE_BASELINE") if not compare and not generate: logging.info("Nothing to do") return True # create namelists for case if they haven't been already casedocs = os.path.join(caseroot, "CaseDocs") if not os.path.exists(os.path.join(casedocs, "drv_in")): create_namelists(case) test_name = casebaseid if casebaseid is not None else case.get_value("CASE") with TestStatus(test_dir=caseroot, test_name=test_name) as ts: try: # Inside this try are where we catch non-fatal errors, IE errors involving # baseline operations which may not directly impact the functioning of the viability of this case if compare and not compare_name: compare_name = case.get_value("BASELINE_NAME_CMP") compare_name = get_current_branch() if compare_name is None else compare_name expect(compare_name, "Was asked to do baseline compare but unable to determine baseline name") logging.info("Comparing namelists with baselines '%s'" % compare_name) if generate and not generate_name: generate_name = case.get_value("BASELINE_NAME_GEN") generate_name = get_current_branch() if generate_name is None else generate_name expect(generate_name, "Was asked to do baseline generation but unable to determine baseline name") logging.info("Generating namelists to baselines '%s'" % generate_name) success = True output = "" if compare: success, output = _do_full_nl_comp(case, test_name, compare_name, baseline_root) if generate: _do_full_nl_gen(case, test_name, generate_name, baseline_root) except: ts.set_status(NAMELIST_PHASE, TEST_FAIL_STATUS) success = False warn = "Exception during namelist operations:\n%s\n%s" % (sys.exc_info()[1], traceback.format_exc()) output += warn logging.warning(warn) finally: ts.set_status(NAMELIST_PHASE, TEST_PASS_STATUS if success else TEST_FAIL_STATUS) append_status(output, caseroot=caseroot, sfile=logfile_name) return success
def build_checks(case, build_threaded, comp_interface, use_esmf_lib, debug, compiler, mpilib, complist, ninst_build, smp_value, model_only): ############################################################################### """ check if a build needs to be done and warn if a clean is warrented first returns the relative sharedpath directory for sharedlibraries """ ninst_value = case.get_value("NINST_VALUE") smp_build = case.get_value("SMP_BUILD") build_status = case.get_value("BUILD_STATUS") smpstr = "" inststr = "" for model, _, nthrds, ninst, _ in complist: if nthrds > 1: build_threaded = True if build_threaded: smpstr += "%s1"%model[0] else: smpstr += "%s0"%model[0] inststr += "%s%d"%(model[0],ninst) if build_threaded: os.environ["SMP"] = "TRUE" else: os.environ["SMP"] = "FALSE" case.set_value("SMP_VALUE", smpstr) os.environ["SMP_VALUE"] = smpstr case.set_value("NINST_VALUE", inststr) os.environ["NINST_VALUE"] = inststr debugdir = "debug" if debug else "nodebug" threaddir = "threads" if (os.environ["SMP"] == "TRUE" or build_threaded) else "nothreads" sharedpath = os.path.join(compiler, mpilib, debugdir, threaddir) logger.debug("compiler=%s mpilib=%s debugdir=%s threaddir=%s"% (compiler,mpilib,debugdir,threaddir)) expect( ninst_build == ninst_value or ninst_build == "0", """ ERROR, NINST VALUES HAVE CHANGED NINST_BUILD = %s NINST_VALUE = %s A manual clean of your obj directories is strongly recommended You should execute the following: ./case.build --clean Then rerun the build script interactively ---- OR ---- You can override this error message at your own risk by executing: ./xmlchange -file env_build.xml -id NINST_BUILD -val 0 Then rerun the build script interactively """ % (ninst_build, ninst_value)) expect( smp_build == smpstr or smp_build == "0", """ ERROR, SMP VALUES HAVE CHANGED SMP_BUILD = %s SMP_VALUE = %s smpstr = %s A manual clean of your obj directories is strongly recommended You should execute the following: ./case.build --clean Then rerun the build script interactively ---- OR ---- You can override this error message at your own risk by executing: ./xmlchange -file env_build.xml -id SMP_BUILD -val 0 Then rerun the build script interactively """ % (smp_build, smp_value, smpstr)) expect(build_status == 0, """ ERROR env_build HAS CHANGED A manual clean of your obj directories is required You should execute the following: ./case.build --clean-all """) expect(comp_interface != "ESMF" or use_esmf_lib, """ ERROR COMP_INTERFACE IS ESMF BUT USE_ESMF_LIB IS NOT TRUE SET USE_ESMF_LIB to TRUE with: ./xmlchange -file env_build.xml -id USE_ESMF_LIB -value TRUE """) expect(mpilib != "mpi-serial" or not use_esmf_lib, """ ERROR MPILIB is mpi-serial and USE_ESMF_LIB IS TRUE MPILIB can only be used with an ESMF library built with mpiuni on Set USE_ESMF_LIB to FALSE with ./xmlchange -file env_build.xml -id USE_ESMF_LIB -val FALSE ---- OR ---- Make sure the ESMF_LIBDIR used was built with mipuni (or change it to one that was) And comment out this if block in Tools/models_buildexe """) case.set_value("BUILD_COMPLETE", False) case.flush() if not model_only: create_namelists(case) return sharedpath
def _case_setup_impl(case, caseroot, clean=False, test_mode=False, reset=False): ############################################################################### os.chdir(caseroot) msg = "case.setup starting" append_status(msg, caseroot=caseroot, sfile="CaseStatus") cimeroot = get_cime_root(case) # Check that $DIN_LOC_ROOT exists - and abort if not a namelist compare tests din_loc_root = case.get_value("DIN_LOC_ROOT") testcase = case.get_value("TESTCASE") expect(not (not os.path.isdir(din_loc_root) and testcase != "SBN"), "inputdata root is not a directory: \"$din_loc_root\" ") # Check that userdefine settings are specified before expanding variable for vid, value in case: expect(not (type(value) is str and "USERDEFINED_required_build" in value), "Parameter '%s' must be defined" % vid) # Create batch script if reset or clean: # Clean batch script backup_dir = "PESetupHist/b.%s" % time.strftime("%y%m%d-%H%M%S") if not os.path.isdir(backup_dir): os.makedirs(backup_dir) # back up relevant files for fileglob in ["case.run", "env_build.xml", "env_mach_pes.xml", "Macros*"]: for filename in glob.glob(fileglob): shutil.copy(filename, backup_dir) if os.path.exists("case.run"): os.remove("case.run") # only do the following if are NOT in testmode if not test_mode: # rebuild the models (even on restart) case.set_value("BUILD_COMPLETE", False) # backup and then clean test script if os.path.exists("case.test"): shutil.copy("case.test", backup_dir) os.remove("case.test") logger.info("Successfully cleaned test script case.test") if os.path.exists("case.testdriver"): shutil.copy("case.testdriver", backup_dir) os.remove("case.testdriver") logger.info("Successfully cleaned test script case.testdriver") logger.info("Successfully cleaned batch script case.run") logger.info("Successfully cleaned batch script case.run") logger.info("Some files have been saved to %s" % backup_dir) msg = "case.setup clean complete" append_status(msg, caseroot=caseroot, sfile="CaseStatus") if not clean: case.load_env() models = case.get_values("COMP_CLASSES") mach = case.get_value("MACH") compiler = case.get_value("COMPILER") debug = case.get_value("DEBUG") mpilib = case.get_value("MPILIB") sysos = case.get_value("OS") expect(mach is not None, "xml variable MACH is not set") # creates the Macros.make, Depends.compiler, Depends.machine, Depends.machine.compiler # and env_mach_specific.xml if they don't already exist. if not os.path.isfile("Macros.make") or not os.path.isfile("env_mach_specific.xml"): configure(Machines(machine=mach), caseroot, ["Makefile"], compiler, mpilib, debug, sysos) # Set tasks to 1 if mpi-serial library if mpilib == "mpi-serial": for vid, value in case: if vid.startswith("NTASKS_") and value != 1: case.set_value(vid, 1) # Check ninst. # In CIME there can be multiple instances of each component model (an ensemble) NINST is the instance of that component. for comp in models: if comp == "DRV": continue ninst = case.get_value("NINST_%s" % comp) ntasks = case.get_value("NTASKS_%s" % comp) if ninst > ntasks: if ntasks == 1: case.set_value("NTASKS_%s" % comp, ninst) else: expect(False, "NINST_%s value %d greater than NTASKS_%s %d" % (comp, ninst, comp, ntasks)) if os.path.exists("case.run"): logger.info("Machine/Decomp/Pes configuration has already been done ...skipping") else: _check_pelayouts_require_rebuild(case, models) if os.path.exists("LockedFiles/env_build.xml"): os.remove("LockedFiles/env_build.xml") case.flush() check_lockedfiles() env_mach_pes = case.get_env("mach_pes") pestot = env_mach_pes.get_total_tasks(models) logger.debug("at update TOTALPES = %s"%pestot) case.set_value("TOTALPES", pestot) thread_count = env_mach_pes.get_max_thread_count(models) if thread_count > 1: case.set_value("BUILD_THREADED", True) expect(not (case.get_value("BUILD_THREADED") and compiler == "nag"), "it is not possible to run with OpenMP if using the NAG Fortran compiler") cost_pes = env_mach_pes.get_cost_pes(pestot, thread_count, machine=case.get_value("MACH")) case.set_value("COST_PES", cost_pes) # create batch file logger.info("Creating batch script case.run") # Use BatchFactory to get the appropriate instance of a BatchMaker, # use it to create our batch scripts env_batch = case.get_env("batch") num_nodes = env_mach_pes.get_total_nodes(pestot, thread_count) tasks_per_node = env_mach_pes.get_tasks_per_node(pestot, thread_count) for job in env_batch.get_jobs(): input_batch_script = os.path.join(case.get_value("MACHDIR"), env_batch.get_value('template', subgroup=job)) if job == "case.test" and testcase is not None and not test_mode: logger.info("Writing %s script" % job) testscript = os.path.join(cimeroot, "scripts", "Testing", "Testcases", "%s_script" % testcase) # Short term fix to be removed when csh tests are removed if not os.path.exists(testscript): env_batch.make_batch_script(input_batch_script, job, case, pestot, tasks_per_node, num_nodes, thread_count) elif job != "case.test": logger.info("Writing %s script from input template %s" % (job, input_batch_script)) env_batch.make_batch_script(input_batch_script, job, case, pestot, tasks_per_node, num_nodes, thread_count) # Make a copy of env_mach_pes.xml in order to be able # to check that it does not change once case.setup is invoked logger.info("Locking file env_mach_pes.xml") case.flush() logger.debug("at copy TOTALPES = %s"%case.get_value("TOTALPES")) shutil.copy("env_mach_pes.xml", "LockedFiles") # Create user_nl files for the required number of instances if not os.path.exists("user_nl_cpl"): logger.info("Creating user_nl_xxx files for components and cpl") # loop over models for model in models: comp = case.get_value("COMP_%s" % model) logger.info("Building %s usernl files"%model) _build_usernl_files(case, model, comp) if comp == "cism": run_cmd_no_fail("%s/../components/cism/cime_config/cism.template %s" % (cimeroot, caseroot)) _build_usernl_files(case, "drv", "cpl") # Create needed directories for case create_dirs(case) logger.info("If an old case build already exists, might want to run \'case.build --clean\' before building") # Create test script if appropriate # Short term fix to be removed when csh tests are removed if os.path.exists("env_test.xml"): if not os.path.exists("case.test"): logger.info("Starting testcase.setup") run_cmd_no_fail("./testcase.setup -caseroot %s" % caseroot) logger.info("Finished testcase.setup") # some tests need namelists created here (ERP) if test_mode: create_namelists(case) msg = "case.setup complete" append_status(msg, caseroot=caseroot, sfile="CaseStatus") # Record env information env_module = case.get_env("mach_specific") env_module.make_env_mach_specific_file(compiler, debug, mpilib, "sh") env_module.make_env_mach_specific_file(compiler, debug, mpilib, "csh") with open("software_environment.txt", "w") as f: f.write(env_module.list_modules()) run_cmd_no_fail("echo -e '\n' >> software_environment.txt && \ env >> software_environment.txt")
def _run_model_impl(case, lid, skip_pnl=False, da_cycle=0): ############################################################################### pre_run_check(case, lid, skip_pnl=skip_pnl, da_cycle=da_cycle) model = case.get_value("MODEL") # Set OMP_NUM_THREADS env_mach_pes = case.get_env("mach_pes") comp_classes = case.get_values("COMP_CLASSES") thread_count = env_mach_pes.get_max_thread_count(comp_classes) os.environ["OMP_NUM_THREADS"] = str(thread_count) # Run the model logger.info("{} MODEL EXECUTION BEGINS HERE".format(time.strftime("%Y-%m-%d %H:%M:%S"))) cmd = case.get_mpirun_cmd(job="case.run") cmd = case.get_resolved_value(cmd) logger.info("run command is {} ".format(cmd)) rundir = case.get_value("RUNDIR") loop = True while loop: loop = False stat = run_cmd(cmd, from_dir=rundir)[0] model_logfile = os.path.join(rundir, model + ".log." + lid) # Determine if failure was due to a failed node, if so, try to restart if stat != 0: node_fail_re = case.get_value("NODE_FAIL_REGEX") if node_fail_re: node_fail_regex = re.compile(node_fail_re) model_logfile = os.path.join(rundir, model + ".log." + lid) if os.path.exists(model_logfile): num_fails = len(node_fail_regex.findall(open(model_logfile, 'r').read())) if num_fails > 0 and case.spare_nodes >= num_fails: # We failed due to node failure! logger.warning("Detected model run failed due to node failure, restarting") # Archive the last consistent set of restart files and restore them case_st_archive(case, no_resubmit=True) restore_from_archive(case) orig_cont = case.get_value("CONTINUE_RUN") if not orig_cont: case.set_value("CONTINUE_RUN", True) create_namelists(case) lid = new_lid() loop = True case.spare_nodes -= num_fails if not loop: # We failed and we're not restarting expect(False, "RUN FAIL: Command '{}' failed\nSee log file for details: {}".format(cmd, model_logfile)) logger.info("{} MODEL EXECUTION HAS FINISHED".format(time.strftime("%Y-%m-%d %H:%M:%S"))) post_run_check(case, lid) return lid
def pre_run_check(case, lid, skip_pnl=False, da_cycle=0): ############################################################################### # Pre run initialization code.. if da_cycle > 0: create_namelists(case, component='cpl') return caseroot = case.get_value("CASEROOT") din_loc_root = case.get_value("DIN_LOC_ROOT") batchsubmit = case.get_value("BATCHSUBMIT") mpilib = case.get_value("MPILIB") rundir = case.get_value("RUNDIR") build_complete = case.get_value("BUILD_COMPLETE") if case.get_value("TESTCASE") == "PFS": env_mach_pes = os.path.join(caseroot,"env_mach_pes.xml") shutil.copy(env_mach_pes,"{}.{}".format(env_mach_pes, lid)) # check for locked files. check_lockedfiles(case) logger.debug("check_lockedfiles OK") # check that build is done expect(build_complete, "BUILD_COMPLETE is not true\nPlease rebuild the model interactively") logger.debug("build complete is {} ".format(build_complete)) # load the module environment... case.load_env() # set environment variables # This is a requirement for yellowstone only if mpilib == "mpi-serial" and "MP_MPILIB" in os.environ: del os.environ["MP_MPILIB"] else: os.environ["MPILIB"] = mpilib if batchsubmit is None or len(batchsubmit) == 0: os.environ["LBQUERY"] = "FALSE" os.environ["BATCHQUERY"] = "undefined" elif batchsubmit == 'UNSET': os.environ["LBQUERY"] = "FALSE" os.environ["BATCHQUERY"] = "undefined" else: os.environ["LBQUERY"] = "TRUE" # create the timing directories, optionally cleaning them if needed. if not os.path.isdir(rundir): os.mkdir(rundir) if os.path.isdir(os.path.join(rundir, "timing")): shutil.rmtree(os.path.join(rundir, "timing")) os.makedirs(os.path.join(rundir, "timing", "checkpoints")) # This needs to be done everytime the LID changes in order for log files to be set up correctly # The following also needs to be called in case a user changes a user_nl_xxx file OR an env_run.xml # variable while the job is in the queue if not skip_pnl: create_namelists(case) logger.info("-------------------------------------------------------------------------") logger.info(" - Prestage required restarts into {}".format(rundir)) logger.info(" - Case input data directory (DIN_LOC_ROOT) is {} ".format(din_loc_root)) logger.info(" - Checking for required input datasets in DIN_LOC_ROOT") logger.info("-------------------------------------------------------------------------")
def _build_checks(case, build_threaded, comp_interface, use_esmf_lib, debug, compiler, mpilib, complist, ninst_build, smp_value, model_only, buildlist): ############################################################################### """ check if a build needs to be done and warn if a clean is warrented first returns the relative sharedpath directory for sharedlibraries """ ninst_value = case.get_value("NINST_VALUE") smp_build = case.get_value("SMP_BUILD") build_status = case.get_value("BUILD_STATUS") expect(comp_interface == "mct", "Only supporting mct comp_interface at this time") smpstr = "" inststr = "" for model, _, nthrds, ninst, _ in complist: if nthrds > 1: build_threaded = True if build_threaded: smpstr += "{}1".format(model[0]) else: smpstr += "{}0".format(model[0]) inststr += "{}{:d}".format((model[0]),ninst) if build_threaded: os.environ["SMP"] = "TRUE" else: os.environ["SMP"] = "FALSE" case.set_value("SMP_VALUE", smpstr) os.environ["SMP_VALUE"] = smpstr case.set_value("NINST_VALUE", inststr) os.environ["NINST_VALUE"] = inststr debugdir = "debug" if debug else "nodebug" threaddir = "threads" if (os.environ["SMP"] == "TRUE" or build_threaded) else "nothreads" sharedpath = os.path.join(compiler, mpilib, debugdir, threaddir) logger.debug("compiler={} mpilib={} debugdir={} threaddir={}" .format(compiler,mpilib,debugdir,threaddir)) expect(ninst_build == ninst_value or ninst_build == "0", """ ERROR, NINST VALUES HAVE CHANGED NINST_BUILD = {} NINST_VALUE = {} A manual clean of your obj directories is strongly recommended You should execute the following: ./case.build --clean Then rerun the build script interactively ---- OR ---- You can override this error message at your own risk by executing: ./xmlchange -file env_build.xml -id NINST_BUILD -val 0 Then rerun the build script interactively """.format(ninst_build, ninst_value)) expect(smp_build == smpstr or smp_build == "0", """ ERROR, SMP VALUES HAVE CHANGED SMP_BUILD = {} SMP_VALUE = {} smpstr = {} A manual clean of your obj directories is strongly recommended You should execute the following: ./case.build --clean Then rerun the build script interactively ---- OR ---- You can override this error message at your own risk by executing: ./xmlchange -file env_build.xml -id SMP_BUILD -val 0 Then rerun the build script interactively """.format(smp_build, smp_value, smpstr)) expect(build_status == 0, """ ERROR env_build HAS CHANGED A manual clean of your obj directories is required You should execute the following: ./case.build --clean-all """) expect(mpilib != "mpi-serial" or not use_esmf_lib, """ ERROR MPILIB is mpi-serial and USE_ESMF_LIB IS TRUE MPILIB can only be used with an ESMF library built with mpiuni on Set USE_ESMF_LIB to FALSE with ./xmlchange -file env_build.xml -id USE_ESMF_LIB -val FALSE ---- OR ---- Make sure the ESMF_LIBDIR used was built with mipuni (or change it to one that was) And comment out this if block in Tools/models_buildexe """) case.set_value("BUILD_COMPLETE", False) # User may have rm -rf their build directory create_dirs(case) case.flush() if not model_only and not buildlist: logger.info("Generating component namelists as part of build") create_namelists(case) return sharedpath
def _case_setup_impl(case, caseroot, clean=False, test_mode=False, reset=False): ############################################################################### os.chdir(caseroot) # Check that $DIN_LOC_ROOT exists - and abort if not a namelist compare tests din_loc_root = case.get_value("DIN_LOC_ROOT") testcase = case.get_value("TESTCASE") expect(not (not os.path.isdir(din_loc_root) and testcase != "SBN"), "inputdata root is not a directory: {}".format(din_loc_root)) # Remove batch scripts if reset or clean: case_run, case_test = get_batch_script_for_job("case.run"), get_batch_script_for_job("case.test") if os.path.exists(case_run): os.remove(case_run) if not test_mode: # rebuild the models (even on restart) case.set_value("BUILD_COMPLETE", False) # backup and then clean test script if os.path.exists(case_test): os.remove(case_test) logger.info("Successfully cleaned test script {}".format(case_test)) logger.info("Successfully cleaned batch script case.run") if not clean: case.load_env() models = case.get_values("COMP_CLASSES") mach = case.get_value("MACH") compiler = case.get_value("COMPILER") debug = case.get_value("DEBUG") mpilib = case.get_value("MPILIB") sysos = case.get_value("OS") expect(mach is not None, "xml variable MACH is not set") # creates the Macros.make, Depends.compiler, Depends.machine, Depends.machine.compiler # and env_mach_specific.xml if they don't already exist. if not os.path.isfile("Macros.make") or not os.path.isfile("env_mach_specific.xml"): configure(Machines(machine=mach), caseroot, ["Makefile"], compiler, mpilib, debug, sysos) # Set tasks to 1 if mpi-serial library if mpilib == "mpi-serial": for vid, value in case: if vid.startswith("NTASKS") and value != 1: case.set_value(vid, 1) # Check ninst. # In CIME there can be multiple instances of each component model (an ensemble) NINST is the instance of that component. multi_driver = case.get_value("MULTI_DRIVER") nthrds = 1 for comp in models: ntasks = case.get_value("NTASKS_{}".format(comp)) nthrds = max(nthrds,case.get_value("NTHRDS_{}".format(comp))) if comp == "CPL": continue ninst = case.get_value("NINST_{}".format(comp)) if multi_driver: expect(case.get_value("NINST_LAYOUT_{}".format(comp)) == "concurrent", "If multi_driver is TRUE, NINST_LAYOUT_{} must be concurrent".format(comp)) case.set_value("NTASKS_PER_INST_{}".format(comp), ntasks) else: if ninst > ntasks: if ntasks == 1: case.set_value("NTASKS_{}".format(comp), ninst) ntasks = ninst else: expect(False, "NINST_{} value {:d} greater than NTASKS_{} {:d}".format(comp, ninst, comp, ntasks)) case.set_value("NTASKS_PER_INST_{}".format(comp), int(ntasks / ninst)) if nthrds > 1: case.set_value("BUILD_THREADED",True) if os.path.exists(get_batch_script_for_job("case.run")): logger.info("Machine/Decomp/Pes configuration has already been done ...skipping") case.initialize_derived_attributes() case.set_value("SMP_PRESENT", case.get_build_threaded()) else: check_pelayouts_require_rebuild(case, models) unlock_file("env_build.xml") unlock_file("env_batch.xml") case.flush() check_lockedfiles(case) case.initialize_derived_attributes() cost_per_node = 16 if case.get_value("MACH") == "yellowstone" else case.get_value("MAX_MPITASKS_PER_NODE") case.set_value("COST_PES", case.num_nodes * cost_per_node) case.set_value("TOTALPES", case.total_tasks) case.set_value("SMP_PRESENT", case.get_build_threaded()) # create batch files env_batch = case.get_env("batch") env_batch.make_all_batch_files(case, test_mode=test_mode) # May need to select new batch settings if pelayout changed (e.g. problem is now too big for prev-selected queue) env_batch.set_job_defaults([(("case.test" if case.get_value("TEST") else "case.run"), {})], case) case.schedule_rewrite(env_batch) # Make a copy of env_mach_pes.xml in order to be able # to check that it does not change once case.setup is invoked case.flush() logger.debug("at copy TOTALPES = {}".format(case.get_value("TOTALPES"))) lock_file("env_mach_pes.xml") lock_file("env_batch.xml") # Create user_nl files for the required number of instances if not os.path.exists("user_nl_cpl"): logger.info("Creating user_nl_xxx files for components and cpl") # loop over models for model in models: comp = case.get_value("COMP_{}".format(model)) logger.debug("Building {} usernl files".format(model)) _build_usernl_files(case, model, comp) if comp == "cism": glcroot = case.get_value("COMP_ROOT_DIR_GLC") run_cmd_no_fail("{}/cime_config/cism.template {}".format(glcroot, caseroot)) _build_usernl_files(case, "drv", "cpl") # Create needed directories for case create_dirs(case) logger.info("If an old case build already exists, might want to run \'case.build --clean\' before building") # Some tests need namelists created here (ERP) - so do this if we are in test mode if test_mode or get_model() == "e3sm": logger.info("Generating component namelists as part of setup") create_namelists(case) # Record env information env_module = case.get_env("mach_specific") env_module.make_env_mach_specific_file("sh", case) env_module.make_env_mach_specific_file("csh", case) env_module.save_all_env_info("software_environment.txt")