def __init__(self, case, remove_dead_tasks=False): self.case = case self.remove_dead_tasks = remove_dead_tasks drv_comp = Component() models = drv_comp.get_valid_model_components() for item in ["COMP", "NTASKS", "NTHRDS", "ROOTPE", "NINST", "PSTRID"]: values = [] for model in models: model = "CPL" if model == "DRV" else model if item == "NINST" and model == "CPL": values.append(0) # no NINST for CPL, set to zero so lists are some length for all items else: vid = "_".join([item, model]) value = case.get_value(vid) setattr(self, vid, value) values.append(value) setattr(self, item, values) for item in [ "MAX_TASKS_PER_NODE", "PES_PER_NODE", "PIO_NUMTASKS", "PIO_ASYNC_INTERFACE", "EXEROOT", "COMPILER", ]: setattr(self, item, case.get_value(item)) self.DEFAULT_RUN_EXE_TEMPLATE_STR = "__DEFAULT_RUN_EXE__" self.MAX_TASKS_PER_NODE = 1 if self.MAX_TASKS_PER_NODE < 1 else self.MAX_TASKS_PER_NODE self._compute_values()
def saveLogs(case, lid): ############################################################################### logdir = case.get_value("LOGDIR") if logdir is not None and len(logdir) > 0: if not os.path.isdir(logdir): os.makedirs(logdir) caseroot = case.get_value("CASEROOT") rundir = case.get_value("RUNDIR") # get components files = Files() config_file = files.get_value("CONFIG_DRV_FILE") component = Component(config_file) comps = [x.lower() for x in component.get_valid_model_components()] comps = [x.replace('drv', 'cpl') for x in comps] model = [case.get_value("MODEL")] comps = comps + model # for each component, compress log files and copy to logdir for comp in comps: logfile = os.path.join(rundir, comp + '.log.' + lid) if os.path.isfile(logfile): f_in = open(logfile) f_out = gzip.open(logfile + '.gz', 'wb') f_out.writelines(f_in) f_out.close() f_in.close() os.remove(logfile) logfile_copy = logfile + '.gz' shutil.copy(logfile_copy, os.path.join(caseroot, logdir, os.path.basename(logfile_copy)))
def get_components(files): """ Determine the valid component classes (e.g. atm) for the driver/cpl These are then stored in comps_array """ infile = files.get_value("CONFIG_CPL_FILE") config_drv = Component(infile, "CPL") return config_drv.get_valid_model_components()
def _get_component_config_data(self): # attributes used for multi valued defaults ($attlist is a hash reference) attlist = {"compset": self._compsetname, "grid": self._gridname} # Determine list of component classes that this coupler/driver knows how # to deal with. This list follows the same order as compset longnames follow. files = Files() # Add the group and elements for the config_files.xml for env_file in self._env_entryid_files: env_file.add_elements_by_group(files, attlist) drv_config_file = files.get_value("CONFIG_DRV_FILE") drv_comp = Component(drv_config_file) for env_file in self._env_entryid_files: env_file.add_elements_by_group(drv_comp, attributes=attlist) # loop over all elements of both component_classes and components - and get config_component_file for # for each component self._component_classes = drv_comp.get_valid_model_components() if len(self._component_classes) > len(self._components): self._components.append('sesp') for i in xrange(1, len(self._component_classes)): comp_class = self._component_classes[i] comp_name = self._components[i - 1] node_name = 'CONFIG_' + comp_class + '_FILE' # Add the group and elements for the config_files.xml comp_config_file = files.get_value(node_name, {"component": comp_name}, resolved=False) self.set_value(node_name, comp_config_file) comp_config_file = self.get_resolved_value(comp_config_file) expect(comp_config_file is not None, "No config file for component %s" % comp_name) compobj = Component(comp_config_file) for env_file in self._env_entryid_files: env_file.add_elements_by_group(compobj, attributes=attlist) for key, value in self.lookups.items(): result = self.set_value(key, value) if result is not None: del self.lookups[key]
def _get_component_config_data(self): # attributes used for multi valued defaults ($attlist is a hash reference) attlist = {"compset":self._compsetname, "grid":self._gridname} # Determine list of component classes that this coupler/driver knows how # to deal with. This list follows the same order as compset longnames follow. files = Files() # Add the group and elements for the config_files.xml for env_file in self._env_entryid_files: env_file.add_elements_by_group(files, attlist) drv_config_file = files.get_value("CONFIG_DRV_FILE") drv_comp = Component(drv_config_file) for env_file in self._env_entryid_files: env_file.add_elements_by_group(drv_comp, attributes=attlist) # loop over all elements of both component_classes and components - and get config_component_file for # for each component self._component_classes =drv_comp.get_valid_model_components() if len(self._component_classes) > len(self._components): self._components.append('sesp') for i in xrange(1,len(self._component_classes)): comp_class = self._component_classes[i] comp_name = self._components[i-1] node_name = 'CONFIG_' + comp_class + '_FILE' # Add the group and elements for the config_files.xml comp_config_file = files.get_value(node_name, {"component":comp_name}, resolved=False) self.set_value(node_name, comp_config_file) comp_config_file = self.get_resolved_value(comp_config_file) expect(comp_config_file is not None,"No config file for component %s"%comp_name) compobj = Component(comp_config_file) for env_file in self._env_entryid_files: env_file.add_elements_by_group(compobj, attributes=attlist) for key,value in self.lookups.items(): result = self.set_value(key,value) if result is not None: del self.lookups[key]
def get_components(self): """ return dictionary of the form [component_class:component], e.g. [atm:cam], for all compset components """ files = Files() drv_comp = Component(files.get_value("CONFIG_DRV_FILE")) # Determine list of component classes that this coupler/driver knows how # to deal with. This list follows the same order as compset longnames follow. component_classes = drv_comp.get_valid_model_components() components = self.get_compset_components() # Note that component classes can have a bigger range than # compents since stub esp (sesp) is an optional component - so # need to take the min of the two below comp_dict = {} for i in xrange(0,len(components)): comp_name = components[i] comp_class = component_classes[i+1] comp_dict[comp_class] = comp_name return comp_dict
def get_components(self): """ return dictionary of the form [component_class:component], e.g. [atm:cam], for all compset components """ files = Files() drv_comp = Component(files.get_value("CONFIG_DRV_FILE")) # Determine list of component classes that this coupler/driver knows how # to deal with. This list follows the same order as compset longnames follow. component_classes = drv_comp.get_valid_model_components() components = self.get_compset_components() # Note that component classes can have a bigger range than # compents since stub esp (sesp) is an optional component - so # need to take the min of the two below comp_dict = {} for i in xrange(0, len(components)): comp_name = components[i] comp_class = component_classes[i + 1] comp_dict[comp_class] = comp_name return comp_dict
def _case_setup_impl(case, caseroot, casebaseid, clean=False, test_mode=False, reset=False): ############################################################################### os.chdir(caseroot) msg = "case.setup starting" append_status(msg, caseroot=caseroot, sfile="CaseStatus") cimeroot = os.environ["CIMEROOT"] # Check that $DIN_LOC_ROOT exists - and abort if not a namelist compare tests din_loc_root = case.get_value("DIN_LOC_ROOT") testcase = case.get_value("TESTCASE") expect(not (not os.path.isdir(din_loc_root) and testcase != "SBN"), "inputdata root is not a directory: \"$din_loc_root\" ") # Check that userdefine settings are specified before expanding variable for vid, value in case: expect(not (type(value) is str and "USERDEFINED_required_build" in value), "Parameter '%s' must be defined" % vid) # Create batch script if reset or clean: # Clean batch script backup_dir = "PESetupHist/b.%s" % time.strftime("%y%m%d-%H%M%S") if not os.path.isdir(backup_dir): os.makedirs(backup_dir) # back up relevant files for fileglob in ["case.run", "env_build.xml", "env_mach_pes.xml", "Macros*"]: for filename in glob.glob(fileglob): shutil.copy(filename, backup_dir) if os.path.exists("case.run"): os.remove("case.run") # only do the following if are NOT in testmode if not test_mode: # rebuild the models (even on restart) case.set_value("BUILD_COMPLETE", False) # backup and then clean test script if os.path.exists("case.test"): shutil.copy("case.test", backup_dir) os.remove("case.test") logger.info("Successfully cleaned test script case.test") if os.path.exists("case.testdriver"): shutil.copy("case.testdriver", backup_dir) os.remove("case.testdriver") logger.info("Successfully cleaned test script case.testdriver") logger.info("Successfully cleaned batch script case.run") logger.info("Successfully cleaned batch script case.run") logger.info("Some files have been saved to %s" % backup_dir) msg = "case.setup clean complete" append_status(msg, caseroot=caseroot, sfile="CaseStatus") if not clean: drv_comp = Component() models = drv_comp.get_valid_model_components() models.remove("DRV") mach, compiler, debug, mpilib = \ case.get_value("MACH"), case.get_value("COMPILER"), case.get_value("DEBUG"), case.get_value("MPILIB") expect(mach is not None, "xml variable MACH is not set") # Create Macros file only if it does not exist if not os.path.exists("Macros"): logger.debug("Creating Macros file for %s" % mach) compilers = Compilers(compiler=compiler, machine=mach, os_=case.get_value("OS"), mpilib=mpilib) compilers.write_macros_file() else: logger.debug("Macros script already created ...skipping") # Set tasks to 1 if mpi-serial library if mpilib == "mpi-serial": for vid, value in case: if vid.startswith("NTASKS_") and value != 1: case.set_value(vid, 1) # Check ninst. # In CIME there can be multiple instances of each component model (an ensemble) NINST is the instance of that component. # Save ninst in a dict to use later in apply_user_mods ninst = dict() for comp in models: comp_model = case.get_value("COMP_%s" % comp) ninst[comp_model] = case.get_value("NINST_%s" % comp) ntasks = case.get_value("NTASKS_%s" % comp) if ninst[comp_model] > ntasks: if ntasks == 1: case.set_value("NTASKS_%s" % comp, ninst[comp_model]) else: expect(False, "NINST_%s value %d greater than NTASKS_%s %d" % (comp, ninst[comp_model], comp, ntasks)) expect(not (case.get_value("BUILD_THREADED") and compiler == "nag"), "it is not possible to run with OpenMP if using the NAG Fortran compiler") if os.path.exists("case.run"): logger.info("Machine/Decomp/Pes configuration has already been done ...skipping") else: _check_pelayouts_require_rebuild(case, models) if os.path.exists("LockedFiles/env_build.xml"): os.remove("LockedFiles/env_build.xml") case.flush() check_lockedfiles() tm = TaskMaker(case) mtpn = case.get_value("MAX_TASKS_PER_NODE") pespn = case.get_value("PES_PER_NODE") # This is hardcoded because on yellowstone by default we # run with 15 pes per node # but pay for 16 pes per node. See github issue #518 if case.get_value("MACH") == "yellowstone": pespn = 16 pestot = tm.totaltasks if mtpn > pespn: pestot = pestot * (mtpn // pespn) case.set_value("COST_PES", tm.num_nodes*pespn) else: # reset cost_pes to totalpes case.set_value("COST_PES", 0) case.set_value("TOTALPES", pestot) # Compute cost based on PE count pval = 1 pcnt = 0 while pval < pestot: pval *= 2 pcnt += 6 # (scaling like sqrt(6/10)) pcost = 3 - pcnt / 10 # (3 is 64 with 6) # Compute cost based on DEBUG dcost = 3 if debug else 0 # Compute cost based on run length # For simplicity, we use a heuristic just based on STOP_OPTION (not considering # STOP_N), and only deal with options longer than ndays lcost = 0 if "nmonth" in case.get_value("STOP_OPTION"): # N months costs 30x as much as N days; since cost is based on log-base-2, add 5 lcost = 5 elif "nyear" in case.get_value("STOP_OPTION"): # N years costs 365x as much as N days; since cost is based on log-base-2, add 9 lcost = 9 estcost = pcost + dcost + lcost for cost in ["CCSM_CCOST", "CCSM_GCOST", "CCSM_TCOST", "CCSM_CCOST"]: estcost += case.get_value(cost) case.set_value("CCSM_PCOST", pcost) case.set_value("CCSM_ESTCOST", estcost) # create batch file logger.info("Creating batch script case.run") # Use BatchFactory to get the appropriate instance of a BatchMaker, # use it to create our batch scripts env_batch = case.get_env("batch") for job in env_batch.get_jobs(): input_batch_script = os.path.join(case.get_value("MACHDIR"), env_batch.get_value('template', subgroup=job)) if job == "case.test" and testcase is not None and not test_mode: logger.info("Writing %s script" % job) testscript = os.path.join(cimeroot, "scripts", "Testing", "Testcases", "%s_script" % testcase) # Short term fix to be removed when csh tests are removed if not os.path.exists(testscript): env_batch.make_batch_script(input_batch_script, job, case) elif job != "case.test": logger.info("Writing %s script" % job) env_batch.make_batch_script(input_batch_script, job, case) # Make a copy of env_mach_pes.xml in order to be able # to check that it does not change once case.setup is invoked logger.info("Locking file env_mach_pes.xml") case.flush() shutil.copy("env_mach_pes.xml", "LockedFiles") # Create user_nl files for the required number of instances if not os.path.exists("user_nl_cpl"): logger.info("Creating user_nl_xxx files for components and cpl") # loop over models for model in models: comp = case.get_value("COMP_%s" % model) logger.info("Building %s usernl files"%model) _build_usernl_files(case, model, comp) if comp == "cism": run_cmd_no_fail("%s/../components/cism/cime_config/cism.template %s" % (cimeroot, caseroot)) _build_usernl_files(case, "drv", "cpl") user_mods_path = case.get_value("USER_MODS_FULLPATH") if user_mods_path is not None: apply_user_mods(caseroot, user_mods_path=user_mods_path, ninst=ninst) elif case.get_value("TEST"): test_mods = parse_test_name(casebaseid)[6] if test_mods is not None: user_mods_path = os.path.join(case.get_value("TESTS_MODS_DIR"), test_mods) apply_user_mods(caseroot, user_mods_path=user_mods_path, ninst=ninst) # Run preview namelists for scripts logger.info("preview_namelists") preview_namelists(case) logger.info("See ./CaseDoc for component namelists") logger.info("If an old case build already exists, might want to run \'case.build --clean\' before building") # Create test script if appropriate # Short term fix to be removed when csh tests are removed if os.path.exists("env_test.xml"): if not os.path.exists("case.test"): logger.info("Starting testcase.setup") run_cmd_no_fail("./testcase.setup -caseroot %s" % caseroot) logger.info("Finished testcase.setup") msg = "case.setup complete" append_status(msg, caseroot=caseroot, sfile="CaseStatus") # Record env information env_module = case.get_env("mach_specific") env_module.make_env_mach_specific_file(compiler, debug, mpilib, "sh") env_module.make_env_mach_specific_file(compiler, debug, mpilib, "csh") with open("software_environment.txt", "w") as f: f.write(env_module.list_modules()) run_cmd_no_fail("echo -e '\n' >> software_environment.txt && \ env >> software_environment.txt")
def _get_component_config_data(self): # attributes used for multi valued defaults # attlist is a dictionary used to determine the value element that has the most matches attlist = { "compset": self._compsetname, "grid": self._gridname, "cime_model": self._cime_model } # Determine list of component classes that this coupler/driver knows how # to deal with. This list follows the same order as compset longnames follow. files = Files() # Add the group and elements for the config_files.xml for env_file in self._env_entryid_files: env_file.add_elements_by_group(files, attlist) drv_config_file = files.get_value("CONFIG_CPL_FILE") drv_comp = Component(drv_config_file) for env_file in self._env_entryid_files: env_file.add_elements_by_group(drv_comp, attributes=attlist) # Add the group and elements for env_batch env_batch = self.get_env("batch") env_batch.add_elements_by_group(drv_comp, attributes=attlist) # loop over all elements of both component_classes and components - and get config_component_file for # for each component self._set_comp_classes(drv_comp.get_valid_model_components()) if len(self._component_classes) > len(self._components): self._components.append('sesp') # put anything in the lookups table into env objects for key, value in self.lookups.items(): result = self.set_value(key, value) if result is not None: del self.lookups[key] for i in xrange(1, len(self._component_classes)): comp_class = self._component_classes[i] comp_name = self._components[i - 1] node_name = 'CONFIG_' + comp_class + '_FILE' # Add the group and elements for the config_files.xml comp_config_file = files.get_value(node_name, {"component": comp_name}, resolved=False) self.set_value(node_name, comp_config_file) comp_config_file = self.get_resolved_value(comp_config_file) expect( comp_config_file is not None and os.path.isfile(comp_config_file), "Config file %s for component %s not found." % (comp_config_file, comp_name)) compobj = Component(comp_config_file) for env_file in self._env_entryid_files: env_file.add_elements_by_group(compobj, attributes=attlist) # final cleanup of lookups table for key, value in self.lookups.items(): result = self.set_value(key, value) if result is not None: del self.lookups[key]
def _xml_phase(self, test): ########################################################################### test_case = CIME.utils.parse_test_name(test)[0] # Create, fill and write an envtest object test_dir = self._get_test_dir(test) envtest = EnvTest(test_dir) # Determine list of component classes that this coupler/driver knows how # to deal with. This list follows the same order as compset longnames follow. files = Files() drv_config_file = files.get_value("CONFIG_DRV_FILE") drv_comp = Component(drv_config_file) component_classes = drv_comp.get_valid_model_components() envtest.add_elements_by_group(drv_comp, {}, "env_test.xml") envtest.set_value("TESTCASE", test_case) envtest.set_value("TEST_TESTID", self._test_id) envtest.set_value("CASEBASEID", test) test_argv = "-testname %s -testroot %s" % (test, self._test_root) if self._generate: test_argv += " -generate %s" % self._baseline_gen_name envtest.set_value("BASELINE_NAME_GEN", self._baseline_gen_name) envtest.set_value("BASEGEN_CASE", os.path.join(self._baseline_gen_name, test)) if self._compare: test_argv += " -compare %s" % self._baseline_cmp_name envtest.set_value("BASELINE_NAME_CMP", self._baseline_cmp_name) envtest.set_value("BASECMP_CASE", os.path.join(self._baseline_cmp_name, test)) envtest.set_value("TEST_ARGV", test_argv) envtest.set_value("CLEANUP", self._clean) if self._generate or self._compare: envtest.set_value("BASELINE_ROOT", self._baseline_root) envtest.set_value("GENERATE_BASELINE", self._generate) envtest.set_value("COMPARE_BASELINE", self._compare) envtest.set_value("CCSM_CPRNC", self._machobj.get_value("CCSM_CPRNC", resolved=False)) """ Add the test instructions from config_test to env_test in the case """ config_test = Tests() testnode = config_test.get_test_node(test_case) envtest.add_test(testnode) # Determine the test_case from the test name test_case, case_opts = CIME.utils.parse_test_name(test)[:2] # Determine case_opts from the test_case if case_opts is not None: logger.debug("case_opts are %s " %case_opts) for opt in case_opts: logger.debug("case_opt is %s" %opt) if opt == 'D': envtest.set_test_parameter("DEBUG", "TRUE") logger.debug (" DEBUG set to TRUE") elif opt == 'E': envtest.set_test_parameter("USE_ESMF_LIB", "TRUE") envtest.set_test_parameter("COMP_INTERFACE", "ESMF") logger.debug (" USE_ESMF_LIB set to TRUE") logger.debug (" COMP_INTERFACE set to ESMF") elif opt == 'CG': envtest.set_test_parameter("CALENDAR", "GREGORIAN") logger.debug (" CALENDAR set to %s" %opt) elif opt.startswith('L'): match = re.match('L([A-Za-z])([0-9]*)', opt) stop_option = {"y":"nyears", "m":"nmonths", "d":"ndays", "h":"nhours", "s":"nseconds", "n":"nsteps"} opt = match.group(1) envtest.set_test_parameter("STOP_OPTION",stop_option[opt]) opti = match.group(2) envtest.set_test_parameter("STOP_N", opti) logger.debug (" STOP_OPTION set to %s" %stop_option[opt]) logger.debug (" STOP_N set to %s" %opti) elif opt.startswith('M'): # M option handled by create newcase continue elif opt.startswith('P'): match1 = re.match('P([0-9]+)', opt) match2 = re.match('P([0-9]+)x([0-9]+)', opt) match3 = re.match('P[SMLX][12]?', opt) opti_tasks = None if match1: opti_tasks = match1.group(1) for component_class in component_classes: if component_class == "DRV": component_class = "CPL" string = "NTASKS_" + component_class envtest.set_test_parameter(string, opti_tasks) string = "NTHRDS_" + component_class envtest.set_test_parameter(string, str(1)) string = "ROOTPE_" + component_class envtest.set_test_parameter(string, str(0)) opti_thrds = 1 elif match2: opti_tasks = match2.group(1) opti_thrds = match2.group(2) for component_class in component_classes: if component_class == "DRV": component_class = "CPL" string = "NTASKS_" + component_class envtest.set_test_parameter(string, opti_tasks) string = "NTHRDS_" + component_class envtest.set_test_parameter(string, opti_thrds) string = "ROOTPE_" + component_class envtest.set_test_parameter(string, str(0)) elif match3: # handled by create_newcase continue if not match3: expect(opti_tasks is not None, "No match found for PE option %s"%opt) logger.debug (" NTASKS_xxx set to %s" %opti_tasks) logger.debug (" NTHRDS_xxx set to %s" %opti_thrds) logger.debug (" ROOTPE_xxx set to %s 0") elif opt.startswith('N'): # handled in create_newcase continue elif opt.startswith('IOP'): logger.warn("IOP test option not yet implemented") else: expect(False, "Could not parse option '%s' " %opt) envtest.write() lockedfiles = os.path.join(test_dir, "LockedFiles") try: os.stat(lockedfiles) except: os.mkdir(lockedfiles) shutil.copy(os.path.join(test_dir,"env_run.xml"), os.path.join(lockedfiles, "env_run.orig.xml")) case = Case(test_dir) case.set_value("SHAREDLIBROOT", os.path.join(self._test_root, "sharedlibroot.%s"%self._test_id)) envtest.set_initial_values(case) return True